diff options
498 files changed, 21541 insertions, 27414 deletions
diff --git a/.travis.yml b/.travis.yml index 9ad7dfb19..8b336e7f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,17 @@ language: python python: - "2.6" - - "2.7" + - "2.7_with_system_site_packages" env: - WITH_OPTIONAL_DEPS=yes - WITH_OPTIONAL_DEPS=no -before_install: - - testsuite/before_install.sh install: - testsuite/install.sh - pip install --use-mirrors -e . script: - nosetests testsuite +after_failure: + - pip freeze branches: except: - maint-1.2 @@ -157,7 +157,8 @@ add themselves to this file. See LICENSE for the full license. - Zach Lowry <zach@mcs.anl.gov> wrote Solaris support and general hardening. -- Michael Fenn <fennm@deshawresearch.com> fixed various small bugs +- Michael Fenn <fennm@deshawresearch.com> implemented the database router + for separately storing the reporting database and fixed various small bugs related to bcfg2 on CentOS 5 - Alexander Sulfrian <alexander@sulfrian.net> fixed various bugs. diff --git a/debian/bcfg2-server.install b/debian/bcfg2-server.install index 533ca2e43..3f1a73b06 100644 --- a/debian/bcfg2-server.install +++ b/debian/bcfg2-server.install @@ -2,7 +2,6 @@ debian/bcfg2-server.default usr/share/bcfg2 debian/tmp/usr/bin/bcfg2-* usr/sbin debian/tmp/usr/lib/python*/*-packages/Bcfg2/Server/* debian/tmp/usr/lib/python*/*-packages/Bcfg2/Reporting/* -debian/tmp/usr/share/bcfg2/Hostbase/* debian/tmp/usr/share/bcfg2/schemas/* debian/tmp/usr/share/bcfg2/xsl-transforms/* debian/tmp/usr/share/man/man8/* diff --git a/debian/bcfg2-server.logcheck.ignore.server b/debian/bcfg2-server.logcheck.ignore.server index 136384f00..be4e45de3 100644 --- a/debian/bcfg2-server.logcheck.ignore.server +++ b/debian/bcfg2-server.logcheck.ignore.server @@ -1,4 +1,4 @@ -^\w{3} [ :0-9]{11} [._[:alnum:]-]+ bcfg2-server\[[0-9]+\]: Processed [0-9]+ (fam|gamin) events in [0-9.]+ seconds\. [0-9]+ coalesced$ +^\w{3} [ :0-9]{11} [._[:alnum:]-]+ bcfg2-server\[[0-9]+\]: Handled [0-9]+ events in [0-9.]+ ^\w{3} [ :0-9]{11} [._[:alnum:]-]+ bcfg2-server\[[0-9]+\]: Generated config for [._[:alnum:]-]+ in [0-9.]+ s$ ^\w{3} [ :0-9]{11} [._[:alnum:]-]+ bcfg2-server\[[0-9]+\]: Client [._[:alnum:]-]+ reported state (clean|dirty)$ ^\w{3} [ :0-9]{11} [._[:alnum:]-]+ bcfg2-server\[[0-9]+\]: Suppressing event for bogus file .*$ diff --git a/debian/bcfg2.install b/debian/bcfg2.install index fc43c3ecb..a240dac53 100644 --- a/debian/bcfg2.install +++ b/debian/bcfg2.install @@ -1,6 +1,7 @@ debian/tmp/usr/bin/bcfg2 usr/sbin debian/tmp/usr/lib/python*/*-packages/Bcfg2/*.py debian/tmp/usr/lib/python*/*-packages/Bcfg2/Client/* +debian/tmp/usr/lib/python*/*-packages/Bcfg2/Options/* debian/tmp/usr/share/man/man1/* debian/tmp/usr/share/man/man5/* examples/bcfg2.conf usr/share/bcfg2 diff --git a/debian/changelog b/debian/changelog index 3879cc50a..c41b2ecc2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +bcfg2 (1.4.0pre1-0.0) unstable; urgency=low + + * New upstream release + + -- Sol Jerome <sol.jerome@gmail.com> Mon, 16 Jun 2014 09:36:13 -0500 + bcfg2 (1.3.5-0.0) unstable; urgency=low * New upstream release diff --git a/debian/control b/debian/control index aee6f1c24..8eb4343c7 100644 --- a/debian/control +++ b/debian/control @@ -33,9 +33,9 @@ Description: Configuration management client Package: bcfg2-server Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, python-lxml (>= 0.9), libxml2-utils (>= 2.6.23), lsb-base (>= 3.1-9), ucf, bcfg2 (= ${binary:Version}), openssl, python (>= 2.6), python-pyinotify | python-gamin, python-daemon +Depends: ${python:Depends}, ${misc:Depends}, python-lxml (>= 0.9), libxml2-utils (>= 2.6.23), lsb-base (>= 3.1-9), ucf, bcfg2 (= ${binary:Version}), openssl, python (>= 2.6), python-pyinotify | python-gamin, python-daemon, python-genshi (>= 0.4.4) Recommends: graphviz, patch -Suggests: python-cheetah, python-genshi (>= 0.4.4), python-profiler, python-sqlalchemy (>= 0.5.0), python-django, mail-transport-agent, bcfg2-doc (= ${binary:Version}) +Suggests: python-cheetah, python-profiler, python-django, mail-transport-agent, bcfg2-doc (= ${binary:Version}) Description: Configuration management server Bcfg2 is a configuration management system that generates configuration sets for clients bound by client profiles. diff --git a/doc/appendix/files/mysql.txt b/doc/appendix/files/mysql.txt index 5adf2e27f..0dbbe9b05 100644 --- a/doc/appendix/files/mysql.txt +++ b/doc/appendix/files/mysql.txt @@ -8,14 +8,14 @@ MySQL example ============= -I had some time ago to continue with putting my configuration into +I had some time ago to continue with putting my configuration into Bcfg2 and maybe this helps someone else. I added a new bundle: .. code-block:: xml - <Bundle name="mysql-server" version="3.0"> + <Bundle> <Path name="/root/bcfg2-install/mysql/users.sh"/> <Path name="/root/bcfg2-install/mysql/users.sql"/> <Action name="users.sh"/> @@ -32,9 +32,9 @@ The ``users.sh`` script looks like this: mysql --defaults-extra-file=/etc/mysql/debian.cnf mysql \ < /root/bcfg2-install/mysql/users.sql -On debian there is a user account in ``/etc/mysql/debian.cnf`` -automatically created, but you could also (manually) create a -user in the database that has enough permissions and add the +On debian there is a user account in ``/etc/mysql/debian.cnf`` +automatically created, but you could also (manually) create a +user in the database that has enough permissions and add the login information in a file yourself. This file looks like this:: [client] diff --git a/doc/appendix/files/ntp.txt b/doc/appendix/files/ntp.txt index e14816f6e..c999841da 100644 --- a/doc/appendix/files/ntp.txt +++ b/doc/appendix/files/ntp.txt @@ -13,7 +13,7 @@ another layer of functionality. * After each change, run ``bcfg-repo-validate -v`` * Run the server with ``bcfg2-server -v`` * Update the client with ``bcfg2 -v -d -n`` (will not actually make - client changes) + client changes) Package only ------------ @@ -43,7 +43,7 @@ a client, a profile group, a list of packages, and an NTP bundle. .. code-block:: xml - <Bundle name="ntp"> + <Bundle> <Package name='ntp'/> </Bundle> @@ -75,7 +75,7 @@ Configure the service, and add it to Rules. .. code-block:: xml - <Bundle name="ntp"> + <Bundle> <Package name='ntp'/> <Service name='ntpd'/> </Bundle> @@ -85,16 +85,14 @@ Add config file Setup an ``etc/`` directory structure, and add it to the base:: - # cat Cfg/etc/ntp.conf/ntp.conf + # cat Cfg/etc/ntp.conf/ntp.conf server ntp1.utexas.edu -``Base/base.xml``: - ``Bundler/ntp.xml``: .. code-block:: xml - <Bundle name="ntp"> + <Bundle> <Package name='ntp'/> <Service name='ntpd'/> <Path name='/etc/ntp.conf'/> @@ -114,18 +112,18 @@ used to provide a single service. This is done for several reasons: packages are upgraded, so that they can be repaired if the package install clobbered them. * Services associated with a bundle get restarted whenever any entity - in that bundle is modified. This ensures that new configuration - files and software are used after installation. + in that bundle is modified. This ensures that new configuration + files and software are used after installation. The config file, package, and service are really all related -components describing the idea of an ntp client, so they should be +components describing the idea of an ntp client, so they should be logically grouped together. We use a bundle to accomplish this. ``Bundler/ntp.xml``: .. code-block:: xml - <Bundle name='ntp'> + <Bundle> <Package name='ntp'/> <Service name='ntpd'/> <Path name='/etc/ntp.conf'/> diff --git a/doc/appendix/guides/authentication.txt b/doc/appendix/guides/authentication.txt index b8ec82590..93a34c9bc 100644 --- a/doc/appendix/guides/authentication.txt +++ b/doc/appendix/guides/authentication.txt @@ -37,7 +37,6 @@ This is a :ref:`Cheetah template per-client bcfg2.conf from the per-client metadata:: [communication] - protocol = xmlrpc/ssl #if $self.metadata.uuid != None user = $self.metadata.uuid #end if diff --git a/doc/appendix/guides/centos.txt b/doc/appendix/guides/centos.txt index 19354b709..44ee08777 100644 --- a/doc/appendix/guides/centos.txt +++ b/doc/appendix/guides/centos.txt @@ -102,7 +102,7 @@ Run bcfg2 to be sure you are able to communicate with the server:: Excluding Packages in global exclude list Finished Loaded tool drivers: - Action Chkconfig POSIX YUMng + Action Chkconfig POSIX YUM Phase: initial Correct entries: 0 @@ -132,7 +132,6 @@ upon connection:: [communication] - protocol = xmlrpc/ssl password = N41lMNeW ca = /etc/bcfg2.crt @@ -147,7 +146,7 @@ Now if you run the client, no more warning:: Excluding Packages in global exclude list Finished Loaded tool drivers: - Action Chkconfig POSIX YUMng + Action Chkconfig POSIX YUM Phase: initial Correct entries: 0 @@ -176,7 +175,7 @@ First, replace **Pkgmgr** with **Packages** in the plugins line of ``bcfg2.conf``. Then create Packages layout (as per :ref:`packages-exampleusage`) in ``/var/lib/bcfg2`` -.. note:: I am using the RawURL syntax here since we are using `mrepo`_ +.. note:: I am using the rawurl syntax here since we are using `mrepo`_ to manage our yum mirrors. .. _mrepo: http://dag.wieers.com/home-made/mrepo/ @@ -184,37 +183,36 @@ line of ``bcfg2.conf``. Then create Packages layout (as per .. code-block:: xml <Sources> - <!-- CentOS (5.4) sources --> - <Source type="yum" rawurl="http://mrepo/centos5-x86_64/RPMS.os"> - <Arch>x86_64</Arch> + <Group name="centos5"> + <!-- CentOS 5 sources --> + <Source type="yum" + rawurl="http://mrepo/centos5-x86_64/RPMS.os"> + <Arch>x86_64</Arch> </Source> - <Source type="yum" rawurl="http://mrepo/centos5-x86_64/RPMS.updates"> - <Arch>x86_64</Arch> + <Source type="yum" + rawurl="http://mrepo/centos5-x86_64/RPMS.updates"> + <Arch>x86_64</Arch> </Source> - <Source type="yum" rawurl="http://mrepo/centos5-x86_64/RPMS.extras"> - <Arch>x86_64</Arch> + <Source type="yum" + rawurl="http://mrepo/centos5-x86_64/RPMS.extras"> + <Arch>x86_64</Arch> </Source> + </Group> </Sources> -Due to the :ref:`server-plugins-generators-packages-magic-groups`, -we need to modify our Metadata. Let's add a **centos5.4** group which -inherits a **centos** group (this should replace the existing **redhat** -group) present in ``/var/lib/bcfg2/Metadata/groups.xml``. The resulting -file should look something like this - -.. note:: - - The reason we are creating a release-specific group in this case is - that the YUMSource above is specific to the 5.4 release of centos. - That is, it should not apply to other releases (5.1, 5.3, etc). +To make these sources apply to our centos 5 clients, we need to modify +our Metadata. Let's add a **centos5** group which inherits a +**centos** group (this should replace the existing **redhat** group) +present in ``/var/lib/bcfg2/Metadata/groups.xml``. The resulting file +should look something like this .. code-block:: xml <Groups version='3.0'> <Group profile='true' public='true' default='true' name='basic'> - <Group name='centos-5.4'/> + <Group name='centos-5'/> </Group> - <Group name='centos-5.4'> + <Group name='centos-5'> <Group name='centos'/> </Group> <Group name='ubuntu'/> @@ -238,7 +236,7 @@ plugin. Add Probes to your plugins line in ``bcfg2.conf`` and create the Probe.:: [root@centos ~]# grep plugins /etc/bcfg2.conf - plugins = Base,Bundler,Cfg,...,Probes + plugins = Bundler,Cfg,...,Probes [root@centos ~]# mkdir /var/lib/bcfg2/Probes [root@centos ~]# cat /var/lib/bcfg2/Probes/groups #!/bin/sh @@ -260,9 +258,8 @@ it with the *yum* package. .. code-block:: xml - [root@centos ~]# cat /var/lib/bcfg2/Bundler/base-packages.xml - <Bundle name='base-packages'> - <Package name='yum'/> + <Bundle> + <Package name='yum'/> </Bundle> You need to reference the bundle from your Metadata. The resulting @@ -272,7 +269,7 @@ profile group might look something like this <Group profile='true' public='true' default='true' name='basic'> <Bundle name='base-packages'/> - <Group name='centos5.4'/> + <Group name='centos5'/> </Group> Now if we run the client, we can see what this has done for us.:: @@ -286,7 +283,7 @@ Now if we run the client, we can see what this has done for us.:: Excluding Packages in global exclude list Finished Loaded tool drivers: - Action Chkconfig POSIX YUMng + Action Chkconfig POSIX YUM Package pam failed verification. Phase: initial @@ -331,7 +328,7 @@ entries?:: Excluding Packages in global exclude list Finished Loaded tool drivers: - Action Chkconfig POSIX YUMng + Action Chkconfig POSIX YUM Extra Package openssh-clients 4.3p2-36.el5_4.4.x86_64. Extra Package libuser 0.54.7-2.1el5_4.1.x86_64. ... @@ -359,22 +356,22 @@ looks like this .. code-block:: xml - <Bundle name='base-packages'> - <Package name='bcfg2-server'/> - <Package name='exim'/> - <Package name='grub'/> - <Package name='kernel'/> - <Package name='krb5-workstation'/> - <Package name='m2crypto'/> - <Package name='openssh-clients'/> - <Package name='openssh-server'/> - <Package name='prelink'/> - <Package name='redhat-lsb'/> - <Package name='rpm-build'/> - <Package name='rsync'/> - <Package name='sysklogd'/> - <Package name='vim-enhanced'/> - <Package name='yum'/> + <Bundle> + <Package name='bcfg2-server'/> + <Package name='exim'/> + <Package name='grub'/> + <Package name='kernel'/> + <Package name='krb5-workstation'/> + <Package name='m2crypto'/> + <Package name='openssh-clients'/> + <Package name='openssh-server'/> + <Package name='prelink'/> + <Package name='redhat-lsb'/> + <Package name='rpm-build'/> + <Package name='rsync'/> + <Package name='sysklogd'/> + <Package name='vim-enhanced'/> + <Package name='yum'/> </Bundle> Now when I run the client, you can see I have only one unmanaged @@ -389,9 +386,7 @@ package:: Excluding Packages in global exclude list Finished Loaded tool drivers: - Action Chkconfig POSIX YUMng - Extra Package gpg-pubkey e8562897-459f07a4.None. - Extra Package gpg-pubkey 217521f6-45e8a532.None. + Action Chkconfig POSIX YUM Phase: initial Correct entries: 187 @@ -405,96 +400,11 @@ package:: Incorrect entries: 0 Total managed entries: 187 Unmanaged entries: 16 - Package:gpg-pubkey Service:atd Service:avahi-daemon Service:bcfg2-server ... -The gpg-pubkey packages are special in that they are not really -packages. Currently, the way to manage them is using :ref:`BoundEntries -<boundentries>`. So, after adding them, our Bundle now looks like this - -.. note:: This does not actually control the contents of the files, - you will need to do this part separately (see below). - -.. code-block:: xml - - <Bundle name='base-packages'> - <BoundPackage name="gpg-pubkey" type="rpm" version="foo"> - <Instance simplefile="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5" version="e8562897" release="459f07a4"/> - <Instance simplefile="/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL" version="217521f6" release="45e8a532"/> - </BoundPackage> - <Package name='bcfg2-server'/> - <Package name='exim'/> - <Package name='grub'/> - <Package name='kernel'/> - <Package name='krb5-workstation'/> - <Package name='m2crypto'/> - <Package name='openssh-clients'/> - <Package name='openssh-server'/> - <Package name='prelink'/> - <Package name='redhat-lsb'/> - <Package name='rpm-build'/> - <Package name='rsync'/> - <Package name='sysklogd'/> - <Package name='vim-enhanced'/> - <Package name='yum'/> - </Bundle> - -.. note:: - - version="foo" is just a dummy attribute for the gpg-pubkey Package - -To actually push the gpg keys out via Bcfg2, you will need to manage the -files as well. This can be done by adding Path entries for each of the -gpg keys you want to manage - -.. code-block:: xml - - <Bundle name='base-packages'> - <Path name='/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5'/> - <Path name='/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL'/> - <BoundPackage name="gpg-pubkey" type="rpm" version="foo"> - <Instance simplefile="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5" version="e8562897" release="459f07a4"/> - <Instance simplefile="/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL" version="217521f6" release="45e8a532"/> - </BoundPackage> - <Package name='bcfg2-server'/> - <Package name='exim'/> - <Package name='grub'/> - <Package name='kernel'/> - <Package name='krb5-workstation'/> - <Package name='m2crypto'/> - <Package name='openssh-clients'/> - <Package name='openssh-server'/> - <Package name='prelink'/> - <Package name='redhat-lsb'/> - <Package name='rpm-build'/> - <Package name='rsync'/> - <Package name='sysklogd'/> - <Package name='vim-enhanced'/> - <Package name='yum'/> - </Bundle> - -Then add the files to Cfg:: - - mkdir -p Cfg/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5 - cp /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5 !$/RPM-GPG-KEY-CentOS-5 - mkdir -p Cfg/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL - cp /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL !$/RPM-GPG-KEY-EPEL - -You will also want to add an *important* attribute to these files so -that they are installed on the client prior to any attempts to install -the **gpg-pubkey** rpm packages. This is especially important during the -bootstrapping phase and can be accomplished using an :ref:`server-info` -file that looks like the following: - -.. code-block:: xml - - <FileInfo> - <Info owner='root' group='root' mode='0644' important='true'/> - </FileInfo> - Now, running the client shows only unmanaged Service entries. Woohoo! Manage services @@ -528,22 +438,22 @@ entries to our bundle. [root@centos ~]# cat /var/lib/bcfg2/Rules/services.xml <Rules priority='1'> - <!-- basic services --> - <Service type='chkconfig' status='on' name='atd'/> - <Service type='chkconfig' status='on' name='avahi-daemon'/> - <Service type='chkconfig' status='on' name='bcfg2-server'/> - <Service type='chkconfig' status='on' name='crond'/> - <Service type='chkconfig' status='on' name='cups'/> - <Service type='chkconfig' status='on' name='gpm'/> - <Service type='chkconfig' status='on' name='lvm2-monitor'/> - <Service type='chkconfig' status='on' name='mcstrans'/> - <Service type='chkconfig' status='on' name='messagebus'/> - <Service type='chkconfig' status='on' name='netfs'/> - <Service type='chkconfig' status='on' name='network'/> - <Service type='chkconfig' status='on' name='postfix'/> - <Service type='chkconfig' status='on' name='rawdevices'/> - <Service type='chkconfig' status='on' name='sshd'/> - <Service type='chkconfig' status='on' name='syslog'/> + <!-- basic services --> + <Service type='chkconfig' status='on' name='atd'/> + <Service type='chkconfig' status='on' name='avahi-daemon'/> + <Service type='chkconfig' status='on' name='bcfg2-server'/> + <Service type='chkconfig' status='on' name='crond'/> + <Service type='chkconfig' status='on' name='cups'/> + <Service type='chkconfig' status='on' name='gpm'/> + <Service type='chkconfig' status='on' name='lvm2-monitor'/> + <Service type='chkconfig' status='on' name='mcstrans'/> + <Service type='chkconfig' status='on' name='messagebus'/> + <Service type='chkconfig' status='on' name='netfs'/> + <Service type='chkconfig' status='on' name='network'/> + <Service type='chkconfig' status='on' name='postfix'/> + <Service type='chkconfig' status='on' name='rawdevices'/> + <Service type='chkconfig' status='on' name='sshd'/> + <Service type='chkconfig' status='on' name='syslog'/> </Rules> Now we run the client and see there are no more unmanaged entries!:: @@ -557,7 +467,7 @@ Now we run the client and see there are no more unmanaged entries!:: Excluding Packages in global exclude list Finished Loaded tool drivers: - Action Chkconfig POSIX YUMng + Action Chkconfig POSIX YUM Phase: initial Correct entries: 205 diff --git a/doc/appendix/guides/converging_rhel5.txt b/doc/appendix/guides/converging_rhel5.txt index d6883c778..38d8761cb 100644 --- a/doc/appendix/guides/converging_rhel5.txt +++ b/doc/appendix/guides/converging_rhel5.txt @@ -24,7 +24,8 @@ Unmanaged entries sudo yum remove PACKAGE - #. Otherwise, add ``<Package name="PACKAGE" />`` to the Base or Bundler configuration. + #. Otherwise, add ``<Package name="PACKAGE" />`` to the Bundler + configuration. * Package (dependency) @@ -38,7 +39,7 @@ Unmanaged entries * Service - #. Add ``<Service name="SERVICE" />`` to the Base or Bundler configuration. + #. Add ``<Service name="SERVICE" />`` to the Bundler configuration. #. Add ``<Service name="SERVICE" status="on" type="chkconfig" />`` to ``/var/lib/bcfg2/Rules/services.xml``. @@ -57,8 +58,8 @@ For a "Package" * For example, ``/etc/motd`` to ``/var/lib/bcfg2/Cfg/etc/motd/motd``. Yes, there is an extra directory level named after the file. - #. Specify configuration files as ``<Path name='PATH' />`` in the Base - or Bundler configuration. + #. Specify configuration files as ``<Path name='PATH' />`` in the + Bundler configuration. #. Add directories to ``/var/lib/bcfg2/Rules/directories.xml``. For example: @@ -73,13 +74,13 @@ For a "Package" * Option A: Explicitly list the instances - #. Drop the ``<Package />`` from the Base or Bundler configuration. + #. Drop the ``<Package />`` from the Bundler configuration. #. Add an explicit ``<BoundPackage>`` and ``<Instance />`` configuration to a new Bundle, like the following: .. code-block:: xml - <Bundle name='keys'> + <Bundle> <!-- GPG keys --> <BoundPackage name="gpg-pubkey" type="rpm" version="foo"> <Instance simplefile="/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL" version="217521f6" release="45e8a532"/> diff --git a/doc/appendix/guides/fedora.txt b/doc/appendix/guides/fedora.txt deleted file mode 100644 index f89daaf0b..000000000 --- a/doc/appendix/guides/fedora.txt +++ /dev/null @@ -1,493 +0,0 @@ -.. -*- mode: rst -*- - -.. This guide is based on the Centos guide. - -.. _guide-fedora: - -====== -Fedora -====== - -This guide is work in progess. - - -This is a complete getting started guide for Fedora. With this -document you should be able to install a Bcfg2 server, a Bcfg2 client, -and change the ``/etc/motd`` file on the client. - -Prerequisites -============= - -To setup a configuration management system based on Bcfg2 only a few -prerequisites need to be fullfilled. - -* A server machine that can host the Bcfg2 -* Internet access for the installation process -* A working network with DNS - - -Install Bcfg2 From RPM -====================== - -The fastest way to get Bcfg2 onto your system is to use ``yum`` -or PackageKit. ``yum`` will pull all dependencies of Bcfg2 -automatically in. :: - - $ su -c 'yum install bcfg2-server bcfg2' - -Your system should now have the necessary software to use Bcfg2. -The next step is to set up your Bcfg2 :term:`repository`. - - -Initialize your repository -========================== - -Now that you're done with the install, you need to initialize your -repository and setup your ``/etc/bcfg2.conf``. ``bcfg2-admin init`` -is a tool which allows you to automate this: - -.. code-block:: sh - - # bcfg2-admin init - Store bcfg2 configuration in [/etc/bcfg2.conf]: - Location of bcfg2 repository [/var/lib/bcfg2]: - Directory /var/lib/bcfg2 exists. Overwrite? [y/N]:y - Input password used for communication verification (without echoing; leave blank for a random): - What is the server's hostname: [config01.local.net] - Input the server location [https://config01.local.net:6789]: - Input base Operating System for clients: - 1: Red Hat/Fedora/RHEL/RHAS/Centos - 2: SUSE/SLES - 3: Mandrake - 4: Debian - 5: Ubuntu - 6: Gentoo - 7: FreeBSD - : 1 - Generating a 1024 bit RSA private key - .......................................................++++++ - .....++++++ - writing new private key to '/etc/bcfg2.key' - ----- - Signature ok - subject=/C=US/ST=Illinois/L=Argonne/CN=config01.local.net - Getting Private key - Repository created successfuly in /var/lib/bcfg2 - -Change responses as necessary. - -Start the server -================ - -You are now ready to start your Bcfg2 server for the first time:: - - $ su -c '/etc/init.d/bcfg2-server start' - Starting Configuration Management Server: bcfg2-server [ OK ] - -To verify that everything started ok, look for the running daemon and -check the logs: - -.. code-block:: sh - - $ su -c 'tail /var/log/messages' - May 16 14:14:57 config01 bcfg2-server[2746]: service available at https://config01.local.net:6789 - May 16 14:14:57 config01 bcfg2-server[2746]: serving bcfg2-server at https://config01.local.net:6789 - May 16 14:14:57 config01 bcfg2-server[2746]: serve_forever() [start] - May 16 14:14:57 config01 bcfg2-server[2746]: Handled 16 events in 0.009s - - -Run ``bcfg2`` to be sure you are able to communicate with the server: - -.. code-block:: sh - - $ su -c 'bcfg2 -vqne' - - /usr/lib/python2.6/site-packages/Bcfg2/Client/Tools/rpmtools.py:23: DeprecationWarning: the md5 module is deprecated; use hashlib instead - import md5 - Loaded plugins: presto, refresh-packagekit - Loaded tool drivers: - Action Chkconfig POSIX YUMng - Extra Package imsettings-libs 0.108.0-2.fc13.i686. - Extra Package PackageKit-device-rebind 0.6.4-1.fc13.i686. - ... - Extra Package newt-python 0.52.11-2.fc13.i686. - Extra Package pulseaudio-gdm-hooks 0.9.21-6.fc13.i686. - - Phase: initial - Correct entries: 0 - Incorrect entries: 0 - Total managed entries: 0 - Unmanaged entries: 1314 - - - Phase: final - Correct entries: 0 - Incorrect entries: 0 - Total managed entries: 0 - Unmanaged entries: 1314 - Package:ConsoleKit Package:jasper-libs Package:pcsc-lite-libs - Package:ConsoleKit-libs Package:java-1.5.0-gcj Package:perf - ... - Package:iw Package:pcre Service:sshd - Package:jack-audio-connection-kit Package:pcsc-lite Service:udev-post - -The ``bcfg2.conf`` file contains only standard plugins so far. - -.. code-block:: sh - - $ su -c 'cat /etc/bcfg2.conf' - - [server] - repository = /var/lib/bcfg2 - plugins = SSHbase,Cfg,Pkgmgr,Rules,Metadata,Base,Bundler - - [statistics] - sendmailpath = /usr/lib/sendmail - - [database] - engine = sqlite3 - # 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. - name = - # Or path to database file if using sqlite3. - #<repository>/etc/brpt.sqlite is default path if left empty - user = - # Not used with sqlite3. - password = - # Not used with sqlite3. - host = - # Not used with sqlite3. - port = - - [communication] - protocol = xmlrpc/ssl - password = test1234 - certificate = /etc/bcfg2.crt - key = /etc/bcfg2.key - ca = /etc/bcfg2.crt - - [components] - bcfg2 = https://config01.local.net:6789 - - -Add the machines to Bcfg2 -------------------------- - -``bcfg2-admin`` can be used to add a machine to Bcfg2 easily. You -need to know the Fully Qualified Domain Name (FQDN) of ever system -you want to control through Bcfg2. :: - - bcfg2-admin client add <FQDN machine> - -Bring your first machine under Bcfg2 control --------------------------------------------- - -Now it is time to get the first machine's configuration into the -Bcfg2 repository. The server will be the first machine. It's -already in the ``Metadata/client.xml``. - - -Setup the :ref:`server-plugins-generators-packages` plugin -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -First, replace **Pkgmgr** with **Packages** in the plugins -line of ``bcfg2.conf``. Then create a `Packages/` directory in -``/var/lib/bcfg2`` :: - - $ su -c 'mkdir /var/lib/bcfg2/Packages' - -Create a ``packages.conf`` in the ``/var/lib/bcfg2/Packages`` directory -with the following contents:: - - [global] - -Create a ``sources.xml`` file for the packages in -``/var/lib/bcfg2/Packages`` with the following content. Choose a mirror -near your location according the `Mirror list`_ . - -.. _Mirror list: http://mirrors.fedoraproject.org/publiclist/ - -.. code-block:: xml - - <Sources> - <Group name="fedora-13"> - <Source type="yum" url="ftp://fedora.tu-chemnitz.de/pub/linux/fedora/linux/releases/" version="13"> - <Component>Fedora</Component> - <Arch>i386</Arch> - <Arch>x86_64</Arch> - <Source> - </Group> - </Sources> - - -Due to the :ref:`server-plugins-generators-packages-magic-groups`, -we need to modify our Metadata. Let's add a **fedora13** group which -inherits a **fedora** group (this should replace the existing **redhat** -group) present in ``/var/lib/bcfg2/Metadata/groups.xml``. The resulting -file should look something like this - -.. note:: - - The reason we are creating a release-specific group in this case is - that the YUMSource above is specific to the 13th release of fedora. - That is, it should not apply to other releases (14, 15, etc). - -.. code-block:: xml - - <Groups version='3.0'> - <Group profile='true' public='true' default='true' name='basic'> - <Group name='fedora13'/> - </Group> - <Group name='fedora13'/> - <Group name='fedora'/> - <Group name='ubuntu'/> - <Group name='debian'/> - <Group name='freebsd'/> - <Group name='gentoo'/> - <Group name='fedora'/> - <Group name='suse'/> - <Group name='mandrake'/> - <Group name='solaris'/> - </Groups> - -.. note:: - When editing your xml files by hand, it is useful to occasionally - run ``bcfg2-lint`` to ensure that your xml validates properly. - -Add a probe -+++++++++++ - -The next step for the client will be to have the proper arch group -membership. For this, we will make use of the -:ref:`server-plugins-probes-dynamic-groups` capabilities of the Probes -plugin. Add **Probes** to your plugins line in ``bcfg2.conf`` and -create the Probe: - -.. code-block:: sh - - $ su -c 'mkdir /var/lib/bcfg2/Probes' - $ su -c 'cat /var/lib/bcfg2/Probes/groups' - #!/bin/sh - - echo "group:`uname -m`" - -Now a restart of ``bcfg2-server`` is needed:: - - $ su -c '/etc/init.d/bcfg2-server restart' - -To test the Probe just run ``bcfg2 -vqn``. - -.. code-block:: xml - - $ su -c 'bcfg2 -vqn' - Running probe group - Probe group has result: - group:i686 - ... - -Start managing packages -+++++++++++++++++++++++ - -Add a base-packages bundle. Let's see what happens when we just populate -it with the *yum* package. Create the ``base-packages.xml`` in your -``Bundler/`` directory with a entry for ``yum``. - -.. code-block:: xml - - $ cat /var/lib/bcfg2/Bundler/base-packages.xml - <Bundle name='base-packages'> - <Package name='yum'/> - </Bundle> - -You need to reference the bundle from your ``group.xml``. The resulting -profile group might look something like this - -.. code-block:: xml - - <Group profile='true' public='true' default='true' name='basic'> - <Bundle name='base-packages'/> - <Group name='fedora13'/> - </Group> - -Now if we run the client, we can see what this has done for us.:: - - output - -As you can see, the Packages plugin has generated the dependencies -required for the yum package automatically. The ultimate goal should -be to move all the packages from the **Unmanaged** entries section -to the **Managed** entries section. So, what exactly *are* those -Unmanaged entries?:: - - output - -Now you can go through these and continue adding the packages you -want to your Bundle. After a while, I ended up with a minimal bundle -that looks like this - -.. code-block:: xml - - <Bundle name='base-packages'> - - </Bundle> - -Now when I run the client, you can see I have only one unmanaged -package:: - - outout - -The gpg-pubkey packages are special in that they are not really -packages. Currently, the way to manage them is using -:ref:`BoundEntries <boundentries>`. So, after adding them, our -Bundle now looks like this - -.. note:: This does not actually control the contents of the files, - you will need to do this part separately (see below). - -.. code-block:: xml - - <Bundle name='base-packages'> - <BoundPackage name="gpg-pubkey" type="rpm" version="foo"> - <Instance simplefile="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5" version="e8562897" release="459f07a4"/> - <Instance simplefile="/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL" version="217521f6" release="45e8a532"/> - </BoundPackage> - <Package name='bcfg2-server'/> - <Package name='exim'/> - <Package name='grub'/> - <Package name='kernel'/> - <Package name='krb5-workstation'/> - <Package name='m2crypto'/> - <Package name='openssh-clients'/> - <Package name='openssh-server'/> - <Package name='prelink'/> - <Package name='redhat-lsb'/> - <Package name='rpm-build'/> - <Package name='rsync'/> - <Package name='sysklogd'/> - <Package name='vim-enhanced'/> - <Package name='yum'/> - </Bundle> - -.. note:: - - version="foo" is just a dummy attribute for the gpg-pubkey Package - -To actually push the gpg keys out via Bcfg2, you will need to manage -the files as well. This can be done by adding Path entries for each -of the gpg keys you want to manage - -.. code-block:: xml - - <Bundle name='base-packages'> - <Path name='/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5'/> - <Path name='/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL'/> - <BoundPackage name="gpg-pubkey" type="rpm" version="foo"> - <Instance simplefile="/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5" version="e8562897" release="459f07a4"/> - <Instance simplefile="/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL" version="217521f6" release="45e8a532"/> - </BoundPackage> - <Package name='bcfg2-server'/> - <Package name='exim'/> - <Package name='grub'/> - <Package name='kernel'/> - <Package name='krb5-workstation'/> - <Package name='m2crypto'/> - <Package name='openssh-clients'/> - <Package name='openssh-server'/> - <Package name='prelink'/> - <Package name='redhat-lsb'/> - <Package name='rpm-build'/> - <Package name='rsync'/> - <Package name='sysklogd'/> - <Package name='vim-enhanced'/> - <Package name='yum'/> - </Bundle> - -Then add the files to Cfg:: - - mkdir -p Cfg/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5 - cp /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5 !$/RPM-GPG-KEY-CentOS-5 - mkdir -p Cfg/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL - cp /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL !$/RPM-GPG-KEY-EPEL - -Now, running the client shows only unmanaged Service entries. Woohoo! - -Manage services -+++++++++++++++ - -Now let's clear up the unmanaged service entries by adding the -following entries to our bundle... - -.. code-block:: xml - - <!-- basic services --> - <Service name='atd'/> - <Service name='avahi-daemon'/> - <Service name='bcfg2-server'/> - <Service name='crond'/> - <Service name='cups'/> - <Service name='gpm'/> - <Service name='lvm2-monitor'/> - <Service name='mcstrans'/> - <Service name='messagebus'/> - <Service name='netfs'/> - <Service name='network'/> - <Service name='postfix'/> - <Service name='rawdevices'/> - <Service name='sshd'/> - <Service name='syslog'/> - -...and bind them in Rules - -.. code-block:: xml - - [root@centos ~]# cat /var/lib/bcfg2/Rules/services.xml - <Rules priority='1'> - <!-- basic services --> - <Service type='chkconfig' status='on' name='atd'/> - <Service type='chkconfig' status='on' name='avahi-daemon'/> - <Service type='chkconfig' status='on' name='bcfg2-server'/> - <Service type='chkconfig' status='on' name='crond'/> - <Service type='chkconfig' status='on' name='cups'/> - <Service type='chkconfig' status='on' name='gpm'/> - <Service type='chkconfig' status='on' name='lvm2-monitor'/> - <Service type='chkconfig' status='on' name='mcstrans'/> - <Service type='chkconfig' status='on' name='messagebus'/> - <Service type='chkconfig' status='on' name='netfs'/> - <Service type='chkconfig' status='on' name='network'/> - <Service type='chkconfig' status='on' name='postfix'/> - <Service type='chkconfig' status='on' name='rawdevices'/> - <Service type='chkconfig' status='on' name='sshd'/> - <Service type='chkconfig' status='on' name='syslog'/> - </Rules> - -Now we run the client and see there are no more unmanaged entries! :: - - $ su -c 'bcfg2 -veqn' - - -Adding Plugins -++++++++++++++ - -Git ---- - -.. _Git tutorial: http://www.kernel.org/pub/software/scm/git/docs/gittutorial.html - -Adding the :ref:`server-plugins-version-git` plugins can preserve -versioning information. The first step is to add *Git* to your -plugin line:: - - plugins = Base,Bundler,Cfg,...,Git - -For tracking the configuration files in the ``/var/lib/bcfg2`` -directory a git repository need to be established:: - - git init - -For more detail about the setup of git please refer to a `git tutorial`_. -The first commit can be the empty or the allready populated directory:: - - git add . && git commit -a - -While running ``bcfg2-info`` the following line will show up:: - - Initialized git plugin with git directory = /var/lib/bcfg2/.git diff --git a/doc/appendix/guides/import-existing-ssh-keys.txt b/doc/appendix/guides/import-existing-ssh-keys.txt index a0a628c58..4e2282044 100644 --- a/doc/appendix/guides/import-existing-ssh-keys.txt +++ b/doc/appendix/guides/import-existing-ssh-keys.txt @@ -22,14 +22,24 @@ Add a bundle for ssh After verifying that SSHbase is listed on the plugins line in ``/etc/bcfg2.conf``, you need to create a bundle containing the -appropriate entries.:: +appropriate entries. In general, you can use a path glob: - cat > /tmp/ssh.xml << EOF - <Bundle name='ssh'> +.. code-block:: xml + + <Bundle> + <Path glob="/etc/ssh/*"/> + </Bundle> + +If you need more granular control -- e.g., other entries in +``/etc/ssh`` are specified in other bundles -- you can also list the +files explicity: + +.. code-block:: xml + + <Bundle> <!-- requires a version of openssh that can generate ecdsa keys --> <Path name="/etc/ssh/ssh_host_ecdsa_key"/> <Path name="/etc/ssh/ssh_host_ecdsa_key.pub"/> - <Path name='/etc/ssh/ssh_host_dsa_key'/> <Path name='/etc/ssh/ssh_host_rsa_key'/> <Path name='/etc/ssh/ssh_host_dsa_key.pub'/> @@ -39,10 +49,6 @@ appropriate entries.:: <Path name='/etc/ssh/ssh_known_hosts'/> </Bundle> -:: - - mv /tmp/ssh.xml /var/lib/bcfg2/Bundle - Next, you need to add the ssh bundle to the client's metadata in groups.xml. diff --git a/doc/appendix/guides/sslca_howto.txt b/doc/appendix/guides/sslca_howto.txt new file mode 100644 index 000000000..8ee0b2b42 --- /dev/null +++ b/doc/appendix/guides/sslca_howto.txt @@ -0,0 +1,182 @@ +.. -*- mode: rst -*- + +.. _appendix-guides-sslca_howto: + +==================================== + Automated Bcfg2 SSL Authentication +==================================== + +This how-to describes one possible scenario for automating SSL +certificate generation and distribution for bcfg2 client/server +communication using the :ref:`SSL CA feature +<server-plugins-generators-cfg-ssl-certificates>` of +:ref:`server-plugins-generators-cfg`. The process involves configuring +a certificate authority (CA), generating the CA cert and key pair, +configuring the Cfg SSL CA feature and a Bundle to use the generated +certs to authenticate the Bcfg2 client and server. + +OpenSSL CA +========== + +If you already have a SSL CA available you can skip this section, +otherwise you can easily build one on the server using openssl. The +paths should be adjusted to suite your preferences. + +#. Prepare the directories and files:: + + mkdir -p /etc/pki/CA/newcerts + mkdir /etc/pki/CA/crl + echo '01' > /etc/pki/CA/serial + touch /etc/pki/CA/index.txt + touch /etc/pki/CA/crlnumber + +#. Edit the ``openssl.cnf`` config file, and in the **[ CA_default ]** + section adjust the following parameters:: + + dir = /etc/pki # Where everything is kept + certs = /etc/pki/CA/certs # Where the issued certs are kept + database = /etc/pki/CA/index.txt # database index file. + new_certs_dir = /etc/pki/CA/newcerts # default place for new certs. + certificate = /etc/pki/CA/certs/bcfg2ca.crt # The CA certificate + serial = /etc/pki/CA/serial # The current serial number + crl_dir = /etc/pki/CA/crl # Where the issued crl are kept + crlnumber = /etc/pki/CA/crlnumber # the current crl number + crl = /etc/pki/CA/crl.pem # The current CRL + private_key = /etc/pki/CA/private/bcfg2ca.key # The private key + +#. Create the CA root certificate and key pair. You'll be asked to + supply a passphrase, and some organizational info. The most + important bit is **Common Name** which you should set to be the + hostname of your bcfg2 server that your clients will see when doing + a reverse DNS query on it's ip address.:: + + openssl req -new -x509 -extensions v3_ca -keyout bcfg2ca.key \ + -out bcfg2ca.crt -days 3650 + +#. Move the generated cert and key to the locations specified in + ``openssl.cnf``:: + + mv bcfg2ca.key /etc/pki/CA/private/ + mv bcfg2ca.crt /etc/pki/CA/certs/ + +Your self-signing CA is now ready to use. + +Bcfg2 +===== + +SSL CA Feature +-------------- + +The SSL CA feature of Cfg was not designed specifically to manage +Bcfg2 client/server communication, though it is certainly able to +provide certificate generation and management services for that +purpose. You'll need to configure Cfg as described in +:ref:`server-plugins-generators-cfg-ssl-certificates`, including: + +* Configuring a ``[sslca_default]`` section in ``bcfg2.conf`` that + describes the CA you created above; +* Creating ``Cfg/etc/pki/tls/certs/bcfg2client.crt/sslcert.xml`` and + ``Cfg/etc/pki/tls/private/bcfg2client.key/sslkey.xml`` to describe + the key and cert you want generated. + +In general, the defaults in ``sslcert.xml`` and ``sslkey.xml`` should +be fine, so those files can look like this: + +``Cfg/etc/pki/tls/certs/bcfg2client.crt/sslcert.xml``: + +.. code-block:: xml + + <CertInfo> + <Cert key="/etc/pki/tls/private/bcfg2client.key"/> + </CertInfo> + +``Cfg/etc/pki/tls/private/bcfg2client.key/sslkey.xml``: + +.. code-block:: xml + + <KeyInfo/> + +Client Bundle +------------- + +To automate the process of generating and distributing certs to the +clients we need define at least the cert and key paths created by Cfg, +as well as the CA certificate path in a Bundle. For example: + +.. code-block:: xml + + <Path name='/etc/pki/tls/certs/bcfg2ca.crt'/> + <Path name='/etc/pki/tls/bcfg2client.crt'/> + <Path name='/etc/pki/tls/private/bcfg2client.key'/> + +Here's a more complete example bcfg2-client bundle: + +.. code-block:: xml + + <Bundle> + <Path name='/etc/bcfg2.conf'/> + <Path name='/etc/cron.d/bcfg2-client'/> + <Package name='bcfg2'/> + <Service name='bcfg2'/> + <Group name='rpm'> + <Path name='/etc/sysconfig/bcfg2'/> + <Path name='/etc/pki/tls/certs/bcfg2ca.crt'/> + <Path name='/etc/pki/tls/certs/bcfg2client.crt'/> + <Path name='/etc/pki/tls/private/bcfg2client.key'/> + </Group> + <Group name='deb'> + <Path name='/etc/default/bcfg2' altsrc='/etc/sysconfig/bcfg2'/> + <Path name='/etc/ssl/certs/bcfg2ca.crt' altsrc='/etc/pki/tls/certs/bcfg2ca.crt'/> + <Path name='/etc/ssl/certs/bcfg2client.crt' altsrc='/etc/pki/tls/certs/bcfg2client.crt'/> + <Path name='/etc/ssl/private/bcfg2client.key' altsrc='/etc/pki/tls/private/bcfg2client.key'/> + </Group> + </Bundle> + +The ``bcfg2.conf`` client config needs at least 5 parameters set for +SSL auth. + +#. ``key`` : This is the host specific key that Cfg will create. +#. ``certificate`` : This is the host specific cert that Cfg will + create. +#. ``ca`` : This is a copy of your CA certificate. Not generated by + Cfg. +#. ``password`` : Set to arbitrary string when using certificate + auth. This also *shouldn't* be required. See: + http://trac.mcs.anl.gov/projects/bcfg2/ticket/1019 + +Here's what a functional **[communication]** section in a +``bcfg2.conf`` genshi template for clients might look like.:: + + [communication] + {% if metadata.uuid != None %}\ + user = ${metadata.uuid} + {% end %}\ + password = DUMMYPASSWORDFORCERTAUTH + {% choose %}\ + {% when 'rpm' in metadata.groups %}\ + certificate = /etc/pki/tls/certs/bcfg2client.crt + key = /etc/pki/tls/private/bcfg2client.key + ca = /etc/pki/tls/certs/bcfg2ca.crt + {% end %}\ + {% when 'deb' in metadata.groups %}\ + certificate = /etc/ssl/certs/bcfg2client.crt + key = /etc/ssl/private/bcfg2client.key + ca = /etc/ssl/certs/bcfg2ca.crt + {% end %}\ + {% end %}\ + +As a client will not be able to authenticate with certificates it does +not yet posses we need to overcome the chicken and egg scenario the +first time we try to connect such a client to the server. We can do so +using password based auth to bootstrap the client manually specifying +all the relevant auth parameters like so:: + + bcfg2 -qv -S https://fqdn.of.bcfg2-server:6789 -u fqdn.of.client \ + -x SUPER_SECRET_PASSWORD + +If all goes well the client should recieve a freshly generated key and +cert and you should be able to run ``bcfg2`` again without specifying +the connection parameters. + +If you do run into problems you may want to review +:ref:`appendix-guides-authentication`. diff --git a/doc/appendix/guides/ubuntu.txt b/doc/appendix/guides/ubuntu.txt index 9bf851632..24bebf023 100644 --- a/doc/appendix/guides/ubuntu.txt +++ b/doc/appendix/guides/ubuntu.txt @@ -172,7 +172,6 @@ Replace Pkgmgr with Packages in the plugins line of ``bcfg2.conf``:: transport = LocalFilesystem [communication] - protocol = xmlrpc/ssl password = secret certificate = /etc/ssl/bcfg2.crt key = /etc/ssl/bcfg2.key @@ -496,7 +495,7 @@ like this: .. code-block:: xml - <Bundle name='base-saucy'> + <Bundle> <!-- packages --> <Package name='bcfg2-server'/> <!-- or dependencies --> diff --git a/doc/appendix/guides/vcs.txt b/doc/appendix/guides/vcs.txt index 6c2879a65..fba61e722 100644 --- a/doc/appendix/guides/vcs.txt +++ b/doc/appendix/guides/vcs.txt @@ -30,7 +30,7 @@ While running ``bcfg2-info`` the following line will show up:: Initialized git plugin with git directory = /var/lib/bcfg2/.git -Mercurial +Mercurial ========= The :ref:`server-plugins-version-hg` plugin also allows you to store @@ -59,7 +59,7 @@ While running ``bcfg2-info`` the following line will show up:: Initialized hg plugin with hg directory = /var/lib/bcfg2/.hg -Darcs +Darcs ===== The :ref:`server-plugins-version-darcs` plugin also allows you to store @@ -70,8 +70,8 @@ be initialized:: darcs initialize -To commit to the darcs repository an author must be added to the -``_darcs/prefs/author`` file. If the ``author`` file is missing, +To commit to the darcs repository an author must be added to the +``_darcs/prefs/author`` file. If the ``author`` file is missing, darcs will ask you to enter your e-mail address. .. code-block:: sh @@ -99,7 +99,7 @@ Cvs The :ref:`server-plugins-version-cvs` plugin also allows you to store version information in the statistics database. - plugins = Base,Bundler,Cfg,...,Cvs + plugins = Bundler,Cfg,...,Cvs The CVS repository must be initialized:: diff --git a/doc/appendix/guides/web-reports-install.txt b/doc/appendix/guides/web-reports-install.txt index f03bad289..06932efc9 100644 --- a/doc/appendix/guides/web-reports-install.txt +++ b/doc/appendix/guides/web-reports-install.txt @@ -28,7 +28,7 @@ Add Reporting to the plugins line of ``bcfg2.conf``. The resulting [server] repository = /var/lib/bcfg2 - plugins = Base,Bundler,Cfg,...,Reporting + plugins = Bundler,Cfg,...,Reporting [reporting] transport = LocalFilesystem @@ -53,7 +53,7 @@ then have something like this:: [server] repository = /var/lib/bcfg2 - plugins = Base,Bundler,Cfg,...,Reporting + plugins = Bundler,Cfg,...,Reporting [database] engine = sqlite3 diff --git a/doc/client/metadata.txt b/doc/client/metadata.txt index 27870ba9a..0dec5e3a7 100644 --- a/doc/client/metadata.txt +++ b/doc/client/metadata.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _client-metadata: @@ -24,12 +25,12 @@ interaction: This construction process spans several server plugins. The :ref:`server-plugins-grouping-metadata` is responsible for initial instance creation, including the client hostname, -profile, and basic group memberships. After this initial creation, -Connector plugins (such as :ref:`server-plugins-probes-index` or -:ref:`server-plugins-connectors-properties`) can add additional group -memberships for clients. These memberships are merged into the instance; -that is, the new group memberships are treated as if they were included -in groups.xml. If any of these groups are defined in groups.xml, +profile, and basic group memberships. After this initial +creation, Connector plugins (such as :ref:`server-plugins-probes` +or :ref:`server-plugins-connectors-properties`) can add additional +group memberships for clients. These memberships are merged into the +instance; that is, the new group memberships are treated as if they were +included in groups.xml. If any of these groups are defined in groups.xml, then groups included there are included in the ClientMetadata instance group list. At the end of this process, the ClientMetadata instance has its complete set of group memberships. At this point, each connector diff --git a/doc/client/tools.txt b/doc/client/tools.txt index 170b30b2e..93eb11925 100644 --- a/doc/client/tools.txt +++ b/doc/client/tools.txt @@ -133,8 +133,6 @@ RPM Executes RPM to manage packages on Redhat-based and similar systems. Consider using the :ref:`YUM <client-tools-yum>` tool instead if possible. -Formerly called ``RPMng``, but was renamed for the 1.3 release. - SMF --- @@ -195,13 +193,5 @@ Upstart service support. Uses `Upstart`_ to configure services. YUM --- -Handles RPMs using the YUM package manager. Renamed from ``YUMng`` for -the 1.3 release. See :ref:`client-tools-yum` for more details. - -YUM24 ------ - -.. warning:: Deprecated in favor of :ref:`YUM <client-tools-yum>` - -Handles RPMs using older versions of the YUM package manager. - +Handles RPMs using the YUM package manager. See +:ref:`client-tools-yum` for more details. diff --git a/doc/client/tools/actions.txt b/doc/client/tools/actions.txt index e5fdb1f39..52d07eb4f 100644 --- a/doc/client/tools/actions.txt +++ b/doc/client/tools/actions.txt @@ -28,18 +28,17 @@ so they can be centrally observed. Actions look like: Note that the status attribute tells the bcfg2 client to ignore return status, causing failures to still not be centrally reported. If central reporting of action failure is desired, set this attribute to -'check'. Also note that Action entries included in Base will not be -executed. +'check'. -Actions may be completely defined inside of a bundle with the use of -:ref:`server-configurationentries`, much like Packages, Services or Paths. -The Rules plugin can also bind these entries. For example to include the -above action in a bundle, first the Action entry must be included in the +Actions may be completely defined inside of a bundle with the use of +:ref:`server-configurationentries`, much like Packages, Services or Paths. +The Rules plugin can also bind these entries. For example to include the +above action in a bundle, first the Action entry must be included in the bundle: .. code-block:: xml - <Bundle name='bundle_name'> + <Bundle> ... <Action name='action_name'/> </Bundle> @@ -56,6 +55,16 @@ Then a corresponding entry must be included in the Rules directory, like: This allows different clients to get different actions as a part of the same bundle based on group membership. +It is also possible to do this in one step in the bundle itself with a +``BoundAction`` tag, e.g.: + +.. code-block:: xml + + <Bundle> + <BoundAction timing='post' when='modified' name='action_name' + command='/path/to/command arg1 arg2' status='ignore'/> + </Rules> + Example Action (add APT keys) ============================= diff --git a/doc/client/tools/yum.txt b/doc/client/tools/yum.txt index 47ef3d5e9..ed1a3d5fd 100644 --- a/doc/client/tools/yum.txt +++ b/doc/client/tools/yum.txt @@ -7,9 +7,7 @@ Bcfg2 RPM/YUM Client Drivers ============================ The RPM and YUM client drivers provide client support for RPMs -(installed directly from URLs) and Yum repositories. These drivers -were formerly called ``RPMng`` and ``YUMng``, respectively, but were -renamed for Bcfg2 1.3.0. +(installed directly from URLs) and Yum repositories. Features ======== diff --git a/doc/conf.py b/doc/conf.py index 3e877ef80..1da6b3b01 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -64,9 +64,9 @@ else: # built documents. # # The short X.Y version. -version = '1.3' +version = '1.4' # The full version, including alpha/beta/rc tags. -release = '1.3.5' +release = '1.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/development/caching.txt b/doc/development/caching.txt new file mode 100644 index 000000000..83ec0290f --- /dev/null +++ b/doc/development/caching.txt @@ -0,0 +1,74 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _development-cache: + +============================ + Server-side Caching System +============================ + +.. versionadded:: 1.4.0 + +Bcfg2 caches two kinds of data: + +* The contents of all files that it reads in, including (often) an + optimized representation. E.g., XML files are cached both in their + raw (text) format, and also as :class:`lxml.etree._Element` objects. +* Arbitrary data, in the server-side caching system documented on this + page. + +The caching system keeps a single unified cache with all cache data in +it. Each individual datum stored in the cache is associated with any +number of "tags" -- simple terms that uniquely identify the datum. +This lets you very easily expire related data from multiple caches at +once; for isntance, for expiring all data related to a host: + +.. code-block:: python + + Bcfg2.Server.Cache.expire("foo.example.com") + +This would expire *all* data related to ``foo.example.com``, +regardless of which plugin cached it, and so on. + +This permits a high level of interoperation between different plugins +and the cache, which is necessary due to the wide distribution of data +in Bcfg2 and the many different data sources that can be incorported. +More technical details about writing code that uses the caches is below. + +Currently known caches are: + +.. currentmodule:: Bcfg2.Server.Plugins.Packages.Collection + ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Tags | Key(s) | Values | Use | ++=============+=======================================+=================================================+======================================================+ +| Metadata | Hostname | :class:`ClientMetadata | The :ref:`Metadata cache <server-caching>` | +| | | <Bcfg2.Server.Plugins.Metadata.ClientMetadata>` | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Probes, | Hostname | ``list`` of group names | Groups set by :ref:`server-plugins-probes` | +| probegroups | | | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Probes, | Hostname | ``dict`` of ``<probe name>``: | Other data set by :ref:`server-plugins-probes` | +| probedata | | :class:`ProbeData | | +| | | <Bcfg2.Server.Plugins.Probes.ProbeData>` | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | :attr:`Packages Collection cache key | :class:`Collection` | Kept by :ref:`server-plugins-generators-packages` in | +| collections | <Collection.cachekey>` | | order to expire repository metadata cached on disk | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | Hostname | :attr:`Packages Collection cache key | Used by the Packages plugin to return Collection | +| clients | | <Collection.cachekey>` | objects for clients. This is cross-referenced with | +| | | | the ``Packages, collections`` cache | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | :attr:`Packages Collection cache key | ``set`` of package names | Cached results from looking up | +| pkg_groups | <Collection.cachekey>`, | | ``<Package group="..."/>`` entries | +| | hash of the selected package groups | | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | :attr:`Packages Collection cache key | ``set`` of package names | Cached results from resolving complete package sets | +| pkg_sets | <Collection.cachekey>`, | | for clients | +| | hash of the initial package selection | | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ + +These are enumerated so that they can be expired as needed by other +plugins or other code points. + +.. automodule:: Bcfg2.Server.Cache diff --git a/doc/development/cfg.txt b/doc/development/cfg.txt index 6533e0d7a..4e967368b 100644 --- a/doc/development/cfg.txt +++ b/doc/development/cfg.txt @@ -55,12 +55,6 @@ exceptions: .. autoexception:: Bcfg2.Server.Plugin.exceptions.PluginInitError :noindex: -Global Variables -================ - -.. autodata:: Bcfg2.Server.Plugins.Cfg.SETUP -.. autodata:: Bcfg2.Server.Plugins.Cfg.CFG - Existing Cfg Handlers ===================== @@ -70,9 +64,11 @@ Generators .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator.CfgPlaintextGenerator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.CfgGenshiGenerator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.CfgCheetahGenerator +.. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.CfgJinja2Generator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator.CfgEncryptedGenerator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator.CfgEncryptedGenshiGenerator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator.CfgEncryptedCheetahGenerator +.. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator.CfgEncryptedJinja2Generator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.CfgAuthorizedKeysGenerator Creators @@ -81,18 +77,11 @@ Creators .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.CfgPrivateKeyCreator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator.CfgPublicKeyCreator -Filters -------- - -.. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgCatFilter.CfgCatFilter -.. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgDiffFilter.CfgDiffFilter - Info Handlers ------------- .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgDefaultInfo .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgInfoXML.CfgInfoXML -.. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgLegacyInfo.CfgLegacyInfo Verifiers --------- @@ -105,6 +94,6 @@ Other Cfg Objects These other objects comprise the remainder of the Cfg plugin, and are included for completeness. -.. autoattribute:: Bcfg2.Server.Plugins.Cfg.DEFAULT_INFO .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEntrySet .. autoclass:: Bcfg2.Server.Plugins.Cfg.Cfg +.. automethod:: Bcfg2.Server.Plugins.Cfg.get_cfg diff --git a/doc/development/core.txt b/doc/development/core.txt index 886a5538b..f5cc7de67 100644 --- a/doc/development/core.txt +++ b/doc/development/core.txt @@ -10,8 +10,10 @@ Bcfg2 1.3 added a pluggable server core system so that the server core itself can be easily swapped out to use different technologies. It -currently ships with two backends: a builtin core written from scratch -using the various server tools in the Python standard library; and an +currently ships with several backends: a builtin core written from +scratch using the various server tools in the Python standard library; +a variant on the builtin core that uses Python 2.6's +:mod:`multiprocessing` library to process requests in parallel; and an experimental `CherryPy <http://www.cherrypy.org/>`_ based core. This page documents the server core interface so that other cores can be written to take advantage of other technologies, e.g., `Tornado @@ -20,20 +22,25 @@ written to take advantage of other technologies, e.g., `Tornado A core implementation needs to: -* Override :func:`Bcfg2.Server.Core.BaseCore._daemonize` to handle - daemonization, writing the PID file, and dropping privileges. -* Override :func:`Bcfg2.Server.Core.BaseCore._run` to handle server +* Override :func:`Bcfg2.Server.Core.Core._run` to handle server startup. -* Override :func:`Bcfg2.Server.Core.BaseCore._block` to run the +* Override :func:`Bcfg2.Server.Core.Core._block` to run the blocking server loop. -* Call :func:`Bcfg2.Server.Core.BaseCore.shutdown` on orderly +* Call :func:`Bcfg2.Server.Core.Core.shutdown` on orderly shutdown. +A core that wants to use the network (i.e., a core that isn't used +entirely for introspection, as in :ref:`bcfg2-info +<server-bcfg2-info>`, or other local tasks) should inherit from +:class:`Bcfg2.Server.Core.NetworkCore`, and must also override +:func:`Bcfg2.Server.Core.NetworkCore._daemonize` to handle daemonization, +writing the PID file, and dropping privileges. + Nearly all XML-RPC handling is delegated entirely to the core implementation. It needs to: -* Call :func:`Bcfg2.Server.Core.BaseCore.authenticate` to authenticate - clients. +* Call :func:`Bcfg2.Server.Core.NetworkCore.authenticate` to + authenticate clients. * Handle :exc:`xmlrpclib.Fault` exceptions raised by the exposed XML-RPC methods as appropriate. * Dispatch XML-RPC method invocations to the appropriate method, @@ -59,7 +66,7 @@ Builtin Core The builtin server core consists of the core implementation (:class:`Bcfg2.Server.BuiltinCore.Core`) and the XML-RPC server -implementation (:mod:`Bcfg2.SSLServer`). +implementation (:mod:`Bcfg2.Server.SSLServer`). Core ~~~~ @@ -69,7 +76,7 @@ Core XML-RPC Server ~~~~~~~~~~~~~~ -.. automodule:: Bcfg2.SSLServer +.. automodule:: Bcfg2.Server.SSLServer Multiprocessing Core -------------------- @@ -79,4 +86,4 @@ Multiprocessing Core CherryPy Core ------------- -.. automodule:: Bcfg2.Server.CherryPyCore +.. automodule:: Bcfg2.Server.CherrypyCore diff --git a/doc/development/fam.txt b/doc/development/fam.txt index c2c3b14f5..e967aaf68 100644 --- a/doc/development/fam.txt +++ b/doc/development/fam.txt @@ -56,11 +56,6 @@ Pseudo .. automodule:: Bcfg2.Server.FileMonitor.Pseudo -Fam ---- - -.. automodule:: Bcfg2.Server.FileMonitor.Fam - Gamin ----- diff --git a/doc/development/lint.txt b/doc/development/lint.txt index 6c0be960d..56a3d8a66 100644 --- a/doc/development/lint.txt +++ b/doc/development/lint.txt @@ -10,14 +10,14 @@ lets you easily write your own plugins to verify various parts of your Bcfg2 specification. -Plugins are loaded in one of two ways: +Plugins are included in a module of the same name as the plugin class +in :mod:`Bcfg2.Server.Lint`, e.g., :mod:`Bcfg2.Server.Lint.Validate`. -* They may be included in a module of the same name as the plugin - class in :mod:`Bcfg2.Server.Lint`, e.g., - :mod:`Bcfg2.Server.Lint.Validate`. -* They may be included directly in a Bcfg2 server plugin, called - "<plugin>Lint", e.g., - :class:`Bcfg2.Server.Plugins.Metadata.MetadataLint`. +.. note:: + + It is no longer possible to include lint plugins directly in a + Bcfg2 server plugin, e.g., + :class:`Bcfg2.Server.Plugins.Metadata.MetadataLint`. Plugin Types ============ @@ -106,15 +106,15 @@ Basics Existing ``bcfg2-lint`` Plugins =============================== -AWSTagsLint ------------ +AWSTags +------- -.. autoclass:: Bcfg2.Server.Plugins.AWSTags.AWSTagsLint +.. automodule:: Bcfg2.Server.Lint.AWSTags -BundlerLint ------------ +Bundler +------- -.. autoclass:: Bcfg2.Server.Plugins.Bundler.BundlerLint +.. automodule:: Bcfg2.Server.Lint.Bundler Comments -------- @@ -131,10 +131,10 @@ GroupNames .. automodule:: Bcfg2.Server.Lint.GroupNames -GroupPatternsLint ------------------ +GroupPatterns +------------- -.. autoclass:: Bcfg2.Server.Plugins.GroupPatterns.GroupPatternsLint +.. automodule:: Bcfg2.Server.Lint.GroupPatterns InfoXML ------- @@ -146,25 +146,25 @@ MergeFiles .. automodule:: Bcfg2.Server.Lint.MergeFiles -MetadataLint ------------- +Metadata +-------- -.. autoclass:: Bcfg2.Server.Plugins.Metadata.MetadataLint +.. automodule:: Bcfg2.Server.Lint.Metadata -PkgmgrLint ----------- +Pkgmgr +------ -.. autoclass:: Bcfg2.Server.Plugins.Pkgmgr.PkgmgrLint +.. automodule:: Bcfg2.Server.Lint.Pkgmgr RequiredAttrs ------------- .. automodule:: Bcfg2.Server.Lint.RequiredAttrs -TemplateHelperLint ------------------- +TemplateHelper +-------------- -.. autoclass:: Bcfg2.Server.Plugins.TemplateHelper.TemplateHelperLint +.. automodule:: Bcfg2.Server.Lint.TemplateHelper Validate -------- diff --git a/doc/development/option_parsing.txt b/doc/development/option_parsing.txt new file mode 100644 index 000000000..091f43cdd --- /dev/null +++ b/doc/development/option_parsing.txt @@ -0,0 +1,246 @@ +.. -*- mode: rst -*- + +.. _development-option-parsing: + +==================== +Bcfg2 Option Parsing +==================== + +Bcfg2 uses an option parsing mechanism based on the Python +:mod:`argparse` module. It does several very useful things that +``argparse`` does not: + +* Collects options from various places, which lets us easily specify + per-plugin options, for example; +* Automatically loads components (such as plugins); +* Synthesizes option values from the command line, config files, and + environment variables; +* Can dynamically create commands with many subcommands (e.g., + bcfg2-info and bcfg2-admin); and +* Supports keeping documentation inline with the option declaration, + which will make it easier to generate man pages. + + +Collecting Options +================== + +One of the more important features of the option parser is its ability +to automatically collect options from loaded components (e.g., Bcfg2 +server plugins). Given the highly pluggable architecture of Bcfg2, +this helps ensure two things: + +#. We do not have to specify all options in all places, or even in + most places. Options are specified alongside the class(es) that use + them. +#. All options needed for a given script to run are guaranteed to be + loaded, without the need to specify all components that script uses + manually. + +For instance, assume a few plugins: + +* The ``Foo`` plugin takes one option, ``--foo`` +* The ``Bar`` plugin takes two options, ``--bar`` and ``--force`` + +The plugins are used by the ``bcfg2-quux`` command, which itself takes +two options: ``--plugins`` (which selects the plugins) and +``--test``. The options would be selected at runtime, so for instance +these would be valid: + +.. code-block:: bash + + bcfg2-quux --plugins Foo --foo --test + bcfg2-quux --plugins Foo,Bar --foo --bar --force + bcfg2-quux --plugins Bar --force + +But this would not: + + bcfg2-quux --plugins Foo --bar + +The help message would reflect the options that are available to the +default set of plugins. (For this reason, allowing component lists to +be set in the config file is very useful; that way, usage messages +reflect the components in the config file.) + +Components (in this example, the plugins) can be classes or modules. +There is no required interface for an option component. They may +*optionally* have: + +* An ``options`` attribute that is a list of + :class:`Bcfg2.Options.Options.Option` objects or option groups. +* A boolean ``parse_first`` attribute; if set to True, the options for + the component are parsed before all other options. This is useful + for, e.g., Django database settings, which must be parsed before + plugins that use Django can be loaded. +* A function or static method, ``options_parsed_hook``, that is called + when all options have been parsed. (This will be called again if + :func:`Bcfg2.Options.Parser.Parser.reparse` is called.) +* A function or static method, ``component_parsed_hook``, that is + called when early option parsing for a given component has + completed. This is *only* called for components with + ``parse_first`` set to True. It is passed a single argument: a + :class:`argparse.Namespace` object containing the complete set of + early options. + +Options are collected through two primary mechanisms: + +#. The :class:`Bcfg2.Options.Actions.ComponentAction` class. When a + ComponentAction subclass is used as the action of an option, then + options contained in the classes (or modules) given in the option + value will be added to the parser. +#. Modules that are not loaded via a + :class:`Bcfg2.Options.Actions.ComponentAction` option may load + options at runtime. + +Since it is preferred to add components instead of just options, +loading options at runtime is generally best accomplished by creating +a container object whose only purpose is to hold options. For +instance: + +.. code-block:: python + + def foo(): + # do stuff + + class _OptionContainer(object): + options = [ + Bcfg2.Options.BooleanOption("--foo", help="Enable foo")] + + @staticmethod + def options_parsed_hook(): + if Bcfg2.Options.setup.foo: + foo() + + Bcfg2.Options.get_parser().add_component(_OptionContainer) + +The Bcfg2.Options module +======================== + +.. currentmodule:: Bcfg2.Options + +.. autodata:: setup + +Options +------- + +The base :class:`Bcfg2.Options.Option` object represents an option. +Unlike options in :mod:`argparse`, an Option object does not need to +be associated with an option parser; it exists on its own. + +.. autoclass:: Option +.. autoclass:: PathOption +.. autoclass:: BooleanOption +.. autoclass:: PositionalArgument + +The Parser +---------- + +.. autoclass:: Parser +.. autofunction:: get_parser +.. autoexception:: OptionParserException + +Option Groups +------------- + +Options can be grouped in various meaningful ways. This uses a +variety of :mod:`argparse` functionality behind the scenes. + +In all cases, options can be added to groups in-line by simply +specifying them in the object group constructor: + +.. code-block:: python + + options = [ + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.Option(...), + Bcfg2.Options.Option(...), + required=True), + ....] + +Nesting object groups is supported in theory, but barely tested. + +.. autoclass:: OptionGroup +.. autoclass:: ExclusiveOptionGroup +.. autoclass:: Subparser +.. autoclass:: WildcardSectionGroup + +Subcommands +----------- + +This library makes it easier to work with programs that have a large +number of subcommands (e.g., :ref:`bcfg2-info <server-bcfg2-info>` and +:ref:`bcfg2-admin <server-admin-index>`). + +The normal implementation pattern is this: + +#. Define all of your subcommands as children of + :class:`Bcfg2.Options.Subcommand`. +#. Define a :class:`Bcfg2.Options.CommandRegistry` object that will be + used to register all of the commands. Registering a command + collect its options and adds it as a + :class:`Bcfg2.Options.Subparser` option group to the main option + parser. +#. Register your commands with + :func:`Bcfg2.Options.register_commands`, parse options, and run. + +:mod:`Bcfg2.Server.Admin` provides a fairly simple implementation, +where the CLI class is itself the command registry: + +.. code-block:: python + + class CLI(Bcfg2.Options.CommandRegistry): + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, + globals().values(), + parent=AdminCmd) + parser = Bcfg2.Options.get_parser( + description="Manage a running Bcfg2 server", + components=[self]) + parser.parse() + +In this case, commands are collected from amongst all global variables +(the most likely scenario), and they must be children of +:class:`Bcfg2.Server.Admin.AdminCmd`, which itself subclasses +:class:`Bcfg2.Options.Subcommand`. + +Commands are defined by subclassing :class:`Bcfg2.Options.Subcommand`. +At a minimum, the :func:`Bcfg2.Options.Subcommand.run` method must be +overridden, and a docstring written. + +.. autoclass:: Subcommand +.. autoclass:: HelpCommand +.. autoclass:: CommandRegistry +.. autofunction:: register_commands + +Actions +------- + +Several custom argparse `actions +<http://docs.python.org/dev/library/argparse.html#action>`_ provide +some of the option collection magic of :mod:`Bcfg2.Options`. + +.. autoclass:: ConfigFileAction +.. autoclass:: ComponentAction +.. autoclass:: PluginsAction + +Option Types +------------ + +:mod:`Bcfg2.Options` provides a number of useful types for use as the `type +<http://docs.python.org/dev/library/argparse.html#type>`_ keyword +argument to +the :class:`Bcfg2.Options.Option` constructor. + +.. autofunction:: Bcfg2.Options.Types.path +.. autofunction:: Bcfg2.Options.Types.comma_list +.. autofunction:: Bcfg2.Options.Types.colon_list +.. autofunction:: Bcfg2.Options.Types.octal +.. autofunction:: Bcfg2.Options.Types.username +.. autofunction:: Bcfg2.Options.Types.groupname +.. autofunction:: Bcfg2.Options.Types.timeout +.. autofunction:: Bcfg2.Options.Types.size + +Common Options +-------------- + +.. autoclass:: Common diff --git a/doc/development/plugins.txt b/doc/development/plugins.txt index 3f2a888ac..5993c4e29 100644 --- a/doc/development/plugins.txt +++ b/doc/development/plugins.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _development-plugins: @@ -128,13 +129,15 @@ The two attributes you need to know about are: of the caching mode. See :ref:`server-caching` for a description of each mode. * :attr:`Bcfg2.Server.Core.metadata_cache`: A dict-like - :class:`Bcfg2.Cache.Cache` object that stores the cached data. + :class:`Bcfg2.Server.Cache.Cache` object that stores the cached + data. :class:`Bcfg2.Server.Plugin.base.Plugin` objects have access to the :class:`Bcfg2.Server.Core` object as ``self.core``. In general, -you'll be interested in the :func:`Bcfg2.Cache.Cache.expire` method; -if called with no arguments, it expires all cached data; if called -with one string argument, it expires cached data for the named client. +you'll be interested in the :func:`Bcfg2.Server.Cache.Cache.expire` +method; if called with no arguments, it expires all cached data; if +called with one string argument, it expires cached data for the named +client. It's important, therefore, that your Connector plugin can either track when changes are made to the group membership it reports, and expire @@ -145,9 +148,8 @@ For examples, see: * :func:`Bcfg2.Server.Plugins.Probes.ReceiveData` takes a copy of the groups that have been assigned to a client by - :ref:`server-plugins-probes-index`, and if that data changes when - new probe data is received, it invalidates the cache for that - client. + :ref:`server-plugins-probes`, and if that data changes when new probe + data is received, it invalidates the cache for that client. * :func:`Bcfg2.Server.Plugins.GroupPatterns.Index` expires the entire cache whenever a FAM event is received for the :ref:`server-plugins-grouping-grouppatterns` config file. @@ -163,7 +165,7 @@ Tracking Execution Time .. versionadded:: 1.3.0 Statistics can and should track execution time statistics using -:mod:`Bcfg2.Statistics`. This module tracks execution time for the +:mod:`Bcfg2.Server.Statistics`. This module tracks execution time for the server core and for plugins, and exposes that data via ``bcfg2-admin perf``. This data can be invaluable for locating bottlenecks or other performance issues. @@ -184,13 +186,13 @@ This will track the execution time of ``do_something``. More granular usage is possible by using :func:`time.time` to manually determine the execution time of a given event and calling -:func:`Bcfg2.Statistics.Statistics.add_value` with an appropriate +:func:`Bcfg2.Server.Statistics.Statistics.add_value` with an appropriate statistic name. -Bcfg2.Statistics -^^^^^^^^^^^^^^^^ +Bcfg2.Server.Statistics +^^^^^^^^^^^^^^^^^^^^^^^ -.. automodule:: Bcfg2.Statistics +.. automodule:: Bcfg2.Server.Statistics Plugin Helper Classes --------------------- diff --git a/doc/development/setup.txt b/doc/development/setup.txt index 05ad4157f..42aa0b023 100644 --- a/doc/development/setup.txt +++ b/doc/development/setup.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _development-setup: @@ -12,6 +13,11 @@ Checking Out a Copy of the Code git clone https://github.com/Bcfg2/bcfg2.git +.. note:: + + The URL above is read-only. If you are planning on submitting patches + upstream, please see :ref:`development-submitting-patches`. + * Add :file:`bcfg2/src/sbin` to your :envvar:`PATH` environment variable * Add :file:`bcfg2/src/lib` to your :envvar:`PYTHONPATH` environment variable diff --git a/doc/development/submitting-patches.txt b/doc/development/submitting-patches.txt new file mode 100644 index 000000000..04492e6e1 --- /dev/null +++ b/doc/development/submitting-patches.txt @@ -0,0 +1,144 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _development-submitting-patches: + +================== +Submitting Patches +================== + +The purpose of this document is to assist those who may be less familiar +with git in submitting patches upstream. While git is powerful, it can +be somewhat confusing to those who don't use it regularly (and even +those who do). + +.. note:: + + We prefer more in-depth commit messages than those + given below which are purely for brevity in this guide. See + http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html + for more about creating proper git commit messages. + +.. _Github: https://github.com/ + +`Github`_ +========= + +These steps outline one way of submitting patches via `Github`_. First, +you will want to `fork <https://github.com/Bcfg2/bcfg2/fork>`_ the +upstream Bcfg2 repository. + +Create a local branch +--------------------- + +Once you have forked the upstream repository, you should clone a local +copy (where <YOUR USERNAME> is your github username). + +:: + + git clone git@github.com:<YOUR USERNAME>/bcfg2.git + +Create a local feature/bugfix branch off the appropriate upstream +branch. For example, let's say we want to submit a bugfix for +:program:`bcfg2-info` against the 1.2.x series. We can create a +``fix-bcfg2-info`` branch which is a copy of the ``maint-1.2`` branch. + +:: + + git branch fix-bcfg2-info maint-1.2 + git checkout fix-bcfg2-info + +Commit changes to your local branch +----------------------------------- + +Next make whatever changes need to be made and commit them to the +``fix-bcfg2-info`` branch. + +:: + + git add src/sbin/bcfg2-info + git commit -m "Fix bcfg2-info bug" + +Now you need to push your ``fix-bcfg2-info`` branch to github. + +:: + + git push origin fix-bcfg2-info + +Submit pull request +------------------- + +Next, submit a pull request against the proper branch (in this case, +https://github.com/username/bcfg2/pull/new/fix-bcfg2-info -- again, +username is your github username). At the top of the pull request, you can +edit the upstream branch you are targetting so that you create the pull +request against the proper upstream branch (in this case, ``maint-1.2``). + +All that's left to do is to write up a description of your pull request +and click **Send pull request**. Since your local branch is specific to +this fix, you can add additional commits if needed and push them. They +will automatically be added to the pull request. + +Remove local branch +------------------- + +Once we have merged your pull request, you can safely delete your local +feature/bugfix branch. To do so, you must first checkout a different branch. + +:: + + git checkout master # switch to a different branch + git branch -d fix-bcfg2-info # delete your local copy of fix-bcfg2-info + git push origin :fix-bcfg2-info # delete fix-bcfg2-info from github + +Mailing List +============ + +The following lists the steps needed to use git's facilities for +emailing patches to the mailing list. + +Commit changes to your local clone +---------------------------------- + +For example, let's say we want to fix a big in :program:`bcfg2-info`. +For the 1.2.x series. + +:: + + git clone https://github.com/Bcfg2/bcfg2.git + git checkout maint-1.2 + # make changes + git add src/sbin/bcfg2-info + git commit -m "Fix bcfg2-info bug" + +Setup git for gmail (optional) +------------------------------ + +If you would like to use the GMail SMTP server, you can add the following +to your ~/.gitconfig file as per the :manpage:`git-send-email(1)` manpage. + +:: + + [sendemail] + smtpencryption = tls + smtpserver = smtp.gmail.com + smtpuser = yourname@gmail.com + smtpserverport = 587 + +Format patches +-------------- + +Use git to create patches formatted for email with the following. + +:: + + git format-patch --cover-letter -M origin/maint-1.2 -o outgoing/ + + +Send emails to the mailing list +------------------------------- + +Edit ``outgoing/0000-*`` and then send your emails to the mailing list +(bcfg-dev@lists.mcs.anl.gov):: + + git send-email outgoing/* diff --git a/doc/development/unit-testing.txt b/doc/development/unit-testing.txt index 7311f49d7..8007e8c75 100644 --- a/doc/development/unit-testing.txt +++ b/doc/development/unit-testing.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _development-unit-testing: @@ -13,7 +14,7 @@ You will first need to install the `Python Mock Module`_ and `Python Nose`_ modules. You can then run the existing tests with the following: -.. code-block: bash +.. code-block: sh cd testsuite nosetests @@ -123,7 +124,7 @@ writing tests for the base :class:`Bcfg2.Server.Plugin.base.Plugin` class, which all Bcfg2 :ref:`server-plugins-index` inherit from via the :mod:`Plugin interfaces <Bcfg2.Server.Plugin.interfaces>`, yielding several levels of often-multiple inheritance. To make this -easier, our unit tests adhere to several design considerations: +easier, our unit tests adhere to several design considerations. Inherit Tests ------------- diff --git a/doc/exts/xmlschema.py b/doc/exts/xmlschema.py index c26aed81e..89104c2a6 100644 --- a/doc/exts/xmlschema.py +++ b/doc/exts/xmlschema.py @@ -784,7 +784,7 @@ class XMLDomain(Domain): def clear_doc(self, docname): to_del = [] for dtype in self.types.keys(): - for key, (doc, _) in self.data[dtype].iteritems(): + for key, (doc, _) in self.data[dtype].items(): if doc == docname: to_del.append((dtype, key)) for dtype, key in to_del: @@ -803,7 +803,7 @@ class XMLDomain(Domain): def get_objects(self): for dtype in self.types.keys(): - for name, (docname, tgtid) in self.data[dtype].iteritems(): + for name, (docname, tgtid) in self.data[dtype].items(): yield (name, name, dtype, docname, tgtid, self.object_types[dtype].attrs['searchprio']) diff --git a/doc/getting_started/index.txt b/doc/getting_started/index.txt index a9b1b847f..f619447e2 100644 --- a/doc/getting_started/index.txt +++ b/doc/getting_started/index.txt @@ -21,7 +21,7 @@ Get and Install Bcfg2 Server We recommend running the server on a Linux machine for ease of deployment due to the availability of packages for the dependencies. -First, you need to download and install Bcfg2. The section +First, you need to download and install Bcfg2. The section :ref:`installation-index` in this manual describes the steps to take. To start, you will need to install the server on one machine and the client on one or more machines. Yes, your server can also be a client @@ -71,7 +71,7 @@ That can be translated as "bcfg2 quick verbose no-op". The output should be something similar to:: Loaded tool drivers: - Chkconfig POSIX YUMng + Chkconfig POSIX YUM Phase: initial Correct entries: 0 @@ -108,7 +108,7 @@ After the above steps, you should have a toplevel repository structure that looks like:: bcfg-server:~ # ls /var/lib/bcfg2 - Base/ Bundler/ Cfg/ Metadata/ Pkgmgr/ Rules/ SSHbase/ etc/ + Bundler/ Cfg/ Metadata/ Pkgmgr/ Rules/ SSHbase/ etc/ The place to start is the Metadata directory, which contains two files: ``clients.xml`` and ``groups.xml``. Your current @@ -169,7 +169,7 @@ Next, we create a motd.xml file in the Bundler directory: .. code-block:: xml - <Bundle name='motd'> + <Bundle> <Path name='/etc/motd' /> </Bundle> @@ -223,7 +223,7 @@ you will find that we now have a correct entry:: Done! Now we just have 242 (or more) entries to take care of! -:ref:`server-plugins-structures-bundler-index` is a +:ref:`server-plugins-structures-bundler` is a relatively easy directory to populate. You can find many samples of Bundles in the :ref:`Bundler Example Repository <server-plugins-structures-bundler-index-examples>`, many of which can @@ -255,6 +255,10 @@ Once you have the server setup, you may be interested in Platform-specific Quickstart Notes ================================== -* :ref:`appendix-guides-centos` -* :ref:`appendix-guides-ubuntu` -* :ref:`getting_started-macosx-notes` +.. toctree:: + :maxdepth: 1 + + CentOS </appendix/guides/centos> + Ubuntu </appendix/guides/ubuntu> + Gentoo </appendix/guides/gentoo> + Mac OS X <macosx/notes> diff --git a/doc/help/troubleshooting.txt b/doc/help/troubleshooting.txt index 72fec4c63..aa46bda2a 100644 --- a/doc/help/troubleshooting.txt +++ b/doc/help/troubleshooting.txt @@ -69,13 +69,13 @@ included with the source distribution and all packages. If the bcfg2 server is not reflecting recent changes, try restarting the bcfg2-server process ============================================================================================= -If this fixes the problem, it is either a bug in the -underlying file monitoring system (fam or gamin) or a bug in -Bcfg2's file monitoring code. In either case, file a `ticket +If this fixes the problem, it is either a bug in the underlying file +monitoring system (inotify or gamin) or a bug in Bcfg2's file +monitoring code. In either case, file a `ticket <https://trac.mcs.anl.gov/projects/bcfg2/newticket>`_ in the tracking system. In the ticket, include: -* filesystem monitoring system (fam or gamin) +* filesystem monitoring system (inotify or gamin) * kernel version (if on linux) * if any messages of the form "Handled N events in M seconds" appeared between the modification event and the client @@ -259,8 +259,7 @@ Server Errors :ref:`server-info` file for this entry. .. [s11] Verify that you have the proper prefix set in bcfg2.conf. .. [s12] Ensure that the client is a member of all the appropriate - :ref:`server-plugins-generators-packages-magic-groups` as - well as any additional groups you may have defined in your + groups you may have defined in your :ref:`server-plugins-generators-packages` configuration. FAQs diff --git a/doc/installation/prerequisites.txt b/doc/installation/prerequisites.txt index f8128b33d..8119be06b 100644 --- a/doc/installation/prerequisites.txt +++ b/doc/installation/prerequisites.txt @@ -60,6 +60,8 @@ Bcfg2 Server +-------------------------------+----------+--------------------------------+ | python-setuptools | Any | | +-------------------------------+----------+--------------------------------+ +| python-genshi | Any | | ++-------------------------------+----------+--------------------------------+ Bcfg2 Reporting --------------- diff --git a/doc/man/bcfg2-admin.txt b/doc/man/bcfg2-admin.txt index 6169ec537..7b19bd366 100644 --- a/doc/man/bcfg2-admin.txt +++ b/doc/man/bcfg2-admin.txt @@ -38,9 +38,6 @@ Modes backup Create an archive of the entire Bcfg2 repository. -bundle *action* - Display details about the available bundles (See BUNDLE OPTIONS - below). client *action* *client* [attribute=value] Add, edit, or remove clients entries in metadata (See CLIENT OPTIONS below). @@ -48,8 +45,12 @@ compare *old* *new* Compare two client configurations. Can be used to verify consistent behavior between releases. Determine differences between files or directories (See COMPARE OPTIONS below). +dbshell + Call the Django 'dbshell' command on the configured database. init Initialize a new repository (interactive). +initreports + Initialize the Reporting database. minestruct *client* [-f xml-file] [-g groups] Build structure entries based on client statistics extra entries (See MINESTRUCT OPTIONS below). @@ -58,34 +59,31 @@ perf pull *client* *entry-type* *entry-name* Install configuration information into repo based on client bad entries (See PULL OPTIONS below). -reports [init|load_stats|purge|scrub|update] - Interact with the dynamic reporting system (See REPORTS OPTIONS - below). -snapshots [init|dump|query|reports] - Interact with the Snapshots database (See SNAPSHOTS OPTIONS below). +purgereports + Purge historic and expired data from the Reporting database +reportssqlall + Call the Django 'shell' command on the Reporting database. +reportsstats + Print Reporting database statistics. +scrubreports + Scrub the Reporting database for duplicate reasons and orphaned + entries. +shell + Call the Django 'shell' command on the configured database. syncdb Sync the Django ORM with the configured database. tidy Remove unused files from repository. +updatereports + Apply database schema updates to the Reporting database. +validatedb + Call the Django 'validate' command on the configured database. viz [-H] [-b] [-k] [-o png-file] Create a graphviz diagram of client, group and bundle information (See VIZ OPTIONS below). xcmd Provides a XML-RPC Command Interface to the bcfg2-server. -BUNDLE OPTIONS -++++++++++++++ - -mode - One of the following. - - *list-xml* - List all available xml bundles - *list-genshi* - List all available genshi bundles - *show* - Interactive dialog to get details about the available bundles - CLIENT OPTIONS ++++++++++++++ @@ -110,11 +108,20 @@ attribute=value COMPARE OPTIONS +++++++++++++++ +-d *N*, --diff-lines *N* + Show only N lines of a diff + +-c, --color + Show colors even if not ryn from a TTY + +-q, --quiet + Only show that entries differ, not how they differ + old - Specify the location of the old configuration file. + Specify the location of the old configuration(s). new - Specify the location of the new configuration file. + Specify the location of the new configuration(s). MINESTRUCT OPTIONS ++++++++++++++++++ @@ -140,51 +147,24 @@ entry type entry name Specify the name of the entry to pull. -REPORTS OPTIONS -+++++++++++++++ - -load_stats [-s] [-c] [-03] - Load statistics data. - -purge [--client [n]] [--days [n]] [--expired] - Purge historic and expired data. - -scrub - Scrub the database for duplicate reasons and orphaned entries. - -update - Apply any updates to the reporting database. - -SNAPSHOTS OPTIONS -+++++++++++++++++ - -init - Initialize the snapshots database. - -query - Query the snapshots database. - -dump - Dump some of the contents of the snapshots database. - -reports [-a] [-b] [-e] [--date=MM-DD-YYYY] - Generate reports for clients in the snapshots database. - VIZ OPTIONS +++++++++++ --H +-H, --includehosts Include hosts in diagram. --b +-b, --includebundles Include bundles in diagram. --o <outfile> +-o *outfile*, --outfile *outfile* Write to outfile file instead of stdout. --k +-k, --includekey Add a shape/color key. +-c *hostname*, --only-client *hostname* + Only show groups and bundles for the named client + See Also -------- diff --git a/doc/man/bcfg2-server.txt b/doc/man/bcfg2-server.txt index 3f8f3ea21..33d0df6cf 100644 --- a/doc/man/bcfg2-server.txt +++ b/doc/man/bcfg2-server.txt @@ -11,7 +11,7 @@ Synopsis **bcfg2-server** [-d] [-v] [-C *configfile*] [-D *pidfile*] [-E *encoding*] [-Q *repo path*] [-S *server url*] [-o *logfile*] [-x -*password*] [--ssl-key=\ *ssl key*] +*password*] [--ssl-key=\ *ssl key*] [--no-fam-blocking] Description ----------- @@ -22,19 +22,20 @@ configurations to clients based on the data in its repository. Options ------- --C configfile Specify alternate bcfg2.conf location. --D pidfile Daemonize, placing the program pid in *pidfile*. --E encoding Specify the encoding of config files. --Q path Specify the path to the server repository. --S server Manually specify the server location (as opposed to - using the value in bcfg2.conf). This should be in - the format "https://server:port" --d Enable debugging output. --v Run in verbose mode. --h Print usage information. ---ssl-key=key Specify the path to the SSL key. +-C configfile Specify alternate bcfg2.conf location. +-D pidfile Daemonize, placing the program pid in *pidfile*. +-E encoding Specify the encoding of config files. +-Q path Specify the path to the server repository. +-S server Manually specify the server location (as opposed to + using the value in bcfg2.conf). This should be in + the format "https://server:port" +-d Enable debugging output. +-v Run in verbose mode. +-h Print usage information. +--ssl-key=key Specify the path to the SSL key. +--no-fam-blocking Synonym for fam_blocking = False in bcfg2.conf See Also -------- -:manpage:`bcfg2(1)`, :manpage:`bcfg2-lint(8)` +:manpage:`bcfg2(1)`, :manpage:`bcfg2-lint(8)`, :manpage:`bcfg2.conf(5)` diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index f55540968..7c265f263 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -43,14 +43,13 @@ filemonitor inotify gamin - fam pseudo fam_blocking Whether the server should block at startup until the file monitor backend has processed all events. This can cause a slower startup, but ensure that all files are recognized before the first client - is handled. + is handled. Defaults to True. ignore_files A comma-separated list of globs that should be ignored by the file @@ -76,24 +75,22 @@ plugins A comma-delimited list of enabled server plugins. Currently available plugins are:: - Account - Base + ACL Bundler Bzr Cfg Cvs Darcs - DBStats Decisions + Defaults Deps - Editor FileProbes Fossil Git + GroupLogic GroupPatterns Guppy Hg - Hostbase Ldap Metadata NagiosGen @@ -108,14 +105,9 @@ plugins Rules SEModules ServiceCompat - Snapshots SSHbase - SSLCA - Statistics Svn - TCheetah TemplateHelper - TGenshi Trigger Descriptions of each plugin can be found in their respective @@ -158,25 +150,10 @@ Server Plugins This section has a listing of all the plugins currently provided with Bcfg2. -Account Plugin -++++++++++++++ - -The account plugin manages authentication data, including the following. - -* ``/etc/passwd`` -* ``/etc/group`` -* ``/etc/security/limits.conf`` -* ``/etc/sudoers`` -* ``/root/.ssh/authorized_keys`` - -Base Plugin -+++++++++++ +ACL Plugin +++++++++++ -The Base plugin is a structure plugin that provides the ability -to add lists of unrelated entries into client configuration entry -inventories. Base works much like Bundler in its file format. This -structure plugin is good for the pile of independent configs needed for -most actual systems. +The ACL plugin controls which hosts can make which XML-RPC calls. Bundler Plugin ++++++++++++++ @@ -203,25 +180,20 @@ contents for clients. In its simplest form, the Cfg repository is just a directory tree modeled off of the directory tree on your client machines. -Cvs Plugin (experimental) -+++++++++++++++++++++++++ +Cvs Plugin +++++++++++ The Cvs plugin allows you to track changes to your Bcfg2 repository using a Concurrent version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -Darcs Plugin (experimental) -+++++++++++++++++++++++++++ +Darcs Plugin +++++++++++++ The Darcs plugin allows you to track changes to your Bcfg2 repository using a Darcs version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -DBStats Plugin -++++++++++++++ - -Direct to database statistics plugin. - Decisions Plugin ++++++++++++++++ @@ -245,13 +217,6 @@ Deps Plugin The Deps plugin allows you to make a series of assertions like "Package X requires Package Y (and optionally also Package Z etc.)" -Editor Plugin -+++++++++++++ - -The Editor plugin attempts to allow you to partially manage -configuration for a file. Its use is not recommended and not well -documented. - FileProbes Plugin +++++++++++++++++ @@ -274,6 +239,12 @@ The Git plugin allows you to track changes to your Bcfg2 repository using a Git version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. +GroupLogic Plugin ++++++++++++++++++ + +The GroupLogic plugin lets you flexibly assign group membership with a +Genshi template. + GroupPatterns Plugin ++++++++++++++++++++ @@ -286,18 +257,13 @@ Guppy Plugin The Guppy plugin is used to trace memory leaks within the bcfg2-server process using Guppy. -Hg Plugin (experimental) -++++++++++++++++++++++++ +Hg Plugin ++++++++++ The Hg plugin allows you to track changes to your Bcfg2 repository using a Mercurial version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -Hostbase Plugin -+++++++++++++++ - -The Hostbase plugin is an IP management system built on top of Bcfg2. - Ldap Plugin +++++++++++ @@ -316,8 +282,8 @@ NagiosGen Plugin The NagiosGen plugin dynamically generates Nagios configuration files based on Bcfg2 data. -Ohai Plugin (experimental) -++++++++++++++++++++++++++ +Ohai Plugin ++++++++++++ The Ohai plugin is used to detect information about the client operating system. The data is reported back to the server using JSON. @@ -373,10 +339,10 @@ dynamic reporting system. Rules Plugin ++++++++++++ -The Rules plugin provides literal configuration entries that resolve the -abstract configuration entries normally found in the Bundler and Base -plugins. The literal entries in Rules are suitable for consumption by -the appropriate client drivers. +The Rules plugin provides literal configuration entries that resolve +the abstract configuration entries normally found in Bundler. The +literal entries in Rules are suitable for consumption by the +appropriate client drivers. SEModules Plugin ++++++++++++++++ @@ -389,12 +355,6 @@ ServiceCompat Plugin The ServiceCompat plugin converts service entries for older clients. -Snapshots Plugin -++++++++++++++++ - -The Snapshots plugin stores various aspects of a client’s state when the -client checks in to the server. - SSHbase Plugin ++++++++++++++ @@ -402,17 +362,6 @@ The SSHbase generator plugin manages ssh host keys (both v1 and v2) for hosts. It also manages the ssh_known_hosts file. It can integrate host keys from other management domains and similarly export its keys. -SSLCA Plugin -++++++++++++ - -The SSLCA plugin is designed to handle creation of SSL privatekeys and -certificates on request. - -Statistics -++++++++++ - -The Statistics plugin is deprecated (see Reporting). - Svn Plugin ++++++++++ @@ -420,20 +369,6 @@ The Svn plugin allows you to track changes to your Bcfg2 repository using a Subversion backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -TCheetah Plugin -+++++++++++++++ - -The TCheetah plugin allows you to use the cheetah templating system to -create files. It also allows you to include the results of probes -executed on the client in the created files. - -TGenshi Plugin -++++++++++++++ - -The TGenshi plugin allows you to use the Genshi templating system to -create files. It also allows you to include the results of probes -executed on the client in the created files. - Trigger Plugin ++++++++++++++ @@ -512,7 +447,7 @@ settings used for client-server communication. sets the password to use to connect to the server. protocol - Communication protocol to use. Defaults to xmlrpc/ssl. + Communication protocol to use. Defaults to xmlrpc/tlsv1. retries A client-only option. Number of times to retry network @@ -602,6 +537,10 @@ Packages options The following options are specified in the **[packages]** section. + backends + Comma separated list of backends for the dependency resolution. + Default is "Yum,Apt,Pac,Pkgng". + resolver Enable dependency resolution. Default is 1 (true). @@ -667,25 +606,12 @@ the configuration file. running in paranoid mode. Only the most recent versions of these copies will be kept. -Snapshots options ------------------ - -Specified in the **[snapshots]** section. These options control the -server snapshots functionality. - - driver - sqlite - - database - The name of the database to use for statistics data. - - e.g.: ``$REPOSITORY_DIR/etc/bcfg2.sqlite`` - -SSLCA options -------------- +SSL CA options +-------------- -These options are necessary to configure the SSLCA plugin and can be -found in the **[sslca_default]** section of the configuration file. +These options are necessary to configure the SSL CA feature of the Cfg +plugin and can be found in the **[sslca_default]** section of the +configuration file. config Specifies the location of the openssl configuration file for @@ -710,7 +636,7 @@ Server-only, specified in the **[database]** section. These options control the database connection of the server. engine - The database engine used by the statistics module. One of the + The database engine used by server plugins. One of the following:: postgresql @@ -719,9 +645,9 @@ control the database connection of the server. ado_mssql name - The name of the database to use for statistics data. If + The name of the database to use for server data. If 'database_engine' is set to 'sqlite3' this is a file path to - the sqlite file and defaults to ``$REPOSITORY_DIR/etc/brpt.sqlite``. + the sqlite file and defaults to ``$REPOSITORY_DIR/etc/bcfg2.sqlite``. user User for database connections. Not used for sqlite3. @@ -739,6 +665,41 @@ control the database connection of the server. Various options for the database connection. The value expected is the literal value of the django OPTIONS setting. + reporting_engine + The database engine used by the Reporting plugin. One of the + following:: + + postgresql + mysql + sqlite3 + ado_mssql + + If reporting_engine is not specified, the Reporting plugin uses + the same database as the other server plugins. + + reporting_name + The name of the database to use for reporting data. If + 'database_engine' is set to 'sqlite3' this is a file path to + the sqlite file and defaults to + ``$REPOSITORY_DIR/etc/reporting.sqlite``. + + reporting_user + User for reporting database connections. Not used for sqlite3. + + reporting_password + Password for reporting database connections. Not used for sqlite3. + + reporting_host + Host for reporting database connections. Not used for sqlite3. + + reporting_port + Port for reporting database connections. Not used for sqlite3. + + reporting_options + Various options for the reporting database connection. The value + is expected as multiple key=value pairs, separated with commas. + The concrete value depends on the database engine. + Reporting options ----------------- @@ -754,6 +715,10 @@ Reporting options web_debug Turn on Django debugging. + max_children + Maximum number of children for the reporting collector. Use 0 to + disable the limit. (default is 0) + See Also -------- diff --git a/doc/releases/1.4.0pre1.txt b/doc/releases/1.4.0pre1.txt new file mode 100644 index 000000000..779873f41 --- /dev/null +++ b/doc/releases/1.4.0pre1.txt @@ -0,0 +1,182 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.4.0pre1: + +1.4.0pre1 +========= + +The first prerelease for Bcfg2 1.4.0 is now available at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +Bcfg2 1.4.0pre1 is a prerelease, and contains many new features, +including some that are backwards-incompatible with Bcfg2 1.3.x and +earlier. Please read the release notes thoroughly. This is a prerelease +and as such is not likely suitable for general production deployment. +That said, please help us test the release in non- and preproduction +environments. + +backwards-incompatible user-facing changes +------------------------------------------ + +* Completely rewrote option parser + + Many single character options now have long equivalents. Some + subcommand interfaces (``bcfg2-info``, ``bcfg2-admin``) have been + reorganized to some degree. ``bcfg2-reports`` syntax is completely + different. + +* Added new :ref:`server-plugins-misc-acl` plugin + + Default ACLs only allow clients to perform bcfg2 client runs, and only + permit `bcfg2-admin xcmd` calls from localhost. If you want to change + this, you must enable the ACL plugin and configure your own ACLs. + +* Added genshi requirement for the server + +* :ref:`server-plugins-generators-decisions` + + * Switch plugin to use StructFile instead of host- or group-specific XML + files (this allows a single e.g. whitelist.xml file with <Group> tags) + + You can convert your existing decisions using + ``tools/upgrade/1.4/migrate_decisions.py``. + + +deprecated features (will be removed in a future release, likely 1.5) +--------------------------------------------------------------------- + +* :ref:`server-plugins-structures-bundler` + + * Deprecated use of an explicit name attribute + + You can convert your existing bundles using + ``tools/upgrade/1.4/convert_bundles.py``. + + * Deprecated :ref:`.genshi bundles + <server-plugins-structures-bundler-index-genshi-templates>` (use + .xml bundles and specify the genshi namespace instead) + +* SSLCA + + * Deprecated plugin + * SSLCA functionality has been added to the Cfg plugin: + see :ref:`server-plugins-generators-cfg-ssl-certificates` + +deprecated plugins and features which have been removed +------------------------------------------------------- + +Plugins +^^^^^^^ + +* PostInstall +* TGenshi +* TCheetah +* Account +* Hostbase +* Snapshots +* Statistics +* Editor +* Base + +Client tools +^^^^^^^^^^^^ + +* RPMng +* YUM24 +* YUMng + +Other features +^^^^^^^^^^^^^^ + +* FAM filemonitor +* Removed mode="inherit" support +* Removed support for .cat/.diff files +* Removed support for info/:info files +* Removed "magic" groups (for the Packages plugin) + +other fixes and new features +---------------------------- + +* Added :ref:`inter-bundle dependencies + <server-plugins-structures-bundler-index-dependencies>` +* Added support for :ref:`independent bundles + <server-plugins-structures-bundler-index-disabling-magic>` (replaces + the functionality of Base): +* Added support for wildcard XIncludes +* Add Solaris 11 IPS Package support +* Add bcfg2-report-collector init script to debian package +* Git VCS plugin enhancements +* Removed deprecated plugins + +* :ref:`server-plugins-structures-bundler` + + * Deprecated use of an explicit name attribute + * Deprecated .genshi bundles + * Added path globbing + +* :ref:`server-plugins-grouping-metadata` + + * Allow setting global default authentication type + +* :ref:`server-plugins-generators-packages` + + * Add yum group support to internal resolver + * Change location of plugin-generated APT sources + * Add new Pkgng plugin + * Add ability for per-package recommended flag override + +* :ref:`server-plugins-statistics-reporting` + + * Add support for POSIX user/group entries + * Add support for Django > 1.4 + * Add support for separate reporting database + +* Added option to periodically dump performance stats to logs +* Added option to force server to wait until all FAM events are + processed + +* :ref:`server-plugins-generators-sshbase` + + * Add support for IPv6 addresses in known_hosts file + * Add support for :ref:`encryption of generated ssh keys + <server-plugins-generators-sshbase-encryption>` + +* APT + + * Allow specification of deb-src lines (resolves + http://trac.mcs.anl.gov/projects/bcfg2/ticket/1148) + +* SSLCA + + * Rewrote SSLCA as Cfg handler + + Existing SSLCA installations will need to migrate to the new format + using ``tools/upgrade/1.4/migrate_sslca.py``. + +* :ref:`server-plugins-generators-nagiosgen` + + * Migrate configuration to conf.d + +* :ref:`server-plugins-probes` + + * Rewritten to improve caching + * Add probes.allowed_groups option to restrict group assignments: + see :ref:`server-plugins-probes-dynamic-groups` + + +Thanks +------ + +Special thanks to the following contributors for this release + + * Alexander Sulfrain + * Chris Brinker + * Duncan Hutty + * Jason Kincl + * John Morris + * Matt Schwager + * Michael Fenn + * Stéphane Graber + * Tim Laszlo diff --git a/doc/releases/1.4.0pre2.txt b/doc/releases/1.4.0pre2.txt new file mode 100644 index 000000000..7bbed5603 --- /dev/null +++ b/doc/releases/1.4.0pre2.txt @@ -0,0 +1,37 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.4.0pre2: + +1.4.0pre2 +========= + +The second prerelease for Bcfg2 1.4.0 is now available at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +Bcfg2 1.4.0pre2 is a prerelease, and contains many new features, +including some that are backwards-incompatible with Bcfg2 1.3.x and +earlier. Please read the release notes thoroughly. This is a prerelease +and as such is not likely suitable for general production deployment. +That said, please help us test the release in non- and preproduction +environments. + +backwards-incompatible user-facing changes +------------------------------------------ + +* Changed default communication protocol to xmlrpc/tlsv1 + +* Diff output from files sent to the Reports plugin from the client will now be + in a unified diff format rather than the previous n-diff format. + + This fixes potentially long client runs when comparing files that have + diverged significantly. + +Thanks +------ + +Special thanks to the following contributors for this release + + * Alexander Sulfrain + * Matt Kemp diff --git a/doc/releases/index.txt b/doc/releases/index.txt index 1ed644f9c..271fc23cc 100644 --- a/doc/releases/index.txt +++ b/doc/releases/index.txt @@ -7,5 +7,8 @@ Release Announcements ===================== -.. include:: 1.3.5.txt -.. include:: 1.3.4.txt +.. toctree:: + + 1.4.0pre1 + 1.3.5 + 1.3.4 diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 6b8a1f467..38d4c7e3a 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -25,6 +25,7 @@ configuration. Specific features in the new system include: users to drill down to find out about a :ref:`specific host <reports-client-detail>`, rather than only having one huge page with too much information. +* Ability to store reporting data separately from other server data. Installation ============ @@ -214,8 +215,8 @@ database ^^^^^^^^ If you choose to use a different database, you'll need to edit -``/etc/bcfg2.conf``. These fields should be updated in the [database] -section: +``/etc/bcfg2.conf``. These fields should be updated in the +``[database]`` section: * engine @@ -228,11 +229,27 @@ section: * host * port (optional) +To store reporting data separately from the main server data, use +the following options: + +* reporting_engine + + * ex: reporting_engine = mysql + * ex: reporting_engine = postgresql_psycopg2 + +* reporting_name +* reporting_user +* reporting_password +* reporting_host +* reporting_port (optional) + .. warning:: If mysql is used as a backend, it is recommended to use InnoDB for the `storage engine <http://dev.mysql.com/doc/refman/5.1/en/storage-engine-setting.html>`_. +Refer to :ref:`server-database` for a full listing of +available options. statistics ^^^^^^^^^^ @@ -253,6 +270,9 @@ reporting * web_prefix: Prefix to be added to Django's MEDIA_URL * file_limit: The maximum size of a diff or binary data to store in the database. +* max_children: Maximum number of children for the reporting + collector. Use 0 to disable the limit and spawn a thread + as soon as a working file is available. .. _dynamic_transports: diff --git a/doc/reports/index.txt b/doc/reports/index.txt index 1360d5ffd..aaed29dfe 100644 --- a/doc/reports/index.txt +++ b/doc/reports/index.txt @@ -24,5 +24,4 @@ uses django and a database backend. .. toctree:: :maxdepth: 2 - static dynamic diff --git a/doc/reports/static.txt b/doc/reports/static.txt deleted file mode 100644 index 00c1867f8..000000000 --- a/doc/reports/static.txt +++ /dev/null @@ -1,100 +0,0 @@ -.. -*- mode: rst -*- - -.. _reports-static: - -============================= -Bcfg2 Static Reporting System -============================= - -The Bcfg2 reporting system collects and displays information about the -operation of the Bcfg2 client, and the configuration states of target -machines. - -Goals -===== - -The reporting system provides an interface to administrators describing -a few important tasks - -* Client configuration state, particularly aspects that do not match the configuration specification. - Information about bad and extra configuration elements is included. -* Client execution results (a list of configuration elements that were modified) -* Client execution performance data (including operation retry counts, and timings for several critical execution regions) - -This data can be used to understand the current configuration state -of the entire network, the operations performed by the client, how the -configuration changes propagate, and any reconfiguration operations that -have failed. - -Retention Model -=============== - -The current reporting system stores statistics in an XML data store, by -default to ``<repo>/etc/statistics.xml``. It retains either one or two -statistic sets per host. If the client has a clean configuration state, -the most recent (clean) record is retained. If the client has a dirty -configuration state, two records are retained. One record is the last -clean record. The other record is the most recent record collected, -detailing the incorrect state. - -This retention model, while non-optimal, does manage to persistently -record most of the data that users would like. - -Setup -===== - -In order to configure your Bcfg2 server for receiving reports, you -will need to list the Statistics plugin in the plugins line of your -``bcfg2.conf``. You will also need a [statistics] section -in your ``bcfg2.conf``. You can find out more about what goes there in the -``bcfg2.conf`` manpage. - -Output -====== - -Several output reports can be generated from the statistics store with -the command line tool ``bcfg2-build-reports``. - -* Nodes Digest -* Nodes Individual -* Overview Statistics -* Performance - -The data generated by these reports can be delivered by several -mechanisms: - -* HTML -* Email -* RSS - -Shortcomings and Planned Enhancements -===================================== - -When designing the current reporting system, we were overly concerned with -the potential explosion in data size over time. In order to address this, -we opted to use the retention scheme described above. This approach has -several shortcomings: - -* A comprehensive list of reconfiguration operations (with associated - timestamps) isn't retained -* Client results for any given day (except the last one) aren't uniformly - retained. This means that inter-client analysis is difficult, if - not impossible - -We plan to move to a database backend to address the dataset size -problem and start retaining all information. The move to a SQL backend -will allow many more types of queries to be efficiently processed. It -will also make on-demand reports simpler. - -Other sorts of information would also be useful to track. We plan to -add the ability to tag a particular configuration element as security -related, and include this in reports. This will aid in the effective -prioritization of manual and failed reconfiguration tasks. - -Capability Goals (posed as questions) -------------------------------------- - -* What machines have not yet applied critical updates? -* How long did critical updates take to be applied? -* What configuration did machine X have on a particular date? -* When did machine X perform configuration update Y? diff --git a/doc/server/acl.txt b/doc/server/acl.txt new file mode 100644 index 000000000..3cb9d59a1 --- /dev/null +++ b/doc/server/acl.txt @@ -0,0 +1,41 @@ +.. -*- mode: rst -*- + +.. _server-access-control: + +================ + Access Control +================ + +.. versionadded:: 1.4.0 + +Bcfg2 exposes various functions via XML-RPC calls. Some of these are +relatively benign (e.g., the calls necessary to generate a client +configuration) while others can be used to inspect potentially private +data on the server or very easily mount a denial of service attack. +As a result, access control lists to limit exposure of these calls is +built in. There are two possible ACL methods: built-in, and the +:ref:`server-plugins-misc-acl` plugin. + +The built-in approach simply applies a restrictive default ACL that +lets ``localhost`` perform all XML-RPC calls, and restricts all other +machines to only the calls necessary to run the Bcfg2 client. +Specifically: + +* If the remote client is ``127.0.0.1``, the call is allowed. Note + that, depending on where your Bcfg2 server listens and how it + communicates with itself, it likely will not identify to itself as + ``localhost``. +* If the remote client is not ``127.0.0.1`` and the call is any of the + ``set_debug`` or ``toggle_debug`` methods (including + ``[toggle|set]_core_debug``), it is rejected. +* If the remote client is not ``127.0.0.1`` and the call is + ``get_statistics`` (used by ``bcfg2-admin perf``), it is rejected. +* If the remote client is not ``127.0.0.1`` and the call includes a + ``.`` -- i.e., it is dispatched to any plugin, such as + ``Packages.Refresh`` -- then it is rejected. +* Otherwise, the call is allowed. + +The built-in ACL is *only* intended to ensure that Bcfg2 is secure by +default; it will not be sufficient in many (or even most) cases. In +these cases, it's recommended that you use the +:ref:`server-plugins-misc-acl` plugin. diff --git a/doc/server/admin/bundle.txt b/doc/server/admin/bundle.txt deleted file mode 100644 index e9cb79781..000000000 --- a/doc/server/admin/bundle.txt +++ /dev/null @@ -1,34 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-admin-bundle: - -bundle -====== - -For a list of all available xml bundles use ``list-xml``. ``list-genshi`` -will list all available genshi bundles.:: - -.. code-block:: sh - - # bcfg2-admin bundles list-xml - # bcfg2-admin bundles list-genshi - -``show`` provides an interactive dialog to get details about the available -bundles.:: - -.. code-block:: sh - - # bcfg2-admin bundles show - Available bundles (Number of bundles: 4) - ---------------------------------------- - [0] motd.xml - [1] snmpd.xml - [2] bcfg2.xml - [3] ntp.xml - Enter the line number of a bundle for details: 3 - Details for the "ntp" bundle: - Package: xntp - Path: /etc/sysconfig/xntp - Path: /etc/sysconfig/clock - Path: /etc/ntp.conf - Service: xntpd diff --git a/doc/server/admin/compare.txt b/doc/server/admin/compare.txt index 6a770055e..ffe19efdf 100644 --- a/doc/server/admin/compare.txt +++ b/doc/server/admin/compare.txt @@ -6,11 +6,10 @@ compare ======= Determine differences between files or directories of client -specification instances.:: +specification instances:: bcfg2-admin compare <file1> <file2> -If you want to compare two directories recursively then use ``-r`` as an -option. :: +Or:: - bcfg2-admin compare -r <dir1> <dir2> + bcfg2-admin compare <dir1> <dir2> diff --git a/doc/server/admin/index.txt b/doc/server/admin/index.txt index c563ead9c..707f7c724 100644 --- a/doc/server/admin/index.txt +++ b/doc/server/admin/index.txt @@ -16,14 +16,11 @@ functionality. Available modes are listed below. :maxdepth: 1 backup - bundle client compare init minestruct perf pull - snapshots - tidy viz xcmd diff --git a/doc/server/admin/init.txt b/doc/server/admin/init.txt index 0e8b3afd3..db42c8222 100644 --- a/doc/server/admin/init.txt +++ b/doc/server/admin/init.txt @@ -36,7 +36,6 @@ detected or a default value is provided. :: A toplevel repository structure was created under the provided path. :: /var/lib/bcfg2 - |-- Base |-- Bundler |-- Cfg |-- etc diff --git a/doc/server/admin/snapshots.txt b/doc/server/admin/snapshots.txt deleted file mode 100644 index 25a7286c2..000000000 --- a/doc/server/admin/snapshots.txt +++ /dev/null @@ -1,8 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-admin-snapshots: - -snapshots -========= - -Interact with the Snapshots system. diff --git a/doc/server/admin/tidy.txt b/doc/server/admin/tidy.txt deleted file mode 100644 index 816d6cdb3..000000000 --- a/doc/server/admin/tidy.txt +++ /dev/null @@ -1,8 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-admin-tidy: - -tidy -==== - -Clean up useless files in the repo. diff --git a/doc/server/caching.txt b/doc/server/caching.txt index 51245bd08..3557bf0f3 100644 --- a/doc/server/caching.txt +++ b/doc/server/caching.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-caching: @@ -13,7 +14,7 @@ Metadata Caching Caching (or, rather, cache expiration) is always a difficult problem, but it's particularly vexing in Bcfg2 due to the number of different -data sources incorporated. In 1.3.0, we introduce some limited +data sources incorporated. In 1.3.0, we introduced some limited caching of client metadata objects. Since a client metadata object can be generated anywhere from 7 to dozens of times per client run (depending on your templates), and since client metadata generation @@ -42,15 +43,15 @@ descending order of reliability. That is, odds are higher that biggest speed boost. ``off`` will never result in stale data, but it gives no speed boost. -In addition to the :ref:`server-plugins-grouping-metadata` plugin, -Bcfg2 includes three plugins that can set additional groups, and thus -may affect the caching behavior. They are -:ref:`server-plugins-grouping-grouppatterns`, -:ref:`server-plugins-probes-index`, and +In addition to the :ref:`server-plugins-grouping-metadata` +plugin, Bcfg2 includes three plugins that can set additional +groups, and thus may affect the caching behavior. They +are :ref:`server-plugins-grouping-grouppatterns`, +:ref:`server-plugins-probes`, and :ref:`server-plugins-connectors-puppetenc`. All of those plugins -**except** for PuppetENC fully support all caching levels. PuppetENC -is incompatible with ``aggressive``, and may result in some stale data -with ``cautious``. +**except** for PuppetENC fully support all caching levels. PuppetENC is +incompatible with ``aggressive``, and may result in some stale data with +``cautious``. If you are not using the PuppetENC plugin, and do not have any custom plugins that provide additional groups, then all four modes should be diff --git a/doc/server/configuration.txt b/doc/server/configuration.txt index 400db40b5..79d732f6d 100644 --- a/doc/server/configuration.txt +++ b/doc/server/configuration.txt @@ -55,9 +55,7 @@ itself, which would prevent the ``bcfg2`` user from enabling a new plugin. If you depend on this capability (e.g., if your specification is stored in a VCS and checked out onto the Bcfg2 server by a script running as the ``bcfg2`` user), then you would want to ``chown`` and -``chmod`` ``/var/lib/bcfg2`` rather than ``/var/lib/bcfg2/*``. Note -also that the recursive ``chmod`` will change permissions on any files -that are using ``mode="inherit"`` in :ref:`server-info`. +``chmod`` ``/var/lib/bcfg2`` rather than ``/var/lib/bcfg2/*``. The Bcfg2 server also needs to be able to read its SSL certificate, key and the SSL CA certificate: diff --git a/doc/server/configurationentries.txt b/doc/server/configurationentries.txt index 66ff617c0..446257d62 100644 --- a/doc/server/configurationentries.txt +++ b/doc/server/configurationentries.txt @@ -28,7 +28,7 @@ Example: .. code-block:: xml - <Bundle name='ntp'> + <Bundle> <BoundPackage name='ntp' type='deb' version='1:4.2.4p4+dfsg-3ubuntu2.1'/> </Bundle> diff --git a/doc/server/database.txt b/doc/server/database.txt index 15c66754f..67cb065f4 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-database: @@ -9,51 +10,86 @@ Global Database Settings .. versionadded:: 1.3.0 Several Bcfg2 plugins, including -:ref:`server-plugins-grouping-metadata`, -:ref:`server-plugins-probes-index`, and -:ref:`server-plugins-statistics-reporting`, can connect use a -relational database to store data. They use the global database -settings in ``bcfg2.conf``, described in this document, to connect. +:ref:`server-plugins-grouping-metadata`, :ref:`server-plugins-probes`, and +:ref:`server-plugins-statistics-reporting`, can connect use a relational +database to store data. They use the global database settings in +``bcfg2.conf``, described in this document, to connect. .. note:: Although SQLite is supported as a database, it may cause - significant thread contention (and a performance penalty) if you - use SQLite with :ref:`server-plugins-grouping-metadata` or - :ref:`server-plugins-probes-index`. If you are using the - database-backed features of either of those plugins, it's - recommended that you use a higher performance database backend. + significant thread contention (and a performance penalty) if + you use SQLite with :ref:`server-plugins-grouping-metadata` or + :ref:`server-plugins-probes`. If you are using the database-backed + features of either of those plugins, it's recommended that you use + a higher performance database backend. +Separate Reporting Database +=========================== + +.. versionadded:: 1.4.0 + +Bcfg2 supports storing the data generated by the +:ref:`server-plugins-statistics-reporting` in a separate +database from the data generated by the other plugins (e.g. +:ref:`server-plugins-grouping-metadata` and :ref:`server-plugins-probes`). +To activate this support, set the ``reporting_engine``, +``reporting_name``, ``reporting_user``, etc. options in the +``[database]`` section of the config file. The valid values for the +``reporting_*`` options are the same as for the standard database +options. See :ref:`server-database-configuration-options` for a full +listing. + +.. _server-database-configuration-options: + Configuration Options ===================== All of the following options should go in the ``[database]`` section of ``/etc/bcfg2.conf``. -+-------------+------------------------------------------------------------+-------------------------------+ -| Option name | Description | Default | -+=============+============================================================+===============================+ -| engine | The name of the Django database backend to use. See | "sqlite3" | -| | https://docs.djangoproject.com/en/dev/ref/settings/#engine | | -| | for available options (note that django.db.backends is not | | -| | included in the engine name) | | -+-------------+------------------------------------------------------------+-------------------------------+ -| name | The name of the database | "/var/lib/bcfg2/bcfg2.sqlite" | -+-------------+------------------------------------------------------------+-------------------------------+ -| user | The user to connect to the database as | None | -+-------------+------------------------------------------------------------+-------------------------------+ -| password | The password to connect to the database with | None | -+-------------+------------------------------------------------------------+-------------------------------+ -| host | The host to connect to | "localhost" | -+-------------+------------------------------------------------------------+-------------------------------+ -| port | The port to connect to | None | -+-------------+------------------------------------------------------------+-------------------------------+ -| options | Extra parameters to use when connecting to the database. | None | -| | Available parameters vary depending on your database | | -| | backend. The parameters are supplied as the value of the | | -| | django OPTIONS setting. | | -+-------------+------------------------------------------------------------+-------------------------------+ ++--------------------+------------------------------------------------------------+---------------------------------------+ +| Option name | Description | Default | ++====================+============================================================+=======================================+ +| engine | The name of the Django database backend to use. See | "sqlite3" | +| | https://docs.djangoproject.com/en/dev/ref/settings/#engine | | +| | for available options (note that django.db.backends is not | | +| | included in the engine name) | | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| name | The name of the database | "/var/lib/bcfg2/etc/bcfg2.sqlite" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| user | The user to connect to the database as | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| password | The password to connect to the database with | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| host | The host to connect to | "localhost" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| port | The port to connect to | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| options | Extra parameters to use when connecting to the database. | None | +| | Available parameters vary depending on your database | | +| | backend. The parameters are supplied as the value of the | | +| | django OPTIONS setting. | | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_engine | The name of the Django database backend to use for the | None | +| | reporting database. Takes the same values as ``engine``. | | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_name | The name of the reporting database | "/var/lib/bcfg2/etc/reporting.sqlite" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_user | The user to connect to the reporting database as | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_password | The password to connect to the reporting database with | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_host | The host to connect to for the reporting database | "localhost" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_port | The port to connect to for the reporting database | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_options | Extra parameters to use when connecting to the database. | None | +| | Available parameters vary depending on your database | | +| | backend. The parameters are supplied as the value of the | | +| | django OPTIONS setting. | | ++--------------------+------------------------------------------------------------+---------------------------------------+ Database Schema Sync diff --git a/doc/server/encryption.txt b/doc/server/encryption.txt index e31124d4b..db5e2ae29 100644 --- a/doc/server/encryption.txt +++ b/doc/server/encryption.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-encryption: @@ -24,7 +25,7 @@ feature requires M2Crypto 0.18 or newer. single Bcfg2 repository with multiple admins who should not necessarily have access to each other's sensitive data. -Two types of data can be encrypted: +Two basic types of data can be encrypted: * :ref:`server-plugins-generators-cfg` files can be encrypted as whole files. See :ref:`server-plugins-generators-cfg-encryption` @@ -51,6 +52,13 @@ In general, Properties encryption is preferred for a few reasons: amongst different teams, this lets teams collaborate more closely on files and other data. +Other types of data that can be encrypted are: + +* Text content of Path tags in + :ref:`server-plugins-structures-bundler` +* Passphrases in XML description files for generated + :ref:`server-plugins-generators-cfg-sshkeys` + .. _bcfg2-crypt: bcfg2-crypt @@ -203,6 +211,8 @@ get a list of valid algorithms, you can run:: openssl list-cipher-algorithms | grep -v ' => ' | \ tr 'A-Z-' 'a-z_' | sort -u +.. _server-encryption-lax-strict: + Lax vs. Strict decryption ------------------------- @@ -223,7 +233,10 @@ This can be overridden by individual XML files by setting ``decrypt="strict"`` on the top-level tag (or, vice-versa; if strict is the default an XML file can specify ``decrypt="lax"``. +Note that you could, for instance, set lax decryption by default, and +then set strict decryption on individual files. + Encryption API ============== -.. automodule:: Bcfg2.Encryption +.. automodule:: Bcfg2.Server.Encryption diff --git a/doc/server/genshi-xml.txt b/doc/server/genshi-xml.txt deleted file mode 100644 index 3216cc00d..000000000 --- a/doc/server/genshi-xml.txt +++ /dev/null @@ -1,24 +0,0 @@ -.. -*- mode: rst -*- - -.. _xml-genshi-reference: - -=============================== - Genshi XML Template Reference -=============================== - -Genshi's XML templating language is used in -:ref:`server-plugins-structures-bundler-index` for templated bundles. -The language is described in depth at `Genshi -<http://genshi.edgewall.org>`_. The XML schema reference follows. - -Genshi Tags -=========== - -.. xml:group:: genshiElements - :namespace: py - -Genshi Attributes -================= - -.. xml:attributegroup:: genshiAttrs - :namespace: py diff --git a/doc/server/index.txt b/doc/server/index.txt index 2ccc9c923..ce751f2ae 100644 --- a/doc/server/index.txt +++ b/doc/server/index.txt @@ -26,11 +26,11 @@ clients. admin/index configurationentries info - snapshots/index bcfg2-info selinux configuration database caching encryption - genshi-xml + xml-common + acl diff --git a/doc/server/info.txt b/doc/server/info.txt index b4d1f7113..8342e1cee 100644 --- a/doc/server/info.txt +++ b/doc/server/info.txt @@ -7,8 +7,7 @@ info.xml ======== Various file properties for entries served by most generator plugins, -including :ref:`server-plugins-generators-cfg`, -:ref:`server-plugins-generators-sslca`, and +including :ref:`server-plugins-generators-cfg` and :ref:`server-plugins-generators-sshbase`, are controlled through the use of ``info.xml`` files. @@ -53,25 +52,3 @@ A more complex example for a template that generates both See :ref:`server-selinux` for more information on the ``secontext`` attribute and managing SELinux in general. - -:info and info files -==================== - -.. deprecated:: 1.3.0 - -Historically, Bcfg2 also accepted the use of ``:info`` and ``info`` -files, which function the same as ``info.xml``, but are not XML. They -lack the ability to specify different permissions based on client, -group, or path, and cannot be used to specify ACLs, either. - -An example ``:info`` or ``info`` file would look like:: - - owner: www - group: www - mode: 0755 - -All attributes allowed on the ``<Info>`` tag of an ``info.xml`` file -can be used in an ``:info`` or ``info`` file. - -You should not use more than one ``:info``, ``info``, or ``info.xml`` -file for a single entry. diff --git a/doc/server/plugins/connectors/properties.txt b/doc/server/plugins/connectors/properties.txt index 47e82fdbf..a6f6af741 100644 --- a/doc/server/plugins/connectors/properties.txt +++ b/doc/server/plugins/connectors/properties.txt @@ -120,6 +120,8 @@ in ``bcfg2.conf``:: [properties] writes_enabled = false +.. _server-plugins-connectors-properties-xml: + XML Property Files ------------------ @@ -266,47 +268,13 @@ Encrypted Properties data .. versionadded:: 1.3.0 You can encrypt selected data in XML Properties files to protect that -data from other people who need access to the repository. See -:ref:`server-encryption-configuration` for details on configuring -encryption passphrases. The data is decrypted transparently -on-the-fly by the server; you never need to decrypt the data in your -templates. Encryption is only supported on XML properties files. - -.. note:: - - This feature is *not* intended to secure the files against a - malicious attacker who has gained access to your Bcfg2 server, as - the encryption passphrases are held in plaintext in - ``bcfg2.conf``. This is only intended to make it easier to use a - single Bcfg2 repository with multiple admins who should not - necessarily have access to each other's sensitive data. - -Properties files are encrypted on a per-element basis; that is, rather -than encrypting the whole file, only the character content of -individual elements is encrypted. This makes it easier to track -changes to the file in a VCS, and also lets unprivileged users work -with the other data in the file. Only character content of an element -can be encrypted; attribute content and XML elements themselves cannot -be encrypted. - -By default, decryption is *strict*; that is, if any element cannot be -decrypted, parsing of the file is aborted. If you wish for parsing to -continue, with unencryptable elements simply skipped, then you can set -decryption to *lax* in one of two ways: - -* Set ``decrypt=lax`` in the ``[encryption]`` section of - ``bcfg2.conf`` to set lax decryption on all files by default; or -* Set the ``decrypt="lax"`` attribute on the top-level ``Properties`` - tag of a Properties file to set lax decryption for a single file. - -Note that you could, for instance, set lax decryption by default, and -then set strict decryption on individual files. - -To encrypt or decrypt a file, use :ref:`bcfg2-crypt`. - -See :ref:`server-encryption` for more details on encryption in Bcfg2 -in general. +data from other people who need access to the repository. The +data is decrypted transparently on-the-fly by the server; you never +need to decrypt the data in your templates. Encryption is only +supported on XML properties files. +See :ref:`server-encryption` for details on encryption in general, and +:ref:`xml-encryption` for details on encryption in XML files. Accessing Properties contents from Genshi Templates =================================================== diff --git a/doc/server/plugins/connectors/templatehelper.txt b/doc/server/plugins/connectors/templatehelper.txt index 4b1f66aee..d113dcab7 100644 --- a/doc/server/plugins/connectors/templatehelper.txt +++ b/doc/server/plugins/connectors/templatehelper.txt @@ -31,7 +31,7 @@ helpers will be available to all clients. Writing Helpers =============== -A helper module is just a Python module with three special conditions: +A helper module is just a Python module with several special conditions: * The filename must end with ``.py`` * The module must have an attribute, ``__export__``, that lists all of @@ -43,6 +43,12 @@ A helper module is just a Python module with three special conditions: an underscore or double underscore is bad form, and may also produce errors. +Additionally, the module *may* have an attribute, ``__default__``, +that lists all of the symbols that you wish to include by default in +the template namespace. ``name``, ``metadata``, ``source_path``, +``repo``, and ``path`` are reserved names, and should not be included +in ``__default__``. + See ``examples/TemplateHelper`` for examples of helper modules. Usage @@ -54,17 +60,23 @@ a HelperModule object will have, as attributes, all symbols listed in ``__export__``. For example, consider this helper module:: __export__ = ["hello"] - + __default__ = ["pining"] + def hello(metadata): return "Hello, %s!" % metadata.hostname + def pining(text): + return "It's pinin' for the %s!" % text + To use this in a Genshi template, we could do:: ${metadata.TemplateHelper['hello'].hello(metadata)} + ${pining("fjords")} The template would produce:: Hello, foo.example.com! + It's pinin' for the fjords! Note that the client metadata object is not passed to a helper module in any magical way; if you want to access the client metadata object diff --git a/doc/server/plugins/generators/account.txt b/doc/server/plugins/generators/account.txt deleted file mode 100644 index 99c35c814..000000000 --- a/doc/server/plugins/generators/account.txt +++ /dev/null @@ -1,115 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-plugins-generators-account: - -======= -Account -======= - -The account plugin manages authentication data, including - -* ``/etc/passwd`` -* ``/etc/group`` -* ``/etc/security/limits.conf`` -* ``/etc/sudoers`` -* ``/root/.ssh/authorized_keys`` - -User access data is stored in three files in the Account directory: - -* superusers (a list of users who always have root privs) -* rootlist (a list of user:host pairs for scoped root privs) -* useraccess (a list of user:host pairs for login access) - -SSH keys are stored in files named $username.key; these are installed -into root's authorized keys for users in the superusers list as well as -for the pertitent users in the rootlike file (for the current system). - -Authentication data is read in from (static|dyn).(passwd|group) The static -ones are for system local ones, while the dyn. versions are for external -synchronization (from ldap/nis/etc). There is also a static.limits.conf -that provides the limits.conf header and any static entries. - -Files in the Account directory: - -``<username>.key`` - - **Format**: The SSH public key for user <username>. - - If the user is in the "rootlike" or "superusers" group, these - keys will be appended to ``/root/.ssh/auth`` - -``useraccess`` - - **Format**: "user:hostname" on each line. - - Describes who may login where (via PAMs - ``/etc/security/limits.conf``). Everybody else will be denied - access.(?) - - **Example**: - - If Alice should be able to access host "foo", Bob should access - "foo" and "bar":: - - alice:foo.example.com - bob:foo.example.com - bob:bar.example.com - -``rootlike`` - - **Format**: "user:hostname" on each line. - - Describes who will be allowed root access where. The user may - login via public key and use sudo. - - **Example**: - - If Chris should be root only on host "foo":: - - chris:foo.example.com - -``superusers`` - - **Format**: usernames, separated by spaces or newlines. (Any whitespace that makes pythons split() happy.) - - Describes who will be allowed root access on all hosts. The user - may login via public key and use sudo. - - **Example**: - - Daniel, Eve and Faith are global admins:: - - daniel eve - faith - -``static.passwd``, ``static.group`` - - **Format**: Lines from ``/etc/passwd`` or ``/etc/group`` - - These entries are appended to the passwd and group files - (in addition to the auto-generated entries from "useraccess", - "rootlike" and "superusers" above) without doing anything else. - -``dyn.passwd``, ``dyn.group`` - - **Format**: Lines from ``/etc/passwd`` or ``/etc/group`` - - Similar to "static.*" above, but for entries that are managed "on - the network" (yp, LDAP, ...), so it is most likely periodically - (re)filled. - -``static.limits.conf`` - - **Format**: Lines from ``/etc/security/limit.conf`` - - These limits will be appended to limits.conf (in addition to - the auto-generated entries from "useraccess", "rootlike" and - "superusers" above). - -``static.sudoers`` - - **Format**: Lines from ``/etc/sudoers`` - - These lines will be appended to to sudoers file (in addition - to the auto-generated entries from "useraccess", "rootlike" and - "superusers" above). diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index 541531581..c991f20c9 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-plugins-generators-cfg: @@ -29,8 +30,8 @@ in ``Cfg/etc/passwd/passwd``, while the ssh pam module config file, ``/etc/pam.d/sshd``, goes in ``Cfg/etc/pam.d/sshd/sshd``. The reason for the like-name directory is to allow multiple versions of each file to exist, as described below. Note that these files are exact copies of what -will appear on the client machine (except when using Genshi or Cheetah -templating -- see below). +will appear on the client machine (except when using templates -- see +below). Group-Specific Files ==================== @@ -102,9 +103,8 @@ Genshi Templates ---------------- Genshi templates allow you to use the `Genshi -<http://genshi.edgewall.org>`_ templating system. This is similar to -the deprecated :ref:`server-plugins-generators-tgenshi-index` plugin. -Genshi templates should be named with a ``.genshi`` extension, e.g.:: +<http://genshi.edgewall.org>`_ templating system. Genshi templates +should be named with a ``.genshi`` extension, e.g.:: % ls Cfg/etc/motd info.xml motd.genshi @@ -214,9 +214,8 @@ Cheetah Templates ----------------- Cheetah templates allow you to use the `cheetah templating system -<http://www.cheetahtemplate.org/>`_. This is similar to -the deprecated :ref:`server-plugins-generators-tcheetah` plugin. -Cheetah templates should be named with a ``.cheetah`` extension, e.g.:: +<http://www.cheetahtemplate.org/>`_. Cheetah templates should be +named with a ``.cheetah`` extension, e.g.:: % ls Cfg/etc/motd info.xml motd.cheetah @@ -243,6 +242,27 @@ comment to appear in the final config file.:: # This is a comment in my template which will be stripped when it's processed through Cheetah \# This comment will appear in the generated config file. +.. _server-plugins-generators-cfg-jinja2: + +Jinja2 Templates +----------------- + +Jinja2 templates allow you to use the `jinja2 templating system +<http://jinja.pocoo.org/>`_. Jinja2 templates should be +named with a ``.jinja2`` extension, e.g.:: + + % ls Cfg/etc/motd + info.xml motd.jinja2 + +Examples +~~~~~~~~ + +.. toctree:: + :glob: + :maxdepth: 1 + + examples/jinja2/* + Inside Templates ---------------- @@ -264,10 +284,10 @@ Several variables are pre-defined inside templates: | repo | The path to the Bcfg2 repository on the filesystem | +-------------+--------------------------------------------------------+ | path | In Genshi templates, ``path`` is a synonym for | -| | ``source_path``. In Cheetah templates, it's a synonym | -| | for ``name``. For this reason, use of ``path`` is | -| | discouraged, and it may be deprecated in a future | -| | release. | +| | ``source_path``. In Cheetah templates and Jinja2 | +| | templates, it's a synonym for ``name``. For this | +| | reason, use of ``path`` is discouraged, and it may be | +| | deprecated in a future release. | +-------------+--------------------------------------------------------+ To access these variables in a Genshi template, you can simply use the @@ -275,6 +295,10 @@ name, e.g.:: Path to this file: ${name} +Similarly, in a Jinja2 template:: + + Path to this file: {{ name }} + In a Cheetah template, the variables are properties of ``self``, e.g.:: @@ -284,15 +308,15 @@ Notes on Using Templates ------------------------ Templates can be host and group specific as well. Deltas will not be -processed for any Genshi or Cheetah base file. +processed for any Genshi, Cheetah, or Jinja2 base file. .. note:: If you are using templating in combination with host-specific or group-specific files, you will need to ensure that the ``.genshi`` - or ``.cheetah`` extension is at the **end** of the filename. Using the - examples from above for *host.example.com* and group *server* you would - have the following:: + ``.cheetah`` or ``.jinja2`` extension is at the **end** of the filename. + Using the examples from above for *host.example.com* and group *server* + you would have the following:: Cfg/etc/fstab/fstab.H_host.example.com.genshi Cfg/etc/fstab/fstab.G50_server.cheetah @@ -337,7 +361,7 @@ An encrypted file should end with ``.crypt``, e.g.:: Cfg/etc/foo.conf/foo.conf.crypt Cfg/etc/foo.conf/foo.conf.G10_foo.crypt -Encrypted Genshi or Cheetah templates can have the extensions in +Encrypted Genshi, Cheetah, and Jinja2 templates can have the extensions in either order, e.g.:: Cfg/etc/foo.conf/foo.conf.crypt.genshi @@ -406,7 +430,7 @@ See :ref:`server-encryption` for more details on encryption in Bcfg2 in general. ``pubkey.xml`` -~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~ ``pubkey.xml`` only ever contains a single line: @@ -554,110 +578,162 @@ Example Hopefully, the performance concerns can be resolved in a future release and these features can be added. +.. _server-plugins-generators-cfg-ssl-certificates: + +SSL Keys and Certificates +========================= + +Cfg can also create SSL keys and certs on the fly, and store the +generated data in the repo so that subsequent requests do not result +in repeated key/cert recreation. In the event that a new key or cert +is needed, the old file can simply be removed from the +repository, and the next time that host checks in, a new file will be +created. If that file happens to be the key, any dependent +certificates will also be regenerated. + +See also :ref:`appendix-guides-sslca_howto` for a detailed example +that uses the SSL key management feature to automate Bcfg2 certificate +authentication. + +Getting started +--------------- + +In order to use the SSL certificate generation feature, you must first +have at least one CA configured on your system. For details on +setting up your own OpenSSL based CA, please see +http://www.openssl.org/docs/apps/ca.html for details of the suggested +directory layout and configuration directives. + +For SSL cert generation to work, the openssl.cnf (or other +configuration file) for that CA must contain full (not relative) +paths. + +#. Add a section to your ``/etc/bcfg2.conf`` called ``sslca_foo``, + replacing foo with the name you wish to give your CA so you can + reference it in certificate definitions. (If you only have one CA, + you can name it ``sslca_default``, and it will be the default CA + for all other operations.) + +#. Under that section, add a ``config`` option that gives the location + of the ``openssl.cnf`` file for your CA. + +#. If necessary, add a ``passphrase`` option containing the passphrase + for the CA's private key. If no passphrase is entry exists, it is + assumed that the private key is stored unencrypted. + +#. Optionally, add a ``chaincert`` option that points to the location + of your ssl chaining certificate. This is used when preexisting + certificate hostfiles are found, so that they can be validated and + only regenerated if they no longer meet the specification. If + you're using a self signing CA this would be the CA cert that you + generated. If the chain cert is a root CA cert (e.g., if it is a + self-signing CA), also add an entry ``root_ca = true``. If + ``chaincert`` is omitted, certificate verification will not be + performed. + +#. Once all this is done, you should have a section in your + ``/etc/bcfg2.conf`` that looks similar to the following:: + + [sslca_default] + config = /etc/pki/CA/openssl.cnf + passphrase = youReallyThinkIdShareThis? + chaincert = /etc/pki/CA/chaincert.crt + root_ca = true + +#. You are now ready to create key and certificate definitions. For + this example we'll assume you've added Path entries for the key, + ``/etc/pki/tls/private/localhost.key``, and the certificate, + ``/etc/pki/tls/certs/localhost.crt`` to a bundle. + +#. Within the ``Cfg/etc/pki/tls/private/localhost.key`` directory, + create a `sslkey.xml`_ file containing the following: + + .. code-block:: xml + + <KeyInfo/> + +#. This will cause the generation of an SSL key when a client requests + that Path. (By default, it will be a 2048-bit RSA key; see + `sslkey.xml`_ for details on how to change the key type and size.) + +#. Similarly, create `sslcert.xml`_ in + ``Cfg/etc/pki/tls/certs/localhost.cfg/``, containing the following: + + .. code-block:: xml + + <CertInfo> + <Cert key="/etc/pki/tls/private/localhost.key" ca="foo"/> + </CertInfo> + +#. When a client requests the cert path, a certificate will be + generated using the key hostfile at the specified key location, + using the CA matching the ``ca`` attribute. ie. ``ca="foo"`` will + match ``[sslca_default]`` in your ``/etc/bcfg2.conf`` + +The :ref:`Bcfg2 bundle example +<server-plugins-structures-bundler-bcfg2-server>` contains entries to +automate the process of setting up a CA. + Configuration ------------- -In addition to ``privkey.xml`` and ``authorized_keys.xml``, described -above, the behavior of the SSH key generation feature can be -influenced by several options in the ``[sshkeys]`` section of -``bcfg2.conf``: +``bcfg2.conf`` +~~~~~~~~~~~~~~ -+----------------+---------------------------------------------------------+-----------------------+------------+ -| Option | Description | Values | Default | -+================+=========================================================+=======================+============+ -| ``passphrase`` | Use the named passphrase to encrypt private keys on the | String | None | -| | filesystem. The passphrase must be defined in the | | | -| | ``[encryption]`` section. See :ref:`server-encryption` | | | -| | for more details on encryption in Bcfg2 in general. | | | -+----------------+---------------------------------------------------------+-----------------------+------------+ -| ``category`` | Generate keys specific to groups in the given category. | String | None | -| | It is best to pick a category that all clients have a | | | -| | group from. | | | -+----------------+---------------------------------------------------------+-----------------------+------------+ - -Deltas -====== +In ``bcfg2.conf``, you must declare your CA(s) in ``[sslca_<name>]`` +sections. At least one is required. Valid options are detailed +below, in `Cfg Configuration`_. -.. note:: +Only the ``config`` option is required; i.e., the simplest possible CA +section is:: + + [sslca_default] + config = /etc/pki/CA/openssl.cnf + +``sslcert.xml`` +~~~~~~~~~~~~~~~ + +.. xml:schema:: sslca-cert.xsd + :linktotype: + :inlinetypes: CertType + +Example +^^^^^^^ + +.. code-block:: xml + + <CertInfo> + <subjectAltName>test.example.com</subjectAltName> + <Group name="apache"> + <Cert key="/etc/pki/tls/private/foo.key" days="730"/> + </Group> + <Group name="nginx"> + <Cert key="/etc/pki/tls/private/foo.key" days="730" + append_chain="true"/> + </Group> + </CertInfo> + +``sslkey.xml`` +~~~~~~~~~~~~~~ + +.. xml:schema:: sslca-key.xsd + :linktotype: + :inlinetypes: KeyType + +Example +^^^^^^^ + +.. code-block:: xml + + <KeyInfo> + <Group name="fast"> + <Key type="rsa" bits="1024"/> + </Group> + <Group name="secure"> + <Key type="rsa" bits="4096"/> + </Group> + </KeyInfo> - In Bcfg2 1.3 and newer, deltas are deprecated. It is recommended - that you use templates instead. The - :ref:`TemplateHelper plugin - <server-plugins-connectors-templatehelper>` comes with an example - helper that can be used to include other files easily, a subset of - cat file functionality. ``bcfg2-lint`` checks for deltas and - warns about them. - -.. warning:: - - In Bcfg2 1.3, deltas **do not** work with `SSH key or - authorized_keys generation <server-plugins-generators-cfg-sshkeys>`_. - -Bcfg2 has finer grained control over how to deliver configuration -files to a host. Let's say we have a Group named file-server. Members -of this group need the exact same ``/etc/motd`` as all other hosts except -they need one line added. We could copy motd to ``motd.G01_file-server``, -add the one line to the Group specific version and be done with it, -but we're duplicating data in both files. What happens if we need to -update the motd? We'll need to remember to update both files then. Here's -where deltas come in. A delta is a small change to the base file. There -are two types of deltas: cats and diffs. The cat delta simply adds or -removes lines from the base file. The diff delta is more powerful since -it can take a unified diff and apply it to the base configuration file -to create the specialized file. Diff deltas should be used very sparingly. - -Cat Files ---------- - -Continuing our example for cat files, we would first create a file named -``motd.G01_file-server.cat``. The .cat suffix designates that the file is -a diff. We would then edit that file and add the following line:: - - +This is a file server - -The **+** at the begining of the file tells Bcfg2 that the line should be -appended to end of the file. You can also start a line with **-** to tell -Bcfg2 to remove that exact line wherever it might be in the file. How do -we know what base file Bcfg2 will choose to use to apply a delta? The -same rules apply as before: Bcfg2 will choose the highest priority, -most specific file as the base and then apply deltas in the order of -most specific and then increasing in priority. What does this mean in -real life. Let's say our machine is a web server, mail server, and file -server and we have the following configuration files:: - - motd - motd.G01_web-server - motd.G01_mail-server.cat - motd.G02_file-server.cat - motd.H_bar.example.com - motd.H_foo.example.com.cat - -If our machine isn't *foo.example.com* or *bar.example.com*, but -is a web server, then Bcfg2 would choose ``motd.G01_web-server`` as -the base file. It is the most specific base file for this host. Bcfg2 -would apply the ``motd.G01_mail-server.cat`` delta to the -``motd.G01_web-server`` base file. It is the least specific -delta. Bcfg2 would then apply the ``motd.G02_file-server.cat`` delta -to the result of the delta before it. - -If our machine is *foo.example.com* and a web server, then Bcfg2 would -choose ``motd.G01_web-server`` as the base file. It is the most -specific base file for this host. Bcfg2 would apply the -``motd.H_foo.example.com.cat`` delta to the ``motd.G01_web-server`` -base file. The reason the other deltas aren't applied to -*foo.example.com* is because a **.H_** delta is more specific than a -**.G##_** delta. Bcfg2 applies all the deltas at the most specific -level. - -If our machine is *bar.example.com*, then Bcfg2 would chose -``motd.H_foo.example.com`` as the base file because it is the most -specific base file for this host. Regardless of the groups -*bar.example.com* is a member of, **no cat files** would be applied, -because only cat files as specific or more specific than the base file -are applied. (In other words, if a group-specific base file is -selected, only group- or host-specific cat files can be applied; if a -host-specific base file is selected, only host-specific cat files can -be applied.) .. _server-plugins-generators-cfg-validation: @@ -712,3 +788,56 @@ File permissions File permissions for entries handled by Cfg are controlled via the use of :ref:`server-info` files. Note that you **cannot** use both a Permissions entry and a Path entry to handle the same file. + +.. _server-plugins-generators-cfg-configuration: + +Cfg Configuration +================= + +The behavior of many bits of the Cfg plugin can be configured in +``bcfg2.conf`` with the following options. + +In addition to ``privkey.xml`` and ``authorized_keys.xml``, described +above, the behavior of the SSH key generation feature can be +influenced by several options in the ``[sshkeys]`` section of +``bcfg2.conf``: + ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| Section | Option | Description | Values | Default | ++=============+================+=========================================================+=======================+============+ +| ``cfg`` | ``passphrase`` | Use the named passphrase to encrypt created data on the | String | None | +| | | filesystem. (E.g., SSH and SSL keys.) The passphrase | | | +| | | must be defined in the ``[encryption]`` section. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``cfg`` | ``category`` | Generate data (e.g., SSH keys, SSL keys and certs) | String | None | +| | | specific to groups in the given category. It is best to | | | +| | | pick a category that all clients have a group from. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``cfg`` | ``validation`` | Whether or not to perform `Content Validation`_ | Boolean | True | +| | | specific to groups in the given category. It is best to | | | +| | | pick a category that all clients have a group from. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sshkeys`` | ``passphrase`` | Override the global Cfg passphrase with a specific | String | None | +| | | passphrase for encrypting created SSH private keys. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sshkeys`` | ``category`` | Override the global Cfg category with a specific | String | None | +| | | category for created SSH keys. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca`` | ``passphrase`` | Override the global Cfg passphrase with a specific | String | None | +| | | passphrase for encrypting created SSL keys. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca`` | ``category`` | Override the global Cfg category with a specific | String | None | +| | | category for created SSL keys and certs. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca_*`` | ``config`` | Path to the openssl config for the CA | String | None | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca_*`` | ``passphrase`` | Passphrase for the CA private key | String | None | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca_*`` | ``chaincert`` | Path to the SSL chaining certificate for verification | String | None | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca_*`` | ``root_ca`` | Whether or not ``<chaincert>`` is a root CA (as | Boolean | False | +| | | opposed to an intermediate cert) | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ + +See :ref:`server-encryption` for more details on encryption in Bcfg2 +in general. diff --git a/doc/server/plugins/generators/decisions.txt b/doc/server/plugins/generators/decisions.txt index 9a40ab8fd..f0afeba0a 100644 --- a/doc/server/plugins/generators/decisions.txt +++ b/doc/server/plugins/generators/decisions.txt @@ -29,18 +29,23 @@ client's whitelists or blacklists. is not used. See `Decision Mode`_ below. The Decisions plugin uses a directory in the Bcfg2 repository called -Decisions. Files in the Decisions subdirectory are named similarly to -files managed by Cfg and Probes, so you can use host- and -group-specific files and the like after their basename. File basenames -are either ``whitelist`` or ``blacklist``. These files have a simple -format; the following is an example. +Decisions, which may contain two files: ``whitelist.xml`` and +``blacklist.xml``. These files have a simple format: + +.. xml:type:: DecisionsType + :linktotype: + :noautodep: py:genshiElements + +For example: .. code-block:: xml - $ cat Decisions/whitelist + $ cat Decisions/whitelist.xml <Decisions> <Decision type='Service' name='*'/> - <Decision type='Path' name='/etc/apt/apt.conf'/> + <Group name="debian"> + <Decision type='Path' name='/etc/apt/apt.conf'/> + </Group> </Decisions> This example, included as a whitelist due to its name, enables all services, @@ -60,12 +65,6 @@ list. This list is sent to the client. control these via their respective options (``-I`` or ``-n``, for example). -To add syntax highlighting to Decisions files in vim and emacs, you -can add comments such as this:: - - <Decisions><!--*- mode: xml; -*--> - <!-- vim: set ft=xml : --> - Decision Mode ============= diff --git a/doc/server/plugins/generators/examples/genshi/ganglia.txt b/doc/server/plugins/generators/examples/genshi/ganglia.txt index 3a20fde92..d7030e990 100644 --- a/doc/server/plugins/generators/examples/genshi/ganglia.txt +++ b/doc/server/plugins/generators/examples/genshi/ganglia.txt @@ -33,7 +33,7 @@ Bundler/ganglia.xml .. code-block:: xml - <Bundle name='ganglia'> + <Bundle> <Package name='ganglia-gmond' /> <Package name='ganglia-gmond-modules-python' /> <Path name='/etc/ganglia/gmond.conf' /> diff --git a/doc/server/plugins/generators/examples/jinja2/simple.txt b/doc/server/plugins/generators/examples/jinja2/simple.txt new file mode 100644 index 000000000..b4ab844fb --- /dev/null +++ b/doc/server/plugins/generators/examples/jinja2/simple.txt @@ -0,0 +1,53 @@ +.. -*- mode: rst -*- + +========================= + Basic Jinja2 Templates +========================= + +This simple example demonstrates basic usage of Jinja2 templates. + +``/var/lib/bcfg2/Cfg/foo/foo.jinja2`` + +.. code-block:: none + + Hostname is {{ metadata.hostname }} + Filename is {{ name }} + Template is {{ source_path }} + Groups: + {% for group in metadata.groups -%} + * {{ group }} + {% endfor %} + Categories: + {% for category in metadata.categories -%} + * {{ category }} -- {{ metadata.categories[category] }} + {% endfor %} + + Probes: + {% for probe in metadata.Probes -%} + * {{ probe }} -- {{ metadata.Probes[probe] }} + {% endfor %} + +Output +====== + +.. code-block:: xml + + <Path type="file" name="/foo" owner="root" mode="0644" group="root"> + Hostname is topaz.mcs.anl.gov + Filename is /foo + Template is /var/lib/bcfg2/Cfg/foo/foo.jinja2 + Groups: + * desktop + * mcs-base + * ypbound + * workstation + * xserver + * debian-sarge + * debian + * a + Categories: + * test -- a + + Probes: + * os -- debian + </Path> diff --git a/doc/server/plugins/generators/hostbase.txt b/doc/server/plugins/generators/hostbase.txt deleted file mode 100644 index c6007f70e..000000000 --- a/doc/server/plugins/generators/hostbase.txt +++ /dev/null @@ -1,228 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-plugins-generators-hostbase: - -======== -Hostbase -======== - -IP management system built on top of Bcfg2. It has four main parts: a -django data model, a web frontend, command-line utilities, and a Bcfg2 -plugin that generates dhcp, dns, and yp configuration files. - -Installation -============ - -Installation of Hostbase requires installation of a python module, -configuration of database (mysql or postgres), and configuration of an -Apache webserver with mod_python. Hostbase was developed using MySQL, -so this document is aimed at MySQL users. - -Prerequisites -------------- - -* `mysql`_ -* `python-mysqldb`_ -* `Django`_ - -.. _Django: http://www.djangoproject.com -.. _python-mysqldb: http://mysql-python.sourceforge.net/MySQLdb.html -.. _mysql: http://www.mysql.com/ - -Configure the database ----------------------- - -Create the hostbase database and a user. For MySQL users:: - - mysql> CREATE DATABASE hostbase - mysql> quit - - systemprompt#: mysql -u root hostbase - mysql> GRANT ALL PRIVILEGES ON *.* TO hostbaseuser@mycomputer.private.net IDENTIFIED - BY 'password' WITH GRANT OPTION; - mysql> quit - -As of Bcfg2 v0.8.7 configuration options for Hostbase have moved to -``/etc/bcfg2.conf``. There is an example bcfg2.conf with Hostbase -options located at ``bcfg2-tarball/examples/bcfg2.confHostbase``. -Edit the hostbase options to correspond to the database you've -initialized and copy the configuration to ``/etc/bcfg2.conf``. To -finish creating the database, from your ``path to -python/Bcfg2/Server/Hostbase`` directory, run ``python manage.py -syncdb`` to do all table creation. - -Configure the web interface ---------------------------- - -Now it's possible to explore the Hostbase web interface. For -curiosity, you can run Django's built-in development server to take a -peek. Do this by running ``python manage.py runserver -[servername:port]`` from your Hostbase directory. Django will -default to ``localhost:8000`` if no server or port is entered. Now -you can explore the web interface. Try adding a host and a zone. -You'll see that a ".rev" zone already exists. This is where -information for reverse files will go. - -For production, you'll want to have this configured for Apache with -mod_python. Here is an example of how to configure Hostbase as a -virtual host. - -.. code-block:: html - - <VirtualHost hostbase.mcs.anl.gov:80> - ServerAdmin systems@mcs.anl.gov - - DocumentRoot /var/www/hostbase/ - <Directory /> - AllowOverride None - </Directory> - - # Possible values include: debug, info, notice, warn, error, crit, - # alert, emerg. - LogLevel warn - - ServerSignature Off - - # Stop TRACE/TRACK vulnerability - <IfModule mod_rewrite.c> - RewriteEngine on - RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK) - RewriteRule .* - [F] - </IfModule> - - Redirect / https://hostbase.mcs.anl.gov/ - </VirtualHost> - - <VirtualHost hostbase.mcs.anl.gov:443> - ServerAdmin systems@mcs.anl.gov - - DocumentRoot /var/www/hostbase/ - <Directory /> - AllowOverride None - </Directory> - - # Possible values include: debug, info, notice, warn, error, crit, - # alert, emerg. - LogLevel warn - - ServerSignature Off - - # Stop TRACE/TRACK vulnerability - <IfModule mod_rewrite.c> - RewriteEngine on - RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK) - RewriteRule .* - [F] - </IfModule> - - SSLEngine On - SSLCertificateFile /etc/apache2/ssl/hostbase_server.crt - SSLCertificateKeyfile /etc/apache2/ssl/hostbase_server.key - - <Location "/"> - SetHandler python-program - PythonHandler django.core.handlers.modpython - SetEnv DJANGO_SETTINGS_MODULE Bcfg2.Server.Hostbase.settings - PythonDebug On - </Location> - <Location "/site_media/"> - SetHandler None - </Location> - </VirtualHost> - - -You'll need to copy the contents of ``Hostbase/media`` into -``/var/www/hostbase/site_media`` in this configuration to serve the -correct css files. - -Enable the Hostbase plugin --------------------------- - -Now that the database is accessible and there is some data in it, you can -enable the Hostbase plugin on your Bcfg2 server to start generating some -configuration files. All that needs to be done is to add ``Hostbase`` -to the end of the list of generators in your bcfg2.conf file. To see -what's being generated by Hostbase, fire up a Bcfg2 development server: -``bcfg2-info``. For more information on how to use the Bcfg2 development -server, type help at the prompt. For our purposes, type ``debug``. -This will bring you to an interactive python prompt where you can access -bcfg's core data. - -.. code-block:: python - - for each in bcore.plugins['Hostbase'].filedata: - print each - - -The above loop will print out the name of each file that was generated -by Hostbase. You can see the contents of any of these by typing ``print -bcore.plugins['Hostbase'].filedata[filename]``. - -Create a bundle ---------------- - -Bcfg2 needs a way to distribute the files generated by Hostbase. -We'll do this with a bundle. In bcfg's ``Bundler`` directory, touch -``hostbase.xml``. - -.. code-block:: xml - - <Bundle name='hostbase' version='0.1'> - <Package name='dhcp3-server'/> - <Package name='bind9'/> - <Service name='dhcp3-server'/> - <Service name='bind9'/> - <Path name='/etc/dhcp3/dhcpd.conf'/> - <Path name='/etc/bind/[your domain]'/> - <Path name='/etc/bind/xxx.xxx.xxx.rev'/> - </Bundle> - -The above example is a bundle that will deliver both dhcp and dns files. -This can be trivially split into separate bundles. It is planned that -Hostbase will eventually be able to generate the list of ``Paths`` -in its bundles automatically. - -Do a Hostbase push ------------------- - -You'll want to be able to trigger the Hostbase plugin to rebuild -it's config files and push them out when data has been modified -in the database. This can be done through and XMLRPC function -available from the Bcfg2 server. From a client that is configured -to receive one or more hostbase bundles, you'll need to first -edit your ``python/site-packages/Bcfg2/Client/Proxy.py`` file. -Add ``'Hostbase.rebuildState'`` to the list of methods in the Bcfg2 -client proxy object. The modified list is shown below: - -.. code-block:: python - - class bcfg2(ComponentProxy): - '''bcfg2 client code''' - name = 'bcfg2' - methods = ['AssertProfile', 'GetConfig', 'GetProbes', 'RecvProbeData', 'RecvStats', 'Hostbase.rebuildState'] - -Now copy the file ``hostbasepush.py`` from ``bcfg2/tools`` in the Bcfg2 -source to your machine. When this command is run as root, it triggers -the Hostbase to rebuild it's files, then runs the Bcfg2 client on your -local machine to grab the new configs. - -NIS Authentication -================== - -Django allows for custom authentication backends to its login procedure. -Hostbase has an NIS authentication backend that verifies a user to be -in the unix group allowed to modify Hostbase. - -To enable this feature: - -* first edit your ``Hostbase/settings.py`` file and uncomment - the line **Hostbase.backends.NISBackend** in the list of - *AUTHENTICATION_BACKENDS* -* enter the name of the unix group you want to give access to Hostbase - in the *AUTHORIZED_GROUP* variable -* in your ``Hostbase/hostbase/views.py`` file at the very bottom, - uncomment the block(s) of lines that give you the desired level - of access - -Hostbase will now direct the user to a login page if he or she is not -authorized to view a certain page. Users should log in with their -regular Unix username and password. diff --git a/doc/server/plugins/generators/nagiosgen.txt b/doc/server/plugins/generators/nagiosgen.txt index 0ae922fa3..1ccdd66c1 100644 --- a/doc/server/plugins/generators/nagiosgen.txt +++ b/doc/server/plugins/generators/nagiosgen.txt @@ -12,7 +12,7 @@ This page describes the installation and use of the `NagiosGen`_ plugin. Update ``/etc/bcfg2.conf``, adding NagiosGen to plugins:: - plugins = Base,Bundler,Cfg,...,NagiosGen + plugins = Bundler,Cfg,...,NagiosGen Create the NagiosGen directory:: @@ -124,21 +124,21 @@ Create a nagios Bcfg2 bundle ``/var/lib/bcfg2/Bundler/nagios.xml`` .. code-block:: xml - <Bundle name='nagios' version='2.0'> + <Bundle> <Path name='/etc/nagiosgen.status'/> - <Group name='rh'> + <Group name='redhat'> <Group name='nagios-server'> - <Path name='/etc/nagios/nagiosgen.cfg'/> + <Path name='/etc/nagios/conf.d/bcfg2.cfg'/> <Package name='libtool-libs'/> <Package name='nagios'/> <Package name='nagios-www'/> <Service name='nagios'/> </Group> </Group> - <Group name='debian-lenny'> + <Group name='debian-wheezy'> <Group name='nagios-server'> - <Path name='/etc/nagios3/nagiosgen.cfg' - altsrc='/etc/nagios/nagiosgen.cfg'/> + <Path name='/etc/nagios3/conf.d/bcfg2.cfg' + altsrc='/etc/nagios/conf.d/bcfg2.cfg'/> <Package name='nagios3'/> <Package name='nagios3-common'/> <Package name='nagios3-doc'/> @@ -161,10 +161,6 @@ Assign clients to nagios groups in <Bundle name='nagios'/> </Group> -Update nagios configuration file to use ``nagiosgen.cfg``:: - - cfg_file=/etc/nagios/nagiosgen.cfg - Note that some of these files are built on demand, each time a client in group "nagios-server" checks in with the Bcfg2 server. Local nagios instances can be configured to use the NagiosGen directory in the Bcfg2 diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index 31f3ccf22..2fe71f895 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -18,14 +18,10 @@ through those channels. Limiting sources to groups ========================== -`sources.xml`_ processes ``<Group>`` and ``<Client>`` tags just like -Bundles. In addition to any groups or clients specified that way, -clients must be a member of the appropriate architecture group as -specified in a Source stanza. In total, in order for a source to be -associated with a client, the client must be in any explicit groups or -clients specified in `sources.xml`_, and any specified architecture -groups. If `"Magic Groups"`_ are enabled, then the client must be a -member of a matching magic group as well. +``Packages/sources.xml`` processes ``<Group>`` and ``<Client>`` tags +just like Bundles. In addition to any groups or clients specified that +way, clients must be a member of the appropriate architecture group as +specified in a Source stanza. Memberships in architecture groups is needed so that Packages can map software sources to clients. There is no other way to handle this than @@ -36,62 +32,6 @@ source to which they apply (based on group memberships, as described above). Packages and dependencies are resolved from all applicable sources. -.. note:: - - To recap, a client needs to be a member of the **Architecture** - group and any other groups defined in your - `sources.xml`_ file in order for the client to be - associated to the proper sources. If you are using - :ref:`server-plugins-generators-packages-magic-groups`, then a - client must also be a member of the appropriate OS group. - -.. _server-plugins-generators-packages-magic-groups: - -"Magic Groups" -============== - -.. deprecated:: 1.3.0 - -Packages has the ability to use a feature known as "magic groups"; it -is the only plugin to use that feature. Most plugins operate based on -client group memberships, without any concern for the particular names -chosen for groups by the user. The Packages plugin is the sole -exception to this rule. Packages needs to "know" two different sorts -of facts about clients. The first is the basic OS/distro of the -client, enabling classes of sources. The second is the architecture of -the client, enabling sources for a given architecture. In addition to -these magic groups, each source may also specify non-magic groups to -limit the source's applicability to group member clients. - -+--------+----------+--------------+ -| Source | OS Group | Architecture | -+========+==========+==============+ -| Apt | debian | i386 | -+--------+----------+--------------+ -| Apt | ubuntu | amd64 | -+--------+----------+--------------+ -| Apt | nexenta | | -+--------+----------+--------------+ -| Apt | apt | | -+--------+----------+--------------+ -| Yum | redhat | i386 | -+--------+----------+--------------+ -| Yum | centos | x86_64 | -+--------+----------+--------------+ -| Yum | fedora | | -+--------+----------+--------------+ -| Yum | yum | | -+--------+----------+--------------+ - -Magic OS groups are disabled by default in Bcfg2 1.3 and greater. If -you require magic groups, you can enable them by setting -``magic_groups`` to ``1`` in the ``[packages]`` section of -``bcfg2.conf``. - -Magic groups will be removed in a future release. - -Magic architecture groups cannot be disabled. - Setup ===== @@ -102,14 +42,13 @@ Three basic steps are required for Packages to work properly. software repositories should be used, and which clients are eligible to use each one. #. Ensure that clients are members of the proper groups. Each client - should be a member of all of the groups listed in the `sources.xml` - (like ubuntu-intrepid or centos-5.2 in the following examples), one - of the architecture groups listed in the source configuration - (i386, amd64 or x86_64 in the following examples), and one of the - magic groups listed above, if magic groups are enabled. '''Failure - to do this will result in the source either not applying to the - client, or only architecture independent packages being made - available to the client.''' + should be a member of all of the groups listed in the + ``sources.xml`` (like ubuntu-intrepid or centos-5.2 in the + following examples), and one of the architecture groups listed in + the source configuration (i386, amd64 or x86_64 in the following + examples). '''Failure to do this will result in the source either + not applying to the client, or only architecture independent + packages being made available to the client.''' #. Add Package entries to bundles. #. Sit back and relax, as dependencies are resolved, and automatically added to client configurations. @@ -122,6 +61,7 @@ Packages plugin. It processes ``<Group>`` and ``<Client>`` tags just like Bundles. The primary element in ``sources.xml`` is the Source tag: .. xml:element:: Source + :noautodep: py:genshiElements Handling GPG Keys ----------------- @@ -198,9 +138,7 @@ processed. After this phase, but before entry binding, a list of packages and the client metadata instance is passed into Packages' resolver. This process determines a superset of packages that will fully satisfy dependencies of all package entries included in structures, and reports -any prerequisites that cannot be satisfied. This facility should largely -remove the need to use the :ref:`Base <server-plugins-structures-base>` -plugin. +any prerequisites that cannot be satisfied. Disabling dependency resolution ------------------------------- @@ -279,10 +217,6 @@ something like this: <Source type="apt" recommended="true" ...> - .. warning:: You must regenerate the Packages cache when adding or - removing the recommended attribute (``bcfg2-admin xcmd - Packages.Refresh``). - .. [#f1] Bcfg2 will by default add **Essential** packages to the client specification. You can disable this behavior by setting the :xml:attribute:`SourceType:essential` @@ -409,9 +343,85 @@ This is done automatically any time `sources.xml`_ is updated. Availability ============ -Support for clients using yum and apt is currently available. Support for +Support for the following clients is currently available. Support for other package managers (Portage, Zypper, IPS, etc) remain to be added. +apt +--- + +All dpkg based clients (for example Debian, Ubuntu or Nexenta) could be +handled with the apt module: + +.. code-block:: xml + + <Source type="apt" + url="http://us.archive.ubuntu.com/ubuntu" + version="intrepid"> + <Component>main</Component> + <Component>universe</Component> + <Arch>i386</Arch> + <Arch>amd64</Arch> + </Source> + + +pac +--- + +For Arch Linux or Parabola GNU/Linux-libre you could use the pac module +for packages. You do not need to supply a version attribute as the mirrors +are rolling release and does not supply different versions. + +.. code-block:: xml + + <Source type="pac" + url="http://mirrors.kernel.org/archlinux/"> + <Component>core</Component> + <Component>extra</Component> + <Component>community</Component> + <Arch>i686</Arch> + <Arch>x86_64</Arch> + </Source> + + +pkgng +----- + +The support for the Next Generation package management tool for FreeBSD +is called pkgng. It downloads the packagesite file from the mirror +and parses the dependencies out of it. It currently does not use the +DNS SRV record lookup mechanism to get the correct mirror and does +not verify the signature inside the packagesite file. + +.. code-block:: xml + + <Source type="pkgng" + url="http://pkg.freebsd.org/" + version="10"> + <Component>latest</Component> + <Arch>x86:64</Arch> + <Arch>x86:32</Arch> + </Source> + + +yum +--- + +Rpm based clients (for example RedHat, CentOS or Fedora) could be handled +with the yum module: + +.. code-block:: xml + + <Source type="yum" + url="http://mirror.centos.org/centos/" + version="5.2"> + <Component>os</Component> + <Component>updates</Component> + <Component>extras</Component> + <Arch>i386</Arch> + <Arch>x86_64</Arch> + </Source> + + Package Checking and Verification ================================= @@ -451,7 +461,7 @@ attribute, e.g.: .. code-block:: xml - <Bundle name="yum"> + <Bundle> <Group name="sles"> <Path name="/etc/yum/yum.repos.d/bcfg2.repo" altsrc="/etc/yum.repos.d/bcfg2.repo"/> @@ -707,6 +717,9 @@ It understands the following directives: +-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ | Name | Description | Values | Default | +=============+======================================================+==========+===================================================================+ +| backends | List of backends that should be loaded for the | List | Yum,Apt,Pac,Pkgng | +| | dependency resolution. | | | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ | resolver | Enable dependency resolution | Boolean | True | +-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ | metadata | Enable metadata processing. Disabling ``metadata`` | Boolean | True | diff --git a/doc/server/plugins/generators/pkgmgr.txt b/doc/server/plugins/generators/pkgmgr.txt index ace7c16ef..8d9979ba0 100644 --- a/doc/server/plugins/generators/pkgmgr.txt +++ b/doc/server/plugins/generators/pkgmgr.txt @@ -10,10 +10,10 @@ The Pkgmgr plugin resolves the Abstract Configuration Entity "Package" to a package specification that the client can use to detect, verify and install the specified package. -For a package specification to be included in the Literal configuration -the name attribute from an Abstract Package Tag (from Base or Bundler) -must match the name attribute of a Package tag in Pkgmgr, along with -the appropriate group associations of course. +For a package specification to be included in the Literal +configuration the name attribute from an abstract Package tag (from +Bundler) must match the name attribute of a Package tag in Pkgmgr, +along with the appropriate group associations of course. Each file in the Pkgmgr directory has a priority. This allows the same package to be served by multiple files. The priorities can be diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index 77ce63e51..86478a5ae 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -20,32 +20,14 @@ The Rules plugin resolves the following Abstract Configuration Entities: to literal configuration entries suitable for the client drivers to consume. -For an entity specification to be included in the Literal configuration -the name attribute from an Abstract Entity Tag (from Base or Bundler) -must match the name attribute of an Entity tag in Rules, along with the -appropriate group associations of course. +For an entity specification to be included in the Literal +configuration the name attribute from an abstract entity tag (from +Bundler) must match the name attribute of an entity tag in Rules, +along with the appropriate group associations of course. Each file in the Rules directory has a priority. This allows the same Entities to be served by multiple files. The priorities can be used to -break ties in the case that multiple files serve data for the same Entity. - - -Usage of Groups in Rules -======================== - -Groups are used by the Rules plugin, along with host metadata, for -selecting the Configuration Entity entries to include in the clients -literal configuration. They can be thought of as:: - - if client is a member of group1 then - assign to literal config - -Nested groups are conjunctive (logical and).:: - - if client is a member of group1 and group2 then - assign to literal config - -Group membership may be negated. +break ties in the case that multiple files serve data for the same entity. Tag Attributes in Rules ======================= @@ -515,8 +497,8 @@ If you wish, you can configure the Rules plugin to support regular expressions. This entails a small performance and memory usage penalty. To do so, add the following setting to ``bcfg2.conf``:: - [rules] - regex = yes + [rules] + regex = yes With regular expressions enabled, you can use a regex in the ``name`` attribute to match multiple abstract configuration entries. diff --git a/doc/server/plugins/generators/semodules.txt b/doc/server/plugins/generators/semodules.txt index 04d72e139..d75160cdf 100644 --- a/doc/server/plugins/generators/semodules.txt +++ b/doc/server/plugins/generators/semodules.txt @@ -41,7 +41,7 @@ SEModules handles ``<SEModule>`` entries. For instance: .. code-block:: xml - <Bundle name="foo"> + <Bundle> <SEModule name="foo.pp"/> </Bundle> @@ -50,7 +50,7 @@ The ``.pp`` extension is optional. .. note:: If you use a ``BoundSEModule`` tag, you must *not* include the - ``.pp`` extension. This is not recommend, though. + ``.pp`` extension. This is not recommended, though. You can also install a disabled module: diff --git a/doc/server/plugins/generators/sshbase.txt b/doc/server/plugins/generators/sshbase.txt index 38631dd3b..540cc1e06 100644 --- a/doc/server/plugins/generators/sshbase.txt +++ b/doc/server/plugins/generators/sshbase.txt @@ -162,6 +162,20 @@ in order to permit :ref:`pulling with bcfg2-admin <server-admin-pull>`. You should almost certainly set ``sensitive`` to "true" in ``info.xml``. + +.. _server-plugins-generators-sshbase-encryption: + +Encryption +========== + +SSHbase can optionally encrypt the private keys that it generates. To +enable this feature, set the ``passphrase`` option in the +``[sshbase]`` section of ``bcfg2.conf`` to the name of the passphrase +that should be used to encrypt all SSH keys. (The passphrases are +enumerated in the ``[encryption]`` section.) See +:ref:`server-encryption` for more details on Bcfg2 encryption in +general. + Blog post ========= diff --git a/doc/server/plugins/generators/sslca.txt b/doc/server/plugins/generators/sslca.txt deleted file mode 100644 index 73527284c..000000000 --- a/doc/server/plugins/generators/sslca.txt +++ /dev/null @@ -1,361 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-plugins-generators-sslca: - -===== -SSLCA -===== - -SSLCA is a generator plugin designed to handle creation of SSL private -keys and certificates on request. - -Borrowing ideas from :ref:`server-plugins-generators-cfg-genshi` and -the :ref:`server-plugins-generators-sshbase` plugin, SSLCA automates -the generation of SSL certificates by allowing you to specify key and -certificate definitions. Then, when a client requests a Path that -contains such a definition within the SSLCA repository, the matching -key/cert is generated, and stored in a hostfile in the repo so that -subsequent requests do not result in repeated key/cert recreation. In -the event that a new key or cert is needed, the offending hostfile can -simply be removed from the repository, and the next time that host -checks in, a new file will be created. If that file happens to be the -key, any dependent certificates will also be regenerated. - -.. _getting-started: - -Getting started -=============== - -In order to use SSLCA, you must first have at least one CA configured -on your system. For details on setting up your own OpenSSL based CA, -please see http://www.openssl.org/docs/apps/ca.html for details of the -suggested directory layout and configuration directives. - -For SSLCA to work, the openssl.cnf (or other configuration file) for -that CA must contain full (not relative) paths. - -#. Add SSLCA to the **plugins** line in ``/etc/bcfg2.conf`` and - restart the server -- This enables the SSLCA plugin on the Bcfg2 - server. - -#. Add a section to your ``/etc/bcfg2.conf`` called ``sslca_foo``, - replacing foo with the name you wish to give your CA so you can - reference it in certificate definitions. - -#. Under that section, add an entry for ``config`` that gives the - location of the openssl configuration file for your CA. - -#. If necessary, add an entry for ``passphrase`` containing the - passphrase for the CA's private key. We store this in - ``/etc/bcfg2.conf`` as the permissions on that file should have it - only readable by the bcfg2 user. If no passphrase entry exists, - it is assumed that the private key is stored unencrypted. - -#. Optionally, Add an entry ``chaincert`` that points to the location - of your ssl chaining certificate. This is used when preexisting - certificate hostfiles are found, so that they can be validated and - only regenerated if they no longer meet the specification. If - you're using a self signing CA this would be the CA cert that you - generated. If the chain cert is a root CA cert (e.g., if it is a - self-signing CA), also add an entry ``root_ca = true``. If - ``chaincert`` is omitted, certificate verification will not be - performed. - -#. Once all this is done, you should have a section in your - ``/etc/bcfg2.conf`` that looks similar to the following:: - - [sslca_default] - config = /etc/pki/CA/openssl.cnf - passphrase = youReallyThinkIdShareThis? - chaincert = /etc/pki/CA/chaincert.crt - root_ca = true - -#. You are now ready to create key and certificate definitions. For - this example we'll assume you've added Path entries for the key, - ``/etc/pki/tls/private/localhost.key``, and the certificate, - ``/etc/pki/tls/certs/localhost.crt`` to a bundle or base. - -#. Defining a key or certificate is similar to defining a Cfg file. - Under your Bcfg2's ``SSLCA/`` directory, create the directory - structure to match the path to your key. In this case this would be - something like - ``/var/lib/bcfg2/SSLCA/etc/pki/tls/private/localhost.key``. - -#. Within that directory, create a `key.xml`_ file containing the - following: - - .. code-block:: xml - - <KeyInfo> - <Key type="rsa" bits="2048" /> - </KeyInfo> - -#. This will cause the generation of an 2048 bit RSA key when a client - requests that Path. Alternatively you can specify ``dsa`` as the - keytype, or a different number of bits. - -#. Similarly, create the matching directory structure for the - certificate path, and a `cert.xml`_ containing the following: - - .. code-block:: xml - - <CertInfo> - <Cert format="pem" key="/etc/pki/tls/private/localhost.key" - ca="default" days="365" c="US" l="New York" st="New York" - o="Your Company Name" /> - </CertInfo> - -#. When a client requests the cert path, a certificate will be - generated using the key hostfile at the specified key location, - using the CA matching the ca attribute. ie. ca="default" will match - [sslca_default] in your ``/etc/bcfg2.conf`` - -.. _sslca-configuration: - -Configuration -============= - -bcfg2.conf ----------- - -``bcfg2.conf`` contains miscellaneous configuration options for the SSLCA -plugin. These are described in some detail above in `getting-started`_, -but are also enumerated here as a reference. Any booleans in the config -file accept the values "1", "yes", "true", and "on" for True, and "0", -"no", "false", and "off" for False. - -Each directive below should appear at most once in each -``[sslca_<name>]`` section. The following directives are understood: - -+--------------+------------------------------------------+---------+---------+ -| Name | Description | Values | Default | -+==============+==========================================+=========+=========+ -| config | Path to the openssl config for the CA | String | None | -+--------------+------------------------------------------+---------+---------+ -| passphrase | Passphrase for the CA private key | String | None | -+--------------+------------------------------------------+---------+---------+ -| chaincert | Path to the SSL chaining certificate for | String | None | -| | verification | | | -+--------------+------------------------------------------+---------+---------+ -| root_ca | Whether or not ``<chaincert>`` is a root | Boolean | false | -| | CA (as opposed to an intermediate cert) | | | -+--------------+------------------------------------------+---------+---------+ - -Only ``config`` is required. - -cert.xml --------- - -.. xml:schema:: sslca-cert.xsd - :linktotype: - :inlinetypes: CertType - -Example -^^^^^^^ - -.. code-block:: xml - - <CertInfo> - <subjectAltName>test.example.com</subjectAltName> - <Group name="apache"> - <Cert key="/etc/pki/tls/private/foo.key" days="730"/> - </Group> - <Group name="nginx"> - <Cert key="/etc/pki/tls/private/foo.key" days="730" - append_chain="true"/> - </Group> - </CertInfo> - -key.xml -------- - -.. xml:schema:: sslca-key.xsd - :linktotype: - :inlinetypes: KeyType - -Example -^^^^^^^ - -.. code-block:: xml - - <KeyInfo> - <Group name="fast"> - <Key type="rsa" bits="1024"/> - </Group> - <Group name="secure"> - <Key type="rsa" bits="4096"/> - </Group> - </KeyInfo> - -Automated Bcfg2 SSL Authentication -================================== - -This section describes one possible scenario for automating ssl -certificate generation and distribution for bcfg2 client/server -communication using SSLCA. The process involves configuring a -certificate authority (CA), generating the CA cert and key pair, -configuring the bcfg2 SSLCA plugin and a Bundle to use the SSLCA -generated certs to authenticate the bcfg2 client and server. - -OpenSSL CA ----------- - -If you already have a SSL CA available you can skip this section, -otherwise you can easily build one on the server using openssl. The -paths should be adjusted to suite your preferences. - -#. Prepare the directories and files:: - - mkdir -p /etc/pki/CA/newcerts - mkdir /etc/pki/CA/crl - echo '01' > /etc/pki/CA/serial - touch /etc/pki/CA/index.txt - touch /etc/pki/CA/crlnumber - -#. Edit the ``openssl.cnf`` config file, and in the **[ CA_default ]** - section adjust the following parameters:: - - dir = /etc/pki # Where everything is kept - certs = /etc/pki/CA/certs # Where the issued certs are kept - database = /etc/pki/CA/index.txt # database index file. - new_certs_dir = /etc/pki/CA/newcerts # default place for new certs. - certificate = /etc/pki/CA/certs/bcfg2ca.crt # The CA certificate - serial = /etc/pki/CA/serial # The current serial number - crl_dir = /etc/pki/CA/crl # Where the issued crl are kept - crlnumber = /etc/pki/CA/crlnumber # the current crl number - crl = /etc/pki/CA/crl.pem # The current CRL - private_key = /etc/pki/CA/private/bcfg2ca.key # The private key - -#. Create the CA root certificate and key pair. You'll be asked to - supply a passphrase, and some organizational info. The most - important bit is **Common Name** which you should set to be the - hostname of your bcfg2 server that your clients will see when doing - a reverse DNS query on it's ip address.:: - - openssl req -new -x509 -extensions v3_ca -keyout bcfg2ca.key \ - -out bcfg2ca.crt -days 3650 - -#. Move the generated cert and key to the locations specified in - ``openssl.cnf``:: - - mv bcfg2ca.key /etc/pki/CA/private/ - mv bcfg2ca.crt /etc/pki/CA/certs/ - -Your self-signing CA is now ready to use. - -Bcfg2 ------ - -SSLCA -^^^^^ - -The SSLCA plugin was not designed specifically to manage bcfg2 -client/server communication though it is certainly able to provide -certificate generation and management services for that -purpose. You'll need to configure the **SSLCA** plugin to serve the -key, and certificate paths that we will define later in our client's -``bcfg2.conf`` file. - -The rest of these instructions will assume that you've configured the -**SSLCA** plugin as described above and that the files -``SSLCA/etc/pki/tls/certs/bcfg2client.crt/cert.xml`` and -``SSLCA/etc/pki/tls/private/bcfg2client.key/key.xml`` represent the -cert and key paths you want generated for SSL auth. - -Client Bundle -^^^^^^^^^^^^^ - -To automate the process of generating and distributing certs to the -clients we need define at least the Cert and Key paths served by the -SSLCA plugin, as well as the ca certificate path in a Bundle. For -example: - -.. code-block:: xml - - <Path name='/etc/pki/tls/certs/bcfg2ca.crt'/> - <Path name='/etc/pki/tls/bcfg2client.crt'/> - <Path name='/etc/pki/tls/private/bcfg2client.key'/> - -Here's a more complete example bcfg2-client bundle: - -.. code-block:: xml - - <Bundle name='bcfg2-client'> - <Path name='/etc/bcfg2.conf'/> - <Path name='/etc/cron.d/bcfg2-client'/> - <Package name='bcfg2'/> - <Service name='bcfg2'/> - <Group name='rpm'> - <Path name='/etc/sysconfig/bcfg2'/> - <Path name='/etc/pki/tls/certs/bcfg2ca.crt'/> - <Path name='/etc/pki/tls/certs/bcfg2client.crt'/> - <Path name='/etc/pki/tls/private/bcfg2client.key'/> - </Group> - <Group name='deb'> - <Path name='/etc/default/bcfg2' altsrc='/etc/sysconfig/bcfg2'/> - <Path name='/etc/ssl/certs/bcfg2ca.crt' altsrc='/etc/pki/tls/certs/bcfg2ca.crt'/> - <Path name='/etc/ssl/certs/bcfg2client.crt' altsrc='/etc/pki/tls/certs/bcfg2client.crt'/> - <Path name='/etc/ssl/private/bcfg2client.key' altsrc='/etc/pki/tls/private/bcfg2client.key'/> - </Group> - </Bundle> - -In the above example we told Bcfg2 that it also needs to serve -``/etc/bcfg2.conf``. This is optional but convenient. - -The ``bcfg2.conf`` client config needs at least 5 parameters set for -SSL auth. - -#. ``key`` : This is the host specific key that SSLCA will generate. -#. ``certificate`` : This is the host specific cert that SSLCA will - generate. -#. ``ca`` : This is a copy of your CA certificate. Not generated by - SSLCA. -#. ``user`` : Usually set to fqdn of client. This *shouldn't* be - required but is as of 1.3.0. See: - http://trac.mcs.anl.gov/projects/bcfg2/ticket/1019 -#. ``password`` : Set to arbitrary string when using certificate - auth. This also *shouldn't* be required. See: - http://trac.mcs.anl.gov/projects/bcfg2/ticket/1019 - -Here's what a functional **[communication]** section in a -``bcfg2.conf`` genshi template for clients might look like.:: - - [communication] - protocol = xmlrpc/ssl - {% if metadata.uuid != None %}\ - user = ${metadata.uuid} - {% end %}\ - password = DUMMYPASSWORDFORCERTAUTH - {% choose %}\ - {% when 'rpm' in metadata.groups %}\ - certificate = /etc/pki/tls/certs/bcfg2client.crt - key = /etc/pki/tls/private/bcfg2client.key - ca = /etc/pki/tls/certs/bcfg2ca.crt - {% end %}\ - {% when 'deb' in metadata.groups %}\ - certificate = /etc/ssl/certs/bcfg2client.crt - key = /etc/ssl/private/bcfg2client.key - ca = /etc/ssl/certs/bcfg2ca.crt - {% end %}\ - {% end %}\ - -As a client will not be able to authenticate with certificates it does -not yet possess we need to overcome the chicken and egg scenario the -first time we try to connect such a client to the server. We can do so -using password based auth to boot strap the client manually specifying -all the relevant auth parameters like so:: - - bcfg2 -qv -S https://fqdn.of.bcfg2-server:6789 -u fqdn.of.client \ - -x SUPER_SECRET_PASSWORD - -If all goes well the client should recieve a freshly generated key and -cert and you should be able to run ``bcfg2`` again without specifying -the connection parameters. - -If you do run into problems you may want to review -:ref:`appendix-guides-authentication`. - -TODO -==== - -#. Add generation of pkcs12 format certs diff --git a/doc/server/plugins/generators/tcheetah.txt b/doc/server/plugins/generators/tcheetah.txt deleted file mode 100644 index c79a8ced5..000000000 --- a/doc/server/plugins/generators/tcheetah.txt +++ /dev/null @@ -1,197 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-plugins-generators-tcheetah: - -======== -TCheetah -======== - -.. warning:: - - TCheetah is deprecated. You should instead use - :ref:`server-plugins-generators-cfg-cheetah` in the Cfg plugin. - -This document reflects the ``TCheetah`` plugin. - -The ``TCheetah`` plugin allows you to use the `cheetah templating system -<http://www.cheetahtemplate.org/>`_ to create files, instead of the -various diff-based methods offered by the ``Cfg`` plugin. It also allows -you to include the results of probes executed on the client in the -created files. - -To begin, you will need to download and install the Cheetah templating -engine from http://www.cheetahtemplate.org/. Once it is installed, -you can enable it by adding ``TCheetah`` to the ``plugins`` line in -``/etc/bcfg2.conf`` on your Bcfg server. For example:: - - plugins = Base,Bundler,Cfg,...,TCheetah - -The ``TCheetah`` plugin makes use of a ``Cfg``-like directory structure -located in in a ``TCheetah`` subdirectory of your repository, usually -``/var/lib/bcfg2/TCheetah``. Each file has a directory containing two -files, ``template`` and ``info``. The template is a standard Cheetah -template with two additions: - -* `self.metadata` is the client's :ref:`metadata <server-plugins-grouping-metadata-clientmetadata>` -* `self.metadata.Properties.xdata` is an xml document of unstructured data - -The ``info`` file is formatted like ``:info`` files from Cfg. - -Mostly, people will want to use client metadata. - -File permissions -================ - -File permissions for entries handled by TCheetah are controlled via the -use of :ref:`server-info` files. Note that you **cannot** use both a -Permissions entry and a Path entry to handle the same file. - -self.metadata variables -======================= - -self.metadata is an instance of the class ClientMetadata and documented -:ref:`here <server-plugins-grouping-metadata-clientmetadata>`. - -self.metadata.Properties.xdata -============================== - -.. note:: - - If you want to use Properties, you will need to enable the - :ref:`server-plugins-connectors-properties` plugin in - ``/etc/bcfg2.conf``. - -Properties.xdata is a python `ElementTree <http://codespeak.net/lxml/>`_ -object, loaded from the data in ``/var/lib/bcfg2/Properties/<properties -file>.xml``. That file should have a ``Properties`` node at its root. - -Example ``Properties/example.xml``: - -.. code-block:: xml - - <Properties> - <host> - <www.example.com> - <rootdev>/dev/sda</rootdev> - </www.example.com> - </host> - </Properties> - -You may use any of the ElementTree methods to access data in your -template. Several examples follow, each producing an identical result -on the host 'www.example.com':: - - $self.metadata.Properties['example.xml'].xdata.find('host').find('www.example.com').find('rootdev').text - $self.metadata.Properties['example.xml'].xdata.find('host').find($self.metadata.hostname).find('rootdev').text - ${self.metadata.Properties['example.xml'].xdata.xpath('host/www.example.com/rootdev')[0].text} - ${self.metadata.Properties['example.xml'].xdata.xpath('host/' + self.metadata.hostname + '/rootdev')[0].text} - #set $path = 'host/' + $self.metadata.hostname + '/rootdev' - ${self.metadata.Properties['example.xml'].xdata.xpath($path)[0].text} - ${self.metadata.Properties['example.xml'].xdata.xpath(path)[0].text} - -Other Variables -=============== - -* **Template.searchList(self)[1]['path']** is the Path name specified in a Bundle -* **Template.searchList(self)[1]['source_path']** is the path to the TCheetah template on the Bcfg2 server - -Simple Example -============== - -TCheetah works similar to Cfg in that you define all literal information -about a particular file in a directory rooted at TCheetah/path_to_file. -The actual file contents are placed in a file named `template` in that -directory. Below is a simple example a file ``/foo``. - -``/var/lib/bcfg2/TCheetah/foo/template`` - -.. code-block:: none - - > buildfile /foo <clientname> - Hostname is $self.metadata.hostname - Filename is $Template.searchList(self)[1]['path'] - Template is $Template.searchList(self)[1]['source_path'] - Groups: - #for $group in $self.metadata.groups: - * $group - #end for - Categories: - #for $category in $self.metadata.categories: - * $category -- $self.metadata.categories[$category] - #end for - - Probes: - #for $probe in $self.metadata.Probes: - * $probe -- $self.metadata.Probes[$probe] - #end for - -``/var/lib/bcfg2/TCheetah/foo/info`` - -.. code-block:: none - - mode: 624 - -Output ------- - -The following output can be generated with bcfg2-info. Note that probe -information is not persistent, hence, it only works when clients directly -query the server. For this reason, bcfg2-info output doesn't reflect -current client probe state. - -.. code-block:: xml - - <Path type="file" name="/foo" owner="root" mode="0624" group="root"> - Hostname is topaz.mcs.anl.gov - Filename is /foo - Template is /var/lib/bcfg2/TCheetah/foo/template - Groups: - * desktop - * mcs-base - * ypbound - * workstation - * xserver - * debian-sarge - * debian - * a - Categories: - * test -- a - - Probes: - </Path> - -Example: Replace the crontab plugin -=================================== - -In many cases you can use the TCheetah plugin to avoid writing custom -plugins in Python. This example randomizes the time of cron.daily -execution with a stable result. Cron.daily is run at a consistent, -randomized time between midnight and 7am.:: - - #import random - #silent random.seed($self.metadata.hostname) - - # /etc/crontab: system-wide crontab - # Unlike any other crontab you don't have to run the `crontab` - # command to install the new version when you edit this file. - # This file also has a username field, that none of the other crontabs do. - - SHELL=/bin/sh - PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin://bin - - # m h dom mon dow user command - 17 * * * * root run-parts --report /etc/cron.hourly - $random.randrange(0,59) $random.randrange(0,6) * * * root test -x /usr/sbin/anacron || run-parts --report /etc/cron.daily - 47 6 * * 7 root test -x /usr/sbin/anacron || run-parts --report /etc/cron.weekly - 52 6 1 * * root test -x /usr/sbin/anacron || run-parts --report /etc/cron.monthly. - -.. note:: Comments and Cheetah - As Cheetah processes your templates it will consider hash "#" style - comments to be actual comments in the template and will strip them - from the final config file. If you would like to preserve the comment - in the final config file you need to escape the hash character '\#' - which will tell Cheetah (and Python) that you do in fact want the - comment to appear in the final config file.:: - - # This is a comment in my template which will be stripped when it's processed through Cheetah - \# This comment will appear in the generated config file. diff --git a/doc/server/plugins/generators/tgenshi.txt b/doc/server/plugins/generators/tgenshi.txt deleted file mode 100644 index 43a02f253..000000000 --- a/doc/server/plugins/generators/tgenshi.txt +++ /dev/null @@ -1,213 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-plugins-generators-tgenshi-index: - -======= -TGenshi -======= - -.. warning:: - - The TGenshi plugin is deprecated. You should instead use - :ref:`server-plugins-generators-cfg-genshi` in the Cfg plugin. - -This page documents the TGenshi plugin. This plugin works with version -0.4 and newer of the genshi library. - -The TGenshi plugin allows you to use the `Genshi -<http://genshi.edgewall.org>`_ templating system to create files, -instead of the various diff-based methods offered by the Cfg -plugin. It also allows you to include the results of probes executed -on the client in the created files. - -To begin, you will need to download and install the Genshi templating engine. - -To install on CentOS or RHEL, run:: - - sudo yum install python-genshi - -Once it is installed, you can enable it by adding ``TGenshi`` to the -generators line in ``/etc/bcfg2.conf`` on your Bcfg server. For example:: - - plugins = Base,Bundler,Cfg,...,TGenshi - -The TGenshi plugin makes use of a Cfg-like directory structure -located in in a TGenshi subdirectory of your repository, usually -``/var/lib/bcfg2/TGenshi``. Each file has a directory containing two file -types, template and info. Templates are named according to the genshi -format used; template.txt uses the genshi text format, and template.xml -uses the XML format. - -If used with Genshi 0.5 or later the plugin also supports the `new -style -<http://genshi.edgewall.org/wiki/Documentation/0.5.x/text-templates.html>`_ -text template format for files named template.newtxt. One of the -advantages of the new format is that it does not use # as a command -delimiter, making it easier to utilize for configuration files that -use # as a comment character. - -Only one template format may be used per file served. Info files are -identical to those used in ``Cfg``, and ``info.xml`` files are -supported. - -Inside of templates -=================== - -* **metadata** is the client's :ref:`metadata - <server-plugins-grouping-metadata-clientmetadata>` -* **metadata.Properties** is an xml document of unstructured data (only - available when used in conjunction with the - :ref:`server-plugins-connectors-properties` plugin) -* **name** is the path name specified in bcfg -* **path** is the path to the TGenshi template. It starts with a - leading slash, and is relative to the Bcfg2 specification root. - E.g., ``/Cfg/etc/foo.conf/foo.conf.genshi`` or - ``/TGenshi/etc/foo.conf/template.newtxt.H_foo.example.com`` - -See the genshi `documentation -<http://genshi.edgewall.org/wiki/Documentation>`_ for examples of -Genshi syntax. - -Examples: Old Genshi Syntax ---------------------------- - -Genshi's web pages recommend against using this syntax, as it may -disappear from future releases. - -Group Negation -^^^^^^^^^^^^^^ - -Templates are also useful for cases where more sophisticated boolean -operations than those supported by Cfg are needed. For example, the -template:: - - #if "ypbound" in metadata.groups and "workstation" in metadata.groups - client is ypbound workstation - #end - #if "ubuntu" not in metadata.groups and "desktop" in metadata.groups - client is a desktop, but not an ubuntu desktop - #end - -Produces: - -.. code-block:: xml - - <Path type="file" name="/bar.conf" owner="root" mode="0644" group="root">client is ypbound workstation - client is a desktop, but not an ubuntu desktop - </Path> - -This flexibility provides the ability to build much more compact and -succinct definitions of configuration contents than Cfg can. - -Troubleshooting -=============== - -When developing a template, you can see what the template would -generate on a client with :ref:`bcfg2-info <server-bcfg2-info>`:: - - bcfg2-info buildfile <path> <hostname> - -E.g.:: - - bcfg2-info buildfile /etc/foo.conf foo.example.com - -To generate a file with an altsrc attribute, you can run:: - - bcfg2-info buildfile /etc/foo/foo.conf --altsrc=/etc/foo.conf \ - foo.example.com - -Sometimes, it's useful to be able to do more in-depth troubleshooting -by running the template manually. To do this, run ``bcfg2-info -debug``, and, once in the Python interpreter, run:: - - metadata = self.build_metadata("<hostname>") - path = "<relative path to template (see note below)>" - -``path`` should be set to the path to the template file with a leading -slash, relative to the Bcfg2 specification root. See `Inside of -Templates`_ for examples. - -Then, run:: - - import os, Bcfg2.Options - from genshi.template import TemplateLoader, NewTextTemplate - name = os.path.dirname(path[path.find('/', 1):]) - setup = Bcfg2.Options.OptionParser({'repo': - Bcfg2.Options.SERVER_REPOSITORY}) - setup.parse('--') - template = TemplateLoader().load(setup['repo'] + path, cls=NewTextTemplate) - print template.generate(metadata=metadata, path=path, name=name).render() - -This gives you more fine-grained control over how your template is -rendered. - -You can also use this approach to render templates that depend on -:ref:`altsrc <server-plugins-structures-altsrc>` tags by setting -``path`` to the path to the template, and setting ``name`` to the path -to the file to be generated, e.g.:: - - metadata = self.build_metadata("foo.example.com") - path = "/Cfg/etc/sysconfig/network-scripts/ifcfg-template/ifcfg-template.genshi" - name = "/etc/sysconfig/network-scripts/ifcfg-bond0" - -File permissions -================ - -File permissions for entries handled by TGenshi are controlled via the -use of :ref:`server-info` files. Note that you **cannot** use both a -Permissions entry and a Path entry to handle the same file. - -Error handling -================ - -Situations may arise where a templated file cannot be generated due to -missing or incomplete information. A TemplateError can be raised to -force a bind failure and prevent sending an incomplete file to the -client. For example, this template:: - - {% python - from genshi.template import TemplateError - grp = None - for g in metadata.groups: - if g.startswith('ganglia-gmond-'): - grp = g - break - else: - raise TemplateError, "Missing group" - %}\ - -will fail to bind if the client is not a member of a group starting with -"ganglia-gmond-". The syslogs on the server will contain this message:: - - bcfg2-server[5957]: Genshi template error: Missing group - bcfg2-server[5957]: Failed to bind entry: Path /etc/ganglia/gmond.conf - -indicating the bind failure and message raised with the TemplateError. - -FAQs -==== - -**Question** - -How do I escape the $ (dollar sign) in a TGenshi text template? For -example, if I want to include SVN (subversion) keywords like $Id$ or -$HeadURL$ in TGenshi-generated files, or am templating a bourne shell -(sh/bash) script or Makefile (make). - -**Answer** - -Use $$ (double dollar sign) to output a literal $ (dollarsign) -in a TGenshi text template. So instead of $Id$, you'd use -$$Id$$. See also Genshi tickets `#282: Document $$ escape -convention <http://genshi.edgewall.org/ticket/282>`_ and -`#283: Allow for redefinition of template syntax per-file -<http://genshi.edgewall.org/ticket/283>`_. - -Examples -======== - -.. toctree:: - :glob: - :maxdepth: 1 - - examples/genshi/* diff --git a/doc/server/plugins/grouping/metadata.txt b/doc/server/plugins/grouping/metadata.txt index ceac5dc24..832b1a13f 100644 --- a/doc/server/plugins/grouping/metadata.txt +++ b/doc/server/plugins/grouping/metadata.txt @@ -90,6 +90,8 @@ Database Settings <server-database>`. The `clients.xml`_-based model remains the default. +.. _server-plugins-grouping-metadata-groups-xml: + groups.xml ========== @@ -180,76 +182,6 @@ groups: .. xml:schema:: metadata.xsd - -XInclude -======== - -.. versionadded:: 0.9.0 - -`XInclude <http://www.w3.org/TR/xinclude/>`_ is a W3C specification -for the inclusion of external XML documents into XML source files, -allowing complex definitions to be split into smaller, more manageable -pieces. The `Metadata`_ plugin supports the use of XInclude -specifications to split the `clients.xml`_ and `groups.xml`_ -files. This mechanism allows the following specification to produce -useful results: - -.. code-block:: xml - - <Groups xmlns:xi="http://www.w3.org/2001/XInclude"> - <xi:include href="my-groups.xml" /> - <xi:include href="their-groups.xml" /> - </Groups> - -Each of the included groups files has the same format. These files are -properly validated by ``bcfg2-lint``. This mechanism is useful for -composing group definitions from multiple sources, or setting -different permissions in an svn repository. - -You can also optionally include a file that may or may not exist with -the ``fallback`` tag: - -.. code-block:: xml - - <Groups xmlns:xi="http://www.w3.org/2001/XInclude"> - <xi:include href="my-groups.xml"/> - <xi:include href="their-groups.xml"><xi:fallback/></xi:include> - </Groups> - -In this case, if ``their-groups.xml`` does not exist, no error will be -raised and everything will work fine. (You can also use ``fallback`` -to include a different file, or explicit content in the case that the -parent include does not exist.) - -Wildcard XInclude ------------------ - -.. versionadded:: 1.3.1 - -Bcfg2 supports an extension to XInclude that allows you to use shell -globbing in the hrefs. (Stock XInclude doesn't support this, since -the href is supposed to be a URL.) - -For instance: - -.. code-block:: xml - - <Groups xmlns:xi="http://www.w3.org/2001/XInclude"> - <xi:include href="groups/*.xml"/> - </Groups> - -This would include all ``*.xml`` files in the ``groups`` subdirectory. - -Note that if a glob finds no files, that is treated the same as if a -single included file does not exist. You should use the ``fallback`` -tag, described above, if a glob may potentially find no files. - -Probes -====== - -The metadata plugin includes client-side probing functionality. This -is fully documented :ref:`here <server-plugins-probes-index>`. - Metadata Caching ================ diff --git a/doc/server/plugins/index.txt b/doc/server/plugins/index.txt index f3d6daa73..b39be0786 100644 --- a/doc/server/plugins/index.txt +++ b/doc/server/plugins/index.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-plugins-index: @@ -13,7 +14,7 @@ perform one of several tasks: #. Generating configuration entry contents for clients #. Probing client-side state (like hardware inventory, etc) -- the generic client probing mechanism is described at - :ref:`server-plugins-probes-index`. + :ref:`server-plugins-probes`. #. Automating administrative tasks (e.g. :ref:`server-plugins-generators-sshbase` which automates ssh key management) diff --git a/doc/server/plugins/misc/acl.txt b/doc/server/plugins/misc/acl.txt new file mode 100644 index 000000000..226b56a44 --- /dev/null +++ b/doc/server/plugins/misc/acl.txt @@ -0,0 +1,235 @@ +.. -*- mode: rst -*- + +.. _server-plugins-misc-acl: + +=== +ACL +=== + +The ACL plugin lets you set client communication ACLs to prevent +clients from accessing the full range of exposed XML-RPC methods. + +You can get a list of all exposed methods by running:: + + bcfg2-admin xcmd listMethods + +Note that this will only list methods that are available to the client +this is run from; that is, if the ACL plugin is in place, +``listMethods`` will reflect the ACLs. + +ACLs can be set in two different ways: + +* IP-based ACLs allow you to set ACLs based on client IP address or + CIDR range. +* Metadata-based ACLs allow you to set ACLs based on client hostname, + group membership, or complex combinations thereof. + +IP-based ACLs are much faster, but metadata-based ACLs are often +easier and better. + +If you are not going to use any ACLs, it is recommended that you +disable this plugin because using it can incur a slight performance +hit. If you are using IP-based ACLs but *not* metadata-based ACLs, it +is similarly recommended that you ensure that your IP-based ACL file +ends with an explicit Deny for all clients; this will ensure that +metadata-based ACLs are never checked. If you are using +metadata-based ACLs, :ref:`server-caching` can alleviate most of the +performance penalty. + +Enabling the ACL plugin +======================= + +First, create ``/var/lib/bcfg2/ACL/``. Then, add ``ACL`` to your +``plugins`` list in ``bcfg2.conf``:: + + plugins = Bundler, Cfg, ..., Packages, ACL + +Finally, create ``/var/lib/bcfg2/ACL/ip.xml`` (for `IP-based ACLs`_), +``/var/lib/bcfg2/ACL/metadata.xml`` (for `Metadata-based ACLs`_), or +both. + +IP-based ACLs +============= + +IP-based ACLs allow you to set ACLs based on client IP address or CIDR +range. IP-based ACLs are very fast. If you are using IP-based ACLs +but *not* metadata-based ACLs, it is recommended that you ensure that +your IP-based ACL file ends with an explicit Deny for all clients; +this will ensure that metadata-based ACLs are never checked. + +IP-based ACLs are defined in ``ACL/ip.xml``. The file is parsed +sequentially; the first matching rule applies. Each rule is either +Allow (to allow the client access), Deny (to deny the client access), +or Defer (to defer to `Metadata-based ACLs`_). The last rule in +``ip.xml`` is an implicit default allow for 127.0.0.1, and an implicit +default defer for all other machines. + +If no ``ip.xml`` file exists, then ACL checking will be deferred to +metadata-based ACLs. + +Example +------- + +.. code-block:: xml + + <ACL> + <Allow address="192.168.1.10" method="*.*"/> + <Deny address="192.168.2.0" netmask="255.255.255.0" + method="AssertProfile"/> + <Allow address="192.168.1.12" method="Git.Update"/> + <Allow method="*"/> + </ACL> + +In this example: + +* The machine at 192.168.1.10 (perhaps the Bcfg2 server) can call all + plugin XML-RPC methods; +* Machines in the 192.168.2.0/24 network cannot assert their own + profiles; +* The machine at 192.168.1.12 (perhaps the Git server) can call the + Git.Update method; +* All machines can call core methods (except 192.168.2.0/24, which can + call all core methods except AssertProfile). + +Implicitly, all machines (except localhost) except 192.168.1.10 are +disallowed access to the plugin methods. + +You can also provide a minimal configuration to try to weed out some +obvious bad requests before doing the more expensive `Metadata-based +ACLs`_. For instance: + +.. code-block:: xml + + <ACL> + <Allow method="*"/> + <Defer address="192.168.1.0" netmask="24" method="*.*"/> + <Deny method="*.*"/> + </ACL> + +In this example: + +* All machines can call all core methods without checking metadata + ACLs; +* Plugin method calls from machines in 192.168.1.0/24 are deferred to + metadata ACLs; and +* All other plugin method calls are denied. + +The only time metadata ACLs would be checked in this example would be +plugin method calls by machines in 192.168.1.0/24. + +Reference +--------- + +.. xml:type: IPACLContainerType + +Metadata-based ACLs +=================== + +Metadata-based ACLs let you set ACLs based on client hostname or group +membership, which is much more flexible and maintainable than +`IP-based ACLs`_. The downside is that it is slower, because it +requires generating client metadata for each machine that tries to +authenticate. Without :ref:`server-caching`, using metadata-based +ACLs will double the number of client metadata builds per client run, +which could be a sizeable performance penalty. + +In order to limit the performance penalty, it's highly recommended +to: + +* Enable :ref:`server-caching` in ``cautious`` or ``aggressive`` mode; + and +* Deny as many clients as possible with `IP-based ACLs`_. + +Metadata-based ACLs are defined in ``ACL/metadata.xml``. Only Allow +and Deny rules are supported, not Defer rules. The file is parsed +sequentially; the first matching rule applies. The last rule in +``metadata.xml`` is an implicit default allow for machines called +``localhost`` or ``localhost.localdomain``, and an implicit default +deny for all other machines. + +If no ``metadata.xml`` file exists, then all requests are implicitly +allowed. + +Example +------- + +This example is functionally identical to the `IP-based ACLs` example +above, but more maintainable in several ways: + +.. code-block:: xml + + <ACL> + <Group name="bcfg2-server"> + <Allow method="*.*"/> + </Group> + <Group name="user-workstations"> + <Deny method="AssertProfile"/> + </Group> + <Group name="git-server"> + <Allow method="Git.Update"/> + </Group> + <Allow method="*"/> + </ACL> + +In this case, if you add a Bcfg2 server or Git server, or one of those +servers changes IP address, you don't need to rewrite your ACLs. +Similarly, you could add a new subnet of user workstations. + +Reference +--------- + +.. xml:type: MetadataACLContainerType + +.. _server-plugins-misc-acl-wildcards: + +Wildcards +========= + +The ACL descriptions allow you to use '*' as a wildcard for any number +of characters *other than* ``.``. That is: + +* ``*`` would match ``DeclareVersion`` and ``GetProbes``, but would + *not* match ``Git.Update``. +* ``*.*`` would match ``Git.Update``, but not ``DeclareVersion`` or + ``GetProbes``. + +Since all plugin methods are scoped to their plugin (i.e., they are +all ``<plugin name>.<method name>``), and all core methods have no +scope, this lets you easily allow or deny core or plugin methods. You +could also do something like ``*.toggle_debug`` to allow a host to +enable or disable debugging for all plugins. + +No other bash globbing is supported. + +Examples +======== + +The :ref:`default ACL list <server-access-control>` can be described +in ``ip.xml`` fairly simply: + +.. code-block:: xml + + <ACL> + <Allow address="127.0.0.1" method="*.*"/> + <Allow address="127.0.0.1" method="*"/> + <Deny method="*.*"/> + <Deny method="*_debug"/> + <Deny method="get_statistics"/> + <Allow method="*"/> + </ACL> + +A basic configuration that is still very secure but perhaps more +functional could be given in ``metadata.xml``: + +.. code-block:: xml + + <ACL> + <Group name="bcfg2-server"> + <Allow method="*.*"/> + <Allow method="*"/> + </Group> + <Deny method="*.*"/> + <Deny method="*_debug"/> + <Deny method="get_statistics"/> + <Allow method="*"/> + </ACL> diff --git a/doc/server/plugins/probes/fileprobes.txt b/doc/server/plugins/probes/fileprobes.txt index 0baec2c59..1bee38c5a 100644 --- a/doc/server/plugins/probes/fileprobes.txt +++ b/doc/server/plugins/probes/fileprobes.txt @@ -1,3 +1,5 @@ +.. -*- mode: rst -*- + .. _server-plugins-probes-fileprobes: ========== diff --git a/doc/server/plugins/probes/index.txt b/doc/server/plugins/probes/index.txt index 2e23c31d5..434ce20a8 100644 --- a/doc/server/plugins/probes/index.txt +++ b/doc/server/plugins/probes/index.txt @@ -1,6 +1,7 @@ .. -*- mode: rst -*- +.. vim: ft=rst -.. _server-plugins-probes-index: +.. _server-plugins-probes: ====== Probes @@ -14,7 +15,7 @@ generate an `/etc/auto.master` autofs config file for each type. Here we will look at how to do this. Probes also allow dynamic group assignment for clients, see -:ref:`_server-plugins-probes-dynamic-groups`. +:ref:`server-plugins-probes-dynamic-groups`. First, create a ``Probes`` directory in our toplevel repository location:: diff --git a/doc/server/plugins/statistics/reporting.txt b/doc/server/plugins/statistics/reporting.txt index c3c51cd29..74ea61e62 100644 --- a/doc/server/plugins/statistics/reporting.txt +++ b/doc/server/plugins/statistics/reporting.txt @@ -9,7 +9,7 @@ Reporting Reporting can be enabled by adding Reporting to the plugins line in ``/etc/bcfg2.conf``: - plugins = Base,Bundler,Cfg,...,Reporting + plugins = Bundler,Cfg,...,Reporting For more information on how to use Reporting to setup reporting, see :ref:`reports-dynamic`. diff --git a/doc/server/plugins/statistics/statistics.txt b/doc/server/plugins/statistics/statistics.txt deleted file mode 100644 index d16f5a828..000000000 --- a/doc/server/plugins/statistics/statistics.txt +++ /dev/null @@ -1,7 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-plugins-statistics-statistics: - -========== -Statistics -========== diff --git a/doc/server/plugins/structures/altsrc.txt b/doc/server/plugins/structures/altsrc.txt index 1268a8584..f3911e33e 100644 --- a/doc/server/plugins/structures/altsrc.txt +++ b/doc/server/plugins/structures/altsrc.txt @@ -11,7 +11,7 @@ altsrc Altsrc is a generic, Bcfg2 server-side mechanism for performing configuration entry name remapping for the purpose of data binding. Altsrc can be used as a parameter for any entry type, and can be used -in any structure, including Bundler and Base. +in any structure. Use Cases ========= @@ -36,7 +36,7 @@ Examples .. code-block:: xml - <Bundle name='netinfo'> + <Bundle> <Group name='solaris'> <Path name='/etc/inet/hosts' altsrc='/etc/hosts'/> </Group> @@ -58,7 +58,7 @@ Examples .. code-block:: xml - <Bundle name='openssl'> + <Bundle> <Package name='openssl' altsrc='openssl-encap'/> <Package name='openssl' altsrc='openssl-rpm'/> </Bundle> @@ -76,7 +76,7 @@ Examples .. code-block:: xml - <Bundle name='firewall'> + <Bundle> ... <Group name='conduit'> <Path name='/etc/firewall-rules' altsrc='/etc/firewall-rules-external'/> @@ -97,7 +97,7 @@ Examples .. code-block:: xml - <Bundle name='netconfig'> + <Bundle> <Path name='/etc/sysconfig/network-scripts/ifcfg-eth0' altsrc='/etc/ifcfg-template'/> <Path name='/etc/sysconfig/network-scripts/ifcfg-eth1' altsrc='/etc/ifcfg-template'/> <Path name='/etc/sysconfig/network-scripts/ifcfg-eth2' altsrc='/etc/ifcfg-template'/> diff --git a/doc/server/plugins/structures/base.txt b/doc/server/plugins/structures/base.txt deleted file mode 100644 index 03eae0573..000000000 --- a/doc/server/plugins/structures/base.txt +++ /dev/null @@ -1,83 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-plugins-structures-base: - -==== -Base -==== - -.. deprecated:: 1.2.0 - -.. warning:: - - The Base plugin no longer receives new features/functionality. - Please use :ref:`server-plugins-structures-bundler-index` instead. - -The Base plugin is a structure plugin that provides the ability to add -lists of unrelated entries into client configuration entry inventories. - -Base works much like Bundler in its file format. The main difference -between Base and Bundler is that Base files are included in all clients' -configuration whereas bundles must be included explicitly in your -Metadata. See the :ref:`server-plugins-structures-bundler-index` page -for details. - -If you have lots of unconnected items (for instance: software packages -whose configuration wasn't modified, and that are also not depended -on by other packages; or single directories or files not belonging -to a package), using Bundles in Metadata would clutter or enlarge -your ``Metadata/groups.xml`` file, because they all would need to be -explicitly specified. ``Base/`` on the other hand is the perfect place -to put these items. - -Without using Base, you would be forced to put them directly -into your group definitions in ``groups.xml``, either as many -small bundles (substantially enlarging it) or into something like -``Bundler/unrelated-entries.xml``. Using the latter is especially bad -if you mix packages and services in your Bundle, since for any updated -package in that bundle, the now-related services would be restarted. - -The Base entries can still be assigned based on group membership, but when -they aren't part of a group, each and every client gets the entry. So Base is -also a great place to put entries that a large number of your clients will -get. - -For example, you could have a file ``Base/packages.xml`` - -.. code-block:: xml - - <Base> - <Package name='acpid'/> - <Package name='auditd'/> - [...] - <Group name='openSUSE11.2'> - <Package name='syslog-ng'/> - </Group> - <Group name='openSUSE11.3'> - <Package name='rsyslog'/> - </Group> - [...] - <Package name='zlib'/> - </Base> - -.. note:: - - You don't have to reference to the files in Base from anywhere. As long - as you include ``Base`` in your ``plugins = ...`` line in ``bcfg2.conf``, - these are included automatically. - -.. note:: - - Your Base files have to match the pattern ``Base/*.xml`` to be included. - - -The decision when to use Base and when to use Bundler depends on the -configuration entry in question, and what you are trying to achieve. - -Base is mainly used for cases where you don't want/need to explicitly -include particular configuration items. Let's say all your machines are -various linux distributions. In this case, you may want to manage the -``/etc/hosts`` file using Base instead of Bundler since you will not have -to include any Bundles in your Metadata. However, you could alternatively -have a base 'linux' group that all the clients inherit which includes a -*linux* Bundle with the ``/etc/hosts`` configuration entry. diff --git a/doc/server/plugins/structures/bundler/bcfg2.txt b/doc/server/plugins/structures/bundler/bcfg2.txt new file mode 100644 index 000000000..0fd0a3fdf --- /dev/null +++ b/doc/server/plugins/structures/bundler/bcfg2.txt @@ -0,0 +1,87 @@ +.. -*- mode: rst -*- + +.. _server-plugins-structures-bundler-bcfg2-server: + +Bcfg2 Server +============ + +These two bundles split out the entries that do require a restart of +``bcfg2-server`` from those that don't. + +These bundles also demonstrate use of bound entries to avoid splitting +entries between Bundler and Rules. + +``Bundler/bcfg2-server.xml``: + +.. code-block:: xml + + <Bundle> + <Bundle name="bcfg2-server-base.xml"/> + + <Path name="/etc/pki/tls/private/bcfg2.key"/> + <Path name="/etc/sysconfig/bcfg2-server"/> + <Path name="/etc/bcfg2.conf"/> + + <BoundPath name="/var/lib/bcfg2/Packages/cache" type="directory" + owner="bcfg2" group="bcfg2" mode="0755"/> + <BoundPath name="/var/lib/bcfg2" type="symlink" + to="/var/lib/bcfg2-vcs/bcfg2/public"/> + <BoundPath name="/var/lib/bcfg2/etc/bcfg2.sqlite" type="permissions" + owner="bcfg2" group="apache" mode="0660"/> + + <BoundService name="bcfg2-server" type="chkconfig" status="on"/> + + <Package name="bcfg2-server"/> + <Package name="python-genshi"/> + <Package name="python-inotify"/> + <Package name="Django"/> + <Package name="Django-south"/> + <Package name="m2crypto"/> + <Package name="GitPython"/> + </Bundle> + +``Bundler/bcfg2-server-base.xml``: + +.. code-block:: xml + + <Bundle> + <Path name="/etc/bcfg2-web.conf"/> + <Path name="/etc/cron.daily/bcfg2_cleanup_db"/> + + <BoundPOSIXGroup name='bcfg2'/> + <BoundPOSIXUser name='bcfg2' shell='/sbin/nologin' gecos='Bcfg2 User'/> + <Path name="/home/bcfg2/.ssh/id_rsa"/> + + <!-- SSL CA setup --> + <BoundPath name="/etc/pki/CA" type="directory" important="true" + owner="bcfg2" group="bcfg2" mode="755"/> + <BoundPath name="/etc/pki/CA/crl" type="directory" owner="bcfg2" + group="bcfg2" mode="755"/> + <BoundPath name="/etc/pki/CA/certs" type="directory" owner="bcfg2" + group="bcfg2" mode="755"/> + <BoundPath name="/etc/pki/CA/newcerts" type="directory" owner="bcfg2" + group="bcfg2" mode="755"/> + <BoundPath name="/etc/pki/CA/private" type="directory" owner="bcfg2" + group="bcfg2" mode="755"/> + <Path name="/etc/pki/CA/openssl.cnf" altsrc="/etc/pki/CA/openssl.cnf"/> + <Path name="/etc/pki/CA/index.txt.attr"/> + <Path name="/etc/pki/CA/CA.crt"/> + <Path name="/etc/pki/CA/CA.key"/> + <Path name="/etc/pki/CA/CA.pem"/> + <Path name="/etc/pki/tls/certs/server-chain.crt"/> + <BoundPath name="/etc/pki/CA/serial" type="permissions" owner="bcfg2" + group="bcfg2" mode="0600"/> + <BoundPath name="/etc/pki/CA/index.txt" type="permissions" owner="bcfg2" + group="bcfg2" mode="0600"/> + <BoundPath name="/etc/pki/CA/crlnumber" type="permissions" owner="bcfg2" + group="bcfg2" mode="0644"/> + <BoundAction + name="create-CA-serial" timing="post" when="always" status="check" + command="[ -e /etc/pki/CA/serial ] || echo '01' > /etc/pki/CA/serial"/> + <BoundAction + name="create-CA-index" timing="post" when="always" status="check" + command="[ -e /etc/pki/CA/index.txt ] || touch /etc/pki/CA/index.txt"/> + <BoundAction + name="create-CA-crlnumber" timing="post" when="always" status="check" + command="[ -e /etc/pki/CA/crlnumber ] || touch /etc/pki/CA/crlnumber"/> + </Bundle> diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt index 51f2da60c..31faeaf17 100644 --- a/doc/server/plugins/structures/bundler/index.txt +++ b/doc/server/plugins/structures/bundler/index.txt @@ -1,7 +1,7 @@ .. -*- mode: rst -*- .. vim: ft=rst -.. _server-plugins-structures-bundler-index: +.. _server-plugins-structures-bundler: ======= Bundler @@ -20,142 +20,118 @@ will receive. Group and Client tags can be used inside of bundles to differentiate which entries particular clients will recieve; this is useful for the case where entries are named differently across systems; for example, -one linux distro may have a package called openssh while another uses -the name ssh. Configuration entries nested inside of Group elements -only apply to clients who are a member of those groups; multiple -nested groups must all apply. Also, groups may be negated; entries -included in such groups will only apply to clients who are not a -member of said group. The same applies to Client elements. +one Linux distro may have a package called ``openssh`` while another +uses the name ``ssh``. See :ref:`xml-group-client-tags` for details +and a longer example. -The following is an annotated copy of a bundle: +A brief example: .. code-block:: xml - <Bundle name='ssh' version='2.0'> - <Path name='/etc/ssh/ssh_host_dsa_key'/> - <Path name='/etc/ssh/ssh_host_rsa_key'/> - <Path name='/etc/ssh/ssh_host_dsa_key.pub'/> - <Path name='/etc/ssh/ssh_host_rsa_key.pub'/> - <Path name='/etc/ssh/ssh_host_key'/> - <Path name='/etc/ssh/ssh_host_key.pub'/> - <Path name='/etc/ssh/sshd_config'/> + <Bundle> <Path name='/etc/ssh/ssh_config'/> - <Path name='/etc/ssh/ssh_known_hosts'/> <Group name='rpm'> - <Package name='openssh'/> - <Package name='openssh-askpass'/> <Service name='sshd'/> - <Group name='fedora' > - <Group name='fc14' negate='true'> - <Package name='openssh-clients'/> - </Group> - <Package name='openssh-server'/> - </Group> + <Package name='openssh-server'/> </Group> <Group name='deb'> <Package name='ssh'/> <Service name='ssh'/> </Group> - <Client name='trust.example.com'> - <Path name='/etc/ssh/shosts.equiv'/> - </Client> </Bundle> -In this bundle, most of the entries are common to all systems. Clients -in group **deb** get one extra package and service, while clients in -group **rpm** get two extra packages and an extra service. In -addition, clients in group **fedora** *and* group **rpm** get one -extra package entries, unless they are not in the **fc14** group, in -which case, they get an extra package. The client -**trust.example.com** gets one extra file that is not distributed to -any other clients. Notice that this file doesn't describe which -versions of these entries that clients should get, only that they -should get them. (Admittedly, this example is slightly contrived, but -demonstrates how group entries can be used in bundles) - -+----------------------------+-------------------------------+ -| Group/Hostname | Entry | -+============================+===============================+ -| all | /etc/ssh/ssh_host_dsa_key | -+----------------------------+-------------------------------+ -| all | /etc/ssh/ssh_host_rsa_key | -+----------------------------+-------------------------------+ -| all | /etc/ssh/ssh_host_dsa_key.pub | -+----------------------------+-------------------------------+ -| all | /etc/ssh/ssh_host_rsa_key.pub | -+----------------------------+-------------------------------+ -| all | /etc/ssh/ssh_host_key | -+----------------------------+-------------------------------+ -| all | /etc/ssh/ssh_host_key.pub | -+----------------------------+-------------------------------+ -| all | /etc/ssh/sshd_config | -+----------------------------+-------------------------------+ -| all | /etc/ssh/ssh_config | -+----------------------------+-------------------------------+ -| all | /etc/ssh/ssh_known_hosts | -+----------------------------+-------------------------------+ -| rpm | Package openssh | -+----------------------------+-------------------------------+ -| rpm | Package openssh-askpass | -+----------------------------+-------------------------------+ -| rpm | Service sshd | -+----------------------------+-------------------------------+ -| rpm and fedora | Package openssh-server | -+----------------------------+-------------------------------+ -| rpm and fedora and not fc4 | Package openssh-clients | -+----------------------------+-------------------------------+ -| deb | Package ssh | -+----------------------------+-------------------------------+ -| deb | Service ssh | -+----------------------------+-------------------------------+ -| trust.example.com | /etc/ssh/shosts.equiv | -+----------------------------+-------------------------------+ +Note that we do not specify *how* a given entry should be managed, +only that it should be. The concrete specification of each entry will +be provided by a different plugin such as +:ref:`server-plugins-generators-cfg`, +:ref:`server-plugins-generators-rules`, or +:ref:`server-plugins-generators-packages`. -Genshi templates -================ +Alternatively, you can use fully-bound entries in Bundler, which has +various uses. For instance: -Genshi XML templates allow you to use the `Genshi -<http://genshi.edgewall.org>`_ templating system to dynamically generate -a bundle. Genshi templates can be specified **one** of two ways: +.. code-block:: xml -* Add an XML-style genshi template to the Bundler directory with a - ``.genshi`` and the associated namespace attribute. -* Simply add the appropriate namespace attribute to your existing XML - bundle. + <Bundle> + <Path name='/etc/ssh/ssh_config'/> + <Group name='rpm'> + <BoundService name='sshd' type="chkconfig" status="on"/> + <BoundPackage name='openssh-server' version='5.8p2' type="yum" /> + </Group> + <Group name='deb'> + <Package name='ssh'/> + <BoundService name='ssh' type="chkconfig" status="on"/> + </Group> + </Bundle> -The top-level Bundle tag should look like the following:: +In this example, both Service tags and one Package tag are fully bound +-- i.e., all information required by the client to manage those +entries is provided in the bundle itself. - <Bundle name="foo" xmlns:py="http://genshi.edgewall.org/"> +.. _server-plugins-structures-bundler-magic: -Several variables are pre-defined inside templates: +Bundle "Magic" +============== -+-------------+--------------------------------------------------------+ -| Name | Description | -+=============+========================================================+ -| metadata | :ref:`Client metadata | -| | <server-plugins-grouping-metadata-clientmetadata>` | -+-------------+--------------------------------------------------------+ -| repo | The path to the Bcfg2 repository on the filesystem | -+-------------+--------------------------------------------------------+ +Bundles are collections of *related* entries. That point is very, +very important, because a bundle performs certain "magic" actions when +one or more entries in it are modified: -.. note:: +* :xml:type:`Service <ServiceType>` entries whose ``restart`` + attribute is ``true`` (the default) will be restarted. +* :xml:type:`Action <ActionType>` entries whose ``when`` attribute is + ``modified`` will be run. + +Because of these two magic actions, it's extremely important to +structure your bundles around Service and Action entries, rather than +around some loose idea of which entries are related. For instance, in +order to manage a Bcfg2 server, a number of packages, paths, services, +etc. must be managed. But not all of these entries would require +``bcfg2-server`` to be restarted, so to limit restarts it's wise to +split these entries into two bundles. See +:ref:`server-plugins-structures-bundler-bcfg2-server` for an example +of this. + + +.. _server-plugins-structures-bundler-index-disabling-magic: + +Disabling Magic +--------------- - ``<Group>`` and ``<Client>`` tags are allowed inside of Genshi - templates as of Bcfg2 1.2. However, they do not behave the same - as using a Genshi conditional, e.g.:: +Disabling magic bundler actions can be done in one of two ways: - <py:if test="'groupname' in metadata.groups"> - </py:if> +* On a per-entry basis. Set ``restart="false"`` on a Service to + prevent it from being restarted when the bundle is modified. Set + ``when="always"`` on an Action to cause it to run every time, + regardless of whether or not the bundle was modified. +* On a per-bundle basis. Set ``independent="true"`` on the top-level + ``Bundle`` tag to signify that the bundle is a collection of + independent (i.e., unrelated) entries, and to prevent any magic + actions from being performed. (This is similar to the ``Base`` + plugin in older versions of Bcfg2.) This was added in Bcfg2 1.4. - The conditional is evaluated when the template is rendered, so - code inside the conditional is not executed if the conditional - fails. A ``<Group>`` tag is evaluated *after* the template is - rendered, so code inside the tag is always executed. This is an - important distinction: if you have code that will fail on some - groups, you *must* use a Genshi conditional, not a ``<Group>`` - tag. The same caveats apply to ``<Client>`` tags. +Service entries in independent bundles are never restarted, and Action +entries in independent bundles are only executed if ``when="always"``. +(I.e., an Action entry in an independent bundle with +``when="modified"`` is useless.) -See also the :ref:`xml-genshi-reference`. + +.. _server-plugins-structures-bundler-index-genshi-templates: + +Genshi templates +================ + +Genshi XML templates allow you to use the `Genshi +<http://genshi.edgewall.org>`_ templating system to dynamically +generate a bundle. Genshi templates can be specified one of two ways: + +1. Add an XML-style genshi template to the Bundler directory with a + ``.genshi`` and the associated namespace attribute. *This is + deprecated as of Bcfg2 1.4.* +2. Add the Genshi namespace to your existing XML + bundle. + +See :ref:`xml-genshi-templating` for details. Troubleshooting --------------- @@ -169,6 +145,58 @@ entries in the bundle. See :ref:`bcfg2-info <server-bcfg2-info>` for more details. + +.. _server-plugins-structures-bundler-index-dependencies: + +Dependencies +============ + +Dependencies on other bundles can be specified by adding an empty +bundle tag that adds another bundle by name, e.g.: + +.. code-block:: xml + + <Bundle> + <Bundle name="nfs-client"/> + ... + </Bundle> + +The dependent bundle is added to the list of bundles sent to the +client, *not* to the parent bundle itself. In other words, if an +entry in the dependent bundle changes, Services are restarted and +Actions are run in the dependent bundle *only*. An example: + +``nfs-client.xml``: + +.. code-block:: xml + + <Bundle> + <Package name="nfs-utils"/> + <Service name="nfslock"/> + <Service name="rpcbind"/> + <Service name="nfs"/> + </Bundle> + +``automount.xml``: + +.. code-block:: xml + + <Bundle> + <Bundle name="nfs-client"/> + + <Path name="/mnt/home"/> + <Path name="/etc/auto.master"/> + <Path name="/etc/auto.misc"/> + <Service name="autofs"/> + <Package name="automount"/> + </Bundle> + +If a new ``nfs-utils`` package was installed, the ``nfslock``, +``rpcbind``, and ``nfs`` services would be restarted, but *not* the +``autofs`` service. Similarly, if a new ``/etc/auto.misc`` file was +sent out, the ``autofs`` service would be restarted, but the +``nfslock``, ``rpcbind``, and ``nfs`` services would not be restarted. + Altsrc ====== @@ -185,8 +213,8 @@ in their name. The following template produces such a config file entry. .. code-block:: xml - <Bundle name='foo' xmlns:py="http://genshi.edgewall.org/"> - <Path name='/etc/package-${metadata.hostname}'/> + <Bundle xmlns:py="http://genshi.edgewall.org/"> + <Path name='/etc/package-${metadata.hostname}'/> </Bundle> Depending on the circumstance, these configuration files can either be @@ -200,7 +228,7 @@ and returns them in a newline delimited string. .. code-block:: xml - <Bundle name="networkinterfaces" xmlns:py="http://genshi.edgewall.org/"> + <Bundle xmlns:py="http://genshi.edgewall.org/"> <?python files = metadata.Probes["getmacs"].split("\n") ?> @@ -220,7 +248,7 @@ if declaration. .. code-block:: xml - <Bundle name='bacula' xmlns:py="http://genshi.edgewall.org/"> + <Bundle xmlns:py="http://genshi.edgewall.org/"> <Path name="/etc/bacula/bconsole.conf"/> <Path name="/etc/bacula/bacula-fd.conf"/> <Path name="/etc/bacula/bacula-sd.conf"/> @@ -232,7 +260,7 @@ or alternately .. code-block:: xml - <Bundle name='bacula' xmlns:py="http://genshi.edgewall.org/"> + <Bundle xmlns:py="http://genshi.edgewall.org/"> <Path name="/etc/bacula/bconsole.conf"/> <Path name="/etc/bacula/bacula-fd.conf"/> <Path name="/etc/bacula/bacula-sd.conf"/> @@ -245,7 +273,7 @@ or yet another way .. code-block:: xml - <Bundle name='bacula' xmlns:py="http://genshi.edgewall.org/"> + <Bundle xmlns:py="http://genshi.edgewall.org/"> <Path name="/etc/bacula/bconsole.conf"/> <Path name="/etc/bacula/bacula-fd.conf"/> <Path name="/etc/bacula/bacula-sd.conf"/> @@ -275,6 +303,7 @@ more complex example Bundles. .. toctree:: :maxdepth: 1 + bcfg2 kernel moab nagios diff --git a/doc/server/plugins/structures/bundler/kernel.txt b/doc/server/plugins/structures/bundler/kernel.txt index d83679683..54f70606f 100644 --- a/doc/server/plugins/structures/bundler/kernel.txt +++ b/doc/server/plugins/structures/bundler/kernel.txt @@ -22,7 +22,7 @@ some of which might be better than this one. Feel free to hack as needed. .. code-block:: xml - <Bundle name='kernel'> + <Bundle> <Group name='sles8'> <!-- =================== ia32 ==================== --> <Group name='ia32'> diff --git a/doc/server/plugins/structures/bundler/moab.txt b/doc/server/plugins/structures/bundler/moab.txt index e0d96be74..8f747376a 100644 --- a/doc/server/plugins/structures/bundler/moab.txt +++ b/doc/server/plugins/structures/bundler/moab.txt @@ -9,7 +9,7 @@ This is a fairly simple Bundle for the Moab workload manager. .. code-block:: xml - <Bundle name='moab' version='2.0'> + <Bundle> <Path name='/var/spool/moab'/> <Path name='/var/spool/moab/moab.cfg'/> <Group name='moab-server'> diff --git a/doc/server/plugins/structures/bundler/nagios.txt b/doc/server/plugins/structures/bundler/nagios.txt index fa5b67f30..47a61b898 100644 --- a/doc/server/plugins/structures/bundler/nagios.txt +++ b/doc/server/plugins/structures/bundler/nagios.txt @@ -12,7 +12,7 @@ the clients. .. code-block:: xml - <Bundle name='nagios-client' version='2.0'> + <Bundle> <Group name='sles8'> <Package name='ucdsnmp'/> </Group> @@ -27,29 +27,14 @@ the clients. <Path name='/etc/hosts.deny'/> <Path name='/etc/services'/> <Path name='/etc/snmpd.conf'/> - <Path name='/usr/lib/nagios/plugins/check_disks_scratchgpfs1.tg'/> - <Path name='/usr/lib/nagios/plugins/check_fs.mds'/> - <Path name='/usr/lib/nagios/plugins/check_gm_network.tg'/> - <Path name='/usr/lib/nagios/plugins/check_gpfs_wan.tg'/> - <Path name='/usr/lib/nagios/plugins/check_hung_jobs.tg'/> - <Path name='/usr/lib/nagios/plugins/check_mem.mds'/> - <Path name='/usr/lib/nagios/plugins/check_mem.tg'/> - <Path name='/usr/lib/nagios/plugins/check_nvidia_acceleration.tg'/> - <Path name='/usr/lib/nagios/plugins/check_os.mds'/> - <Path name='/usr/lib/nagios/plugins/check_procinfo.mds'/> - <Path name='/usr/lib/nagios/plugins/check_torque.tg'/> - <Path name='/usr/lib/nagios/plugins/check_uname_r.tg'/> - <Path name='/usr/lib/nagios/plugins/check_uname_r.tg.conf'/> + <Path glob='/usr/lib/nagios/plugins/*'/> <Service name='snmpd'/> <Group name='nagios-server'> <Package name='nagios'/> <Package name='nagios-devel'/> <Package name='nagios-www'/> <Path name='/etc/httpd/conf.d/nagios.conf'/> - <Path name='/etc/nagios/cgi.cfg'/> - <Path name='/etc/nagios/checkcommands.cfg'/> - <Path name='/etc/nagios/nagios.cfg'/> - <Path name='/etc/nagios/resource.cfg'/> + <Path glob='/etc/nagios/*'/> </Group> </Bundle> diff --git a/doc/server/plugins/structures/bundler/ntp.txt b/doc/server/plugins/structures/bundler/ntp.txt index b1264b5ee..31bc8a97a 100644 --- a/doc/server/plugins/structures/bundler/ntp.txt +++ b/doc/server/plugins/structures/bundler/ntp.txt @@ -12,7 +12,7 @@ better through use of groups. .. code-block:: xml - <Bundle name='ntp'> + <Bundle> <Package name='xntp'/> <Path name='/etc/sysconfig/xntp'/> <Path name='/etc/sysconfig/clock'/> diff --git a/doc/server/plugins/structures/bundler/snmpd.txt b/doc/server/plugins/structures/bundler/snmpd.txt index 2318f8ca1..859e07f7f 100644 --- a/doc/server/plugins/structures/bundler/snmpd.txt +++ b/doc/server/plugins/structures/bundler/snmpd.txt @@ -10,7 +10,7 @@ configuration file. .. code-block:: xml - <Bundle name="snmpd" version="3.0"> + <Bundle> <Package name="snmpd"/> <Service name="snmpd"/> <Path name="/etc/snmp/snmpd.conf"/> diff --git a/doc/server/plugins/structures/bundler/torque.txt b/doc/server/plugins/structures/bundler/torque.txt index 32e6d4c30..01316f3a3 100644 --- a/doc/server/plugins/structures/bundler/torque.txt +++ b/doc/server/plugins/structures/bundler/torque.txt @@ -11,7 +11,7 @@ A longer Bundle that includes many group-specific entries. .. code-block:: xml - <Bundle name='torque' version='1.0'> + <Bundle> <Service name='nfs'/> <Service name='nfslock'/> <BoundPath type='directory' owner='root' group='root' mode='0755' name='/var/spool/torque'/> @@ -29,9 +29,7 @@ A longer Bundle that includes many group-specific entries. <BoundPath type='directory' owner='root' group='root' mode='0755' name='/var/spool/torque/mom_logs'/> <BoundPath type='directory' owner='root' group='root' mode='0755' name='/var/spool/torque/mom_priv'/> <BoundPath type='directory' owner='root' group='root' mode='0755' name='/var/spool/torque/mom_priv/jobs'/> - <Path name='/var/spool/torque/mom_priv/config'/> - <Path name='/var/spool/torque/mom_priv/prologue'/> - <Path name='/var/spool/torque/mom_priv/epilogue'/> + <Path glob='/var/spool/torque/mom_priv/*'/> </Group> <Group name='torque-server'> <Service name='torque_server'/> diff --git a/doc/server/plugins/structures/bundler/yp.txt b/doc/server/plugins/structures/bundler/yp.txt index 6eecb3304..9990fbc2c 100644 --- a/doc/server/plugins/structures/bundler/yp.txt +++ b/doc/server/plugins/structures/bundler/yp.txt @@ -14,7 +14,7 @@ treatment too. .. code-block:: xml - <Bundle name='yp' version='2.0'> + <Bundle> <Package name='yp-tools'/> <Path name='/etc/nsswitch.conf'/> <Path name='/etc/yp.conf'/> diff --git a/doc/server/plugins/version/bzr.txt b/doc/server/plugins/version/bzr.txt index 0755bf80c..ae247985f 100644 --- a/doc/server/plugins/version/bzr.txt +++ b/doc/server/plugins/version/bzr.txt @@ -21,7 +21,7 @@ How to enable the Bazaar plugin Simply add "Bzr" to your plugins line in ``/etc/bcfg2.conf``:: [server] - plugins = Base,Bundler,Cfg,...,Bzr + plugins = Bundler,Cfg,...,Bzr Usage notes =========== diff --git a/doc/server/plugins/version/cvs.txt b/doc/server/plugins/version/cvs.txt index a80b1edbc..f969302d0 100644 --- a/doc/server/plugins/version/cvs.txt +++ b/doc/server/plugins/version/cvs.txt @@ -21,4 +21,4 @@ How to enable the CVS plugin Simply add "Cvs" to your plugins line in ``/etc/bcfg2.conf``:: [server] - plugins = Base,Bundler,Cfg,...,Cvs + plugins = Bundler,Cfg,...,Cvs diff --git a/doc/server/plugins/version/darcs.txt b/doc/server/plugins/version/darcs.txt index 30ac0176a..6fa384679 100644 --- a/doc/server/plugins/version/darcs.txt +++ b/doc/server/plugins/version/darcs.txt @@ -6,7 +6,7 @@ Darcs ===== -This page describes the new Darcs plugin which is experimental. +This page describes the new Darcs plugin which is experimental. Why use the Darcs plugin ======================== @@ -25,4 +25,4 @@ You will need to install Darcs on the Bcfg2 server first. Once installed, simply add Darcs to your plugins line in ``/etc/bcfg2.conf``:: [server] - plugins = Base,Bundler,Cfg,...,Darcs + plugins = Bundler,Cfg,...,Darcs diff --git a/doc/server/plugins/version/fossil.txt b/doc/server/plugins/version/fossil.txt index 7bf523a9e..a19c21760 100644 --- a/doc/server/plugins/version/fossil.txt +++ b/doc/server/plugins/version/fossil.txt @@ -21,4 +21,4 @@ How to enable the Fossil plugin Simply add "Fossil" to your plugins line in ``/etc/bcfg2.conf``:: [server] - plugins = Base,Bundler,Cfg,...,Fossil + plugins = Bundler,Cfg,...,Fossil diff --git a/doc/server/plugins/version/hg.txt b/doc/server/plugins/version/hg.txt index 747699f0e..a11623836 100644 --- a/doc/server/plugins/version/hg.txt +++ b/doc/server/plugins/version/hg.txt @@ -22,4 +22,4 @@ You will need to install Mercurial on the Bcfg2 server first. Simply add Hg to your plugins line in ``/etc/bcfg2.conf``:: [server] - plugins = Base,Bundler,Cfg,...,Hg + plugins = Bundler,Cfg,...,Hg diff --git a/doc/server/snapshots/index.txt b/doc/server/snapshots/index.txt deleted file mode 100644 index a7e5940ed..000000000 --- a/doc/server/snapshots/index.txt +++ /dev/null @@ -1,155 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-snapshots-index: - -=============== -Bcfg2 Snapshots -=============== - -.. versionadded:: 1.0.0 - -This page describes the Snapshots plugin. Snapshots is deprecated, and -will be removed in a future release. - -Before you begin -================ - -Make sure you have version 0.5 or greater of sqlalchemy. - -On CentOS/RHEL 5 ----------------- - -* Download a tarball of SQLAlchemy. -* Extract and build the RPM:: - - tar xzf SQLAlchemy-0.5.6.tar.gz - cd SQLAlchemy-0.5.6 - python setup.py bdist_rpm - -* Copy the RPM in ``SQLAlchemy-0.5.6/dist/`` to your Yum repository, - and rebuild the repository using ``createrepo``. -* Clear the Yum cache:: - - sudo yum clean all - -* Install SQLAlchemy:: - - sudo yum install SQLAlchemy - -* Manage the package in Bcfg2 as you would any other package. - -Configuration -============= - -* A database location needs to be added to ``bcfg2.conf``. Three drivers - are currently supported; mysql, postgres, and sqlite. When using the - sqlite driver, only the driver and database lines are required. - - * For MySQL:: - - [snapshots] - driver = mysql - database = snapshots - user = snapshots - password = snapshots - host = dbserver - - * For SQLite:: - - [snapshots] - driver = sqlite - database = /var/lib/bcfg2/var/snapshots.sqlite - -* The database needs to be initialized.:: - - $ bcfg2-admin snapshots init - 2009-03-22 21:40:24,683 INFO sqlalchemy.engine.base.Engine.0x...3e2c PRAGMA table_info("connkeyval") - PRAGMA table_info("connkeyval") - 2009-03-22 21:40:24,684 INFO sqlalchemy.engine.base.Engine.0x...3e2c () - () - 2009-03-22 21:40:24,686 INFO sqlalchemy.engine.base.Engine.0x...3e2c PRAGMA table_info("package") - PRAGMA table_info("package") - 2009-03-22 21:40:24,687 INFO sqlalchemy.engine.base.Engine.0x...3e2c () - () - ..... - COMMIT - -* The Snapshots plugin needs to be enabled for the bcfg2-server (by adding - Snapshots to the plugins line in ``/etc/bcfg2.conf``). Once done, - this will cause the the server to store statistics information when - clients run. - -Using the reports interface -=========================== - -All hosts:: - - $ bcfg2-admin snapshots reports -a - - ============= ========= ========================================== ============================ - Client Correct Revision Time - ============= ========= ========================================== ============================ - bcfg2client True f46ac7773712bd3c3cfb765ae5d2a3b2a37ac9b7 2009-04-23 11:27:54.378941 - ============= ========= ========================================== ============================ - -List bad entries for a single host:: - - $ bcfg2-admin snapshots reports -b bcfg2client - Bad entries: - Package:nscd - Package:cupsys - File:/etc/ldap.conf - -List extra entries for a single host:: - - $ bcfg2-admin snapshots reports -e bcfg2client - Extra entries: - Package:python-pyxattr - Package:librsync1 - Package:python-pylibacl - Package:gcc-4.2-multilib - Package:nxlibs - Package:freenx-session-launcher - Package:dx-doc - Package:dirdiff - Package:libhdf4g - Package:nxclient - Package:freenx-rdp - Package:freenx-vnc - Package:libxml2-dev - Package:mysql-client - Package:mysql-client-5.0 - Package:libxcompext3 - Package:lib32gomp1 - Package:dx - Package:freenx-media - Package:dxsamples - Package:gcc-multilib - Package:rdiff-backup - Package:libdbd-mysql-perl - Package:libxcomp3 - Package:freenx-server - Package:smbfs - Package:planner - Package:nxagent - Package:libc6-dev-i386 - Package:libfltk1.1-dev - Package:freenx - Package:libdx4 - Package:libxcompshad3 - Service:freenx-server - -Detailed view of hosts for a particular date:: - - $ bcfg2-admin snapshots reports --date 2009 5 30 - ============= ========= ========================================== ============================ - Client Correct Revision Time - ============= ========= ========================================== ============================ - bcfg2client False 10c1a12c62c57c0861cc453b8d2640c4839a7357 2009-05-29 10:52:34.701056 - -TODO/Wishlist -============= - -* Identify per-client changes in correctness over time -* Detailed view for a particular date -* Track entry changes over time (glibc updated on these dates to these versions) diff --git a/doc/server/xml-common.txt b/doc/server/xml-common.txt new file mode 100644 index 000000000..5302a59e4 --- /dev/null +++ b/doc/server/xml-common.txt @@ -0,0 +1,376 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _xml-features: + +===================== + Common XML Features +===================== + +Most of the XML files in Bcfg2 have a common set of features that are +supported. These are described in some detail below, and a precise +rundown of which features are supported by which files is provided. + +.. _xml-group-client-tags: + +Group and Client tags +===================== + +These allow the portions of an XML document inside a Client or Group +tag to only apply to the given client group. That is, they can be +thought of as conditionals, where the following are roughly equivalent: + +.. code-block:: xml + + <Group name="group1"> + <Path name="/etc/foo.conf"/> + </Group> + +And:: + + If client is a member of group1 then + Manage the abstract path "/etc/foo.conf" + +Nested Group and Client tags are conjunctive (logical ``AND``). For +instance, the following are roughly equivalent: + +.. code-block:: xml + + <Group name="group1"> + <Client name="foo.example.com"> + <Package name="bar"/> + </Client> + <Package name="baz"/> + </Group> + +And:: + + If client is a member of group1 and has hostname "foo.example.com" then + Manage the abstract package "bar" + If client is a member of group1 then + Manage the abstract package "baz" + +There is no convenient ``else``; you must specify all conditions +explicitly. To do this, Group and Client tags may be negated, as in: + +.. code-block:: xml + + <Group name="group1"> + <Service name="foo"/> + </Group> + <Group name="group1" negate="true"> + <Service name="bar"/> + </Group> + +This is roughly equivalent to:: + + If client is a member of group1 then + Manage the abstract service "foo" + If client is not a member of group 1 then + Manage the abstract service "bar" + +Or, more compactly: + + If client is a member of group1 then + Manage the abstract service "foo" + Else + Manage the abstract service "bar" + +As an example, consider the following :ref:`bundle +<server-plugins-structures-bundler>`: + +.. code-block:: xml + + <Bundle> + <Path glob='/etc/ssh/*'/> + <Group name='rpm'> + <Package name='openssh'/> + <Package name='openssh-askpass'/> + <Service name='sshd'/> + <Group name='fedora' > + <Group name='fedora14' negate='true'> + <Package name='openssh-clients'/> + </Group> + <Package name='openssh-server'/> + </Group> + </Group> + <Group name='deb'> + <Package name='ssh'/> + <Service name='ssh'/> + </Group> + <Client name='trust.example.com'> + <Path name='/etc/ssh/shosts.equiv'/> + </Client> + </Bundle> + +In this bundle, most of the entries are common to all systems. Clients +in group ``deb`` get one extra package and service, while clients in +group ``rpm`` get two extra packages and an extra service. In +addition, clients in group ``fedora`` *and* group ``rpm`` get one +extra package entries, unless they are not in the ``fedora14`` group, +in which case, they get an extra package. The client +``trust.example.com`` gets one extra file that is not distributed to +any other clients. + ++------------------------+-----------------------------------+ +| Group/Hostname | Entry | ++========================+===================================+ +| all | ``/etc/ssh/*`` | ++------------------------+-----------------------------------+ +| ``rpm`` | Package ``openssh`` | ++------------------------+-----------------------------------+ +| ``rpm`` | Package ``openssh-askpass`` | ++------------------------+-----------------------------------+ +| ``rpm`` | Service ``sshd`` | ++------------------------+-----------------------------------+ +| ``rpm`` AND ``fedora`` | Package ``openssh-server`` | ++------------------------+-----------------------------------+ +| ``rpm`` AND ``fedora`` | Package ``openssh-clients`` | +| AND NOT ``fedora14`` | | ++------------------------+-----------------------------------+ +| ``deb`` | Package ``ssh`` | ++------------------------+-----------------------------------+ +| ``deb`` | Service ``ssh`` | ++------------------------+-----------------------------------+ +| ``trust.example.com`` | ``/etc/ssh/shosts.equiv`` | ++------------------------+-----------------------------------+ + +.. _xml-genshi-templating: + +Genshi templating +================= + +Genshi XML templates allow you to use the `Genshi +<http://genshi.edgewall.org>`_ templating system to dynamically +generate XML file content for a given client. Genshi templating can +be enabled on a file by adding the Genshi namespace to the top-level +tag, e.g.: + +.. code-block:: xml + + <Bundle xmlns:py="http://genshi.edgewall.org/"> + +Several variables are pre-defined inside Genshi XML templates: + ++-------------+--------------------------------------------------------+ +| Name | Description | ++=============+========================================================+ +| metadata | :ref:`Client metadata | +| | <server-plugins-grouping-metadata-clientmetadata>` | ++-------------+--------------------------------------------------------+ +| repo | The path to the Bcfg2 repository on the filesystem | ++-------------+--------------------------------------------------------+ + +.. note:: + + ``<Group>`` and ``<Client>`` tags can be used inside templates as + of Bcfg2 1.2, but they do not behave the same as using a Genshi + conditional, e.g.:: + + <py:if test="'groupname' in metadata.groups"> + </py:if> + + The conditional is evaluated when the template is rendered, so + code inside the conditional is not executed if the conditional + fails. A ``<Group>`` tag is evaluated *after* the template is + rendered, so code inside the tag is always executed. This is an + important distinction: if you have code that will fail on some + groups, you *must* use a Genshi conditional, not a ``<Group>`` + tag. The same caveats apply to ``<Client>`` tags. + +.. _xml-genshi-reference: + +Genshi XML Template Reference +----------------------------- + +The Genshi XML templating language is described in depth at `Genshi +<http://genshi.edgewall.org>`_. The XML schema reference follows. + +Genshi Tags +~~~~~~~~~~~ + +.. xml:group:: genshiElements + :namespace: py + +Genshi Attributes +~~~~~~~~~~~~~~~~~ + +.. xml:attributegroup:: genshiAttrs + :namespace: py + +.. _xml-encryption: + +Encryption +========== + +You can encrypt data in XML files to protect that data from other +people who need access to the repository. The data is decrypted +transparently on-the-fly by the server. + +.. note:: + + This feature is *not* intended to secure the files against a + malicious attacker who has gained access to your Bcfg2 server, as + the encryption passphrases are held in plaintext in + ``bcfg2.conf``. This is only intended to make it easier to use a + single Bcfg2 repository with multiple admins who should not + necessarily have access to each other's sensitive data. + +XML files are encrypted on a per-element basis; that is, rather than +encrypting the whole file, only the character content of individual +elements is encrypted. This makes it easier to track changes to the +file in a VCS, and also lets unprivileged users work with the other +data in the file. Only character content of an element can be +encrypted; attribute content and XML elements themselves cannot be +encrypted. + +By default, decryption is *strict*; that is, if any element cannot be +decrypted, parsing of the file is aborted. See +:ref:`server-encryption-lax-strict` for information on changing this +on a global or per-file basis. + +To encrypt or decrypt a file, use :ref:`bcfg2-crypt`. + +See :ref:`server-encryption` for more details on encryption in Bcfg2 +in general. + +XInclude +======== + +.. versionadded:: 0.9.0 + +`XInclude <http://www.w3.org/TR/xinclude/>`_ is a W3C specification +for the inclusion of external XML documents into XML source files, +allowing complex definitions to be split into smaller, more manageable +pieces. For instance, in the :ref:`server-plugins-grouping-metadata` +``groups.xml`` file, you might do: + +.. code-block:: xml + + <Groups xmlns:xi="http://www.w3.org/2001/XInclude"> + <xi:include href="my-groups.xml" /> + <xi:include href="their-groups.xml" /> + </Groups> + +To enable XInclude on a file, you need only add the XInclude namespace +to the top-level tag. + +You can also *optionally* include a file that may or may not exist +with the ``fallback`` tag: + +.. code-block:: xml + + <Groups xmlns:xi="http://www.w3.org/2001/XInclude"> + <xi:include href="my-groups.xml"/> + <xi:include href="their-groups.xml"><xi:fallback/></xi:include> + </Groups> + +In this case, if ``their-groups.xml`` does not exist, no error will be +raised and everything will work fine. (You can also use ``fallback`` +to include a different file, or explicit content in the case that the +parent include does not exist.) + +XInclude can only include complete, well-formed XML files. In some +cases, it may not be entirely obvious or intuitive how to structure +such an included file to conform to the schema, although in general +the included files should be structure exactly like the parent file. + +Wildcard XInclude +----------------- + +.. versionadded:: 1.3.1 + +Bcfg2 supports an extension to XInclude that allows you to use shell +globbing in the hrefs. (Stock XInclude doesn't support this, since +the href is supposed to be a URL.) + +For instance: + +.. code-block:: xml + + <Groups xmlns:xi="http://www.w3.org/2001/XInclude"> + <xi:include href="groups/*.xml"/> + </Groups> + +This would include all ``*.xml`` files in the ``groups`` subdirectory. + +Note that if a glob finds no files, that is treated the same as if a +single included file does not exist. You should use the ``fallback`` +tag, described above, if a glob may potentially find no files. + +Feature Matrix +============== + ++---------------------------------------------------+--------------+--------+------------+------------+ +| File | Group/Client | Genshi | Encryption | XInclude | ++===================================================+==============+========+============+============+ +| :ref:`ACL ip.xml <server-plugins-misc-acl>` | No | No | No | Yes | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`ACL metadata.xml | Yes | Yes | Yes | Yes | +| <server-plugins-misc-acl>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Bundler | Yes | Yes | Yes | Yes | +| <server-plugins-structures-bundler>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`info.xml <server-info>` | Yes [#f1]_ | Yes | Yes | Yes | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`privkey.xml and pubkey.xml | Yes | Yes | Yes | Yes [#f2]_ | +| <server-plugins-generators-cfg-sshkeys>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`authorizedkeys.xml | Yes | Yes | Yes | Yes | +| <server-plugins-generators-cfg-sshkeys>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`sslcert.xml and sslkey.xml | Yes | Yes | Yes | Yes | +| <server-plugins-generators-cfg-ssl-certificates>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Decisions | Yes | Yes | Yes | Yes | +| <server-plugins-generators-decisions>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Defaults | Yes | Yes | Yes | Yes | +| <server-plugins-structures-defaults>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`FileProbes | Yes | Yes | Yes | Yes | +| <server-plugins-probes-fileprobes>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`GroupPatterns | No | No | No | Yes | +| <server-plugins-grouping-grouppatterns>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Metadata clients.xml | No | No | No | Yes | +| <server-plugins-grouping-metadata-clients-xml>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Metadata groups.xml | Yes [#f3]_ | No | No | Yes | +| <server-plugins-grouping-metadata-groups-xml>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`NagiosGen | Yes | Yes | Yes | Yes | +| <server-plugins-generators-nagiosgen>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Packages | Yes | Yes | Yes | Yes | +| <server-plugins-generators-packages>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Pkgmgr | Yes | No | No | No | +| <server-plugins-generators-pkgmgr>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Properties | Yes [#f4]_ | Yes | Yes | Yes | +| <server-plugins-connectors-properties>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Rules <server-plugins-generators-rules>` | Yes | Yes | Yes | Yes | ++---------------------------------------------------+--------------+--------+------------+------------+ + +.. rubric:: Footnotes + +.. [#f1] ``info.xml`` also supports conditional Path tags; see + :ref:`server-info` for more. +.. [#f2] XInclude is supported, but the schema has not been modified + to allow including files that are structured exactly like the + parent. You may need to read the schema to understand how to + use XInclude properly. +.. [#f3] The semantics of Group tags in ``groups.xml`` is slightly + different; see + :ref:`server-plugins-grouping-metadata-groups-xml` for + details. +.. [#f4] Group and Client tags in XML Properties are not automatic by + default; they can be resolved by use of either the + ``Match()`` or ``XMLMatch()`` methods, or by use of the + :ref:`server-plugins-connectors-properties-automatch` + feature. See :ref:`server-plugins-connectors-properties-xml` + for details. diff --git a/doc/unsorted/howtos.txt b/doc/unsorted/howtos.txt index cef64a394..81b38e54d 100644 --- a/doc/unsorted/howtos.txt +++ b/doc/unsorted/howtos.txt @@ -12,7 +12,7 @@ Here are several howtos that describe different aspects of Bcfg2 deployment * AnnotatedExamples - a description of basic Bcfg2 specification operations * BuildingDebianPackages - How to build debian packages * :ref:`appendix-guides-gentoo` - Issues specific to running Bcfg2 on Gentoo -* :ref:`server-plugins-probes-index` - How to use Probes to gather information from a client machine. +* :ref:`server-plugins-probes` - How to use Probes to gather information from a client machine. * :ref:`client-tools-actions` - How to use Actions * :ref:`server-plugins-probes-dynamic-groups` - Using dynamic groups * :ref:`client-modes-paranoid` - How to run an update in paranoid mode diff --git a/doc/unsorted/index.txt b/doc/unsorted/index.txt index a369ee1b3..74d045990 100644 --- a/doc/unsorted/index.txt +++ b/doc/unsorted/index.txt @@ -13,7 +13,6 @@ list below. .. _TitleIndex: https://trac.mcs.anl.gov/projects/bcfg2/wiki/TitleIndex -* `Plugins/Snapshots` * `PrecompiledPackages` * `SchemaEvolution` * `SecurityDevPlan` diff --git a/doc/unsorted/writing_specification.txt b/doc/unsorted/writing_specification.txt index 700c1ab72..f9dd30a14 100644 --- a/doc/unsorted/writing_specification.txt +++ b/doc/unsorted/writing_specification.txt @@ -26,12 +26,12 @@ that a client needs the Bcfg2 package with .. code-block:: xml - <Package name=bcfg2/> + <Package name="bcfg2"/> but this does not explicitly identify that an RPM package version 0.9.2 should be loaded from http://rpm.repo.server/bcfg2-0.9.2-0.1.rpm. -The abstract configuration is defined in the xml configuration files -for the Base and Bundles plugins. +The abstract configuration is defined in the XML configuration files +for the Bundler plugin. A combination of a clients metadata (group memberships) and abstract configuration is then used to generate the clients literal configuration. @@ -57,35 +57,13 @@ Abstract Configuration (Structures) =================================== A clients Abstract Configuration is the inventory of configuration -entities that should be installed on a client. Two plugins provide the -basis for the abstract configuration, the Bundler and Base. +entities that should be installed on a client. The Bundler plugin +usually provides the abstract configuration. The plugin Bundler builds descriptions of interrelated configuration entities. These are typically used for the representation of services, or other complex groups of entities. -The Base provides a laundry list of configuration entities that need to -be installed on hosts. These entities are independent from one another, -and can be installed individually without worrying about the impact on -other entities. - -Usage of Groups in Base and Bundles ------------------------------------ - -Groups are used by the Base and Bundles plugins for selecting -Configuration Entity Types for inclusion in a clients abstract -configuration. They can be thought of as:: - - if client is a member of group1 then - assign to abstract config - -Nested groups are conjunctive (logical and).:: - - if client is a member of group1 and group2 then - assign to abstract config - -Group membership maybe negated. See "Writing Bundles" for an example. - Configuration Entity Types -------------------------- @@ -121,9 +99,8 @@ consist of If any of these pieces are installed or updated, all should be rechecked and any associated services should be restarted. -All files in the Bundles/ subdirectory of the repository are processed. -Each bundle must be defined in its own file and the filename must be the -same as the bundle name with a .xml suffix.:: +All files in the Bundles/ subdirectory of the repository are +processed. Each bundle must be defined in its own file:: # ls Bundler Glide3.xml @@ -144,17 +121,6 @@ same as the bundle name with a .xml suffix.:: atftp.xml .... -Groups can be used inside of bundles to differentiate which entries -particular clients will receive. This is useful for the case where -entries are named differently across systems; for example, one linux -distro may have a package called openssh while another uses the name ssh. -Configuration entries nested inside of Group elements only apply to -clients who are a member of those groups; multiply nested groups must -all apply. - -Also, groups may be negated; entries included in such groups will only -apply to clients who are not a member of said group. - When packages in a bundle are verified by the client toolset, the Paths included in the same bundle are taken into consideration. That is, a package will not fail verification from a Bcfg2 perspective if the @@ -165,16 +131,8 @@ The following is an annotated copy of a bundle: .. code-block:: xml - <Bundle name='ssh' version='2.0'> - <Path name='/etc/ssh/ssh_host_dsa_key'/> - <Path name='/etc/ssh/ssh_host_rsa_key'/> - <Path name='/etc/ssh/ssh_host_dsa_key.pub'/> - <Path name='/etc/ssh/ssh_host_rsa_key.pub'/> - <Path name='/etc/ssh/ssh_host_key'/> - <Path name='/etc/ssh/ssh_host_key.pub'/> - <Path name='/etc/ssh/sshd_config'/> - <Path name='/etc/ssh/ssh_config'/> - <Path name='/etc/ssh/ssh_known_hosts'/> + <Bundle> + <Path glob='/etc/ssh/*'/> <Group name='rpm'> <Package name='openssh'/> <Package name='openssh-askpass'/> @@ -205,23 +163,7 @@ can be used in bundles) +----------------+-------------------------------+ | Group | Entry | +================+===============================+ -| all | /etc/ssh/ssh_host_dsa_key | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_host_rsa_key | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_host_dsa_key.pub | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_host_rsa_key.pub | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_host_key | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_host_key.pub | -+----------------+-------------------------------+ -| all | /etc/ssh/sshd_config | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_config | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_known_hosts | +| all | /etc/ssh/* | +----------------+-------------------------------+ | rpm | Package openssh | +----------------+-------------------------------+ @@ -268,26 +210,3 @@ A Generator can take care of a particular configuration element. Any time this element is requested by the client, the server dynamically generates it either by crunching data and creating new information or by reading a file off of disk and passes it down to the client for installation. - -Usage of Groups in Generators ------------------------------ - -Similar to Abstract Configuration plugins, groups are used by generator -plugins for selecting Configuration Entities for inclusion in a clients -literal configuration. They can be thought of as:: - - if client is a member of group1 then - assign to abstract config - -Nested groups are conjunctive (logical and).:: - - if client is a member of group1 and group2 then - assign to abstract config - -How the groups are configured is specific to the plugin, but here are -two common methods: - -* xml configuration file (Pkgmgr, Rules) -* file name encoding (Cfg, SSHBase) - -Details are included on each plugin's page. diff --git a/examples/Bundler/dirvish.xml b/examples/Bundler/dirvish.xml new file mode 100644 index 000000000..47bf8cf54 --- /dev/null +++ b/examples/Bundler/dirvish.xml @@ -0,0 +1,20 @@ +<Bundle xmlns:py="http://genshi.edgewall.org/"> + <py:for each="user in metadata.Properties['dirvish.xml'].xdata.find('users')"> + <!-- Generate configs for all users in dirvish.xml --> + <BoundPath + name='/backup/homes/${user.tag}/dirvish/default.conf' + owner='root' + group='root' + perms='0644'>client: nfs-host +tree: /export/homes/${user.text} +exclude: + *~ + .nfs* + </BoundPath> + </py:for> + + <Package name='dirvish'/> + <Path name='/etc/cron.d/dirvish'/> + <Path name='/etc/dirvish/dirvish-cronjob'/> + <Path name='/etc/dirvish/master.conf'/> +</Bundle> diff --git a/examples/Bundler/sgenshi-dirvish.genshi b/examples/Bundler/sgenshi-dirvish.genshi deleted file mode 100644 index b4ea08f2c..000000000 --- a/examples/Bundler/sgenshi-dirvish.genshi +++ /dev/null @@ -1,23 +0,0 @@ -<!-- -vim: ft=xml ---> -<Bundle name='sgenshi-dirvish' xmlns:py="http://genshi.edgewall.org/"> -<py:for each="user in metadata.Properties['dirvish.xml'].xdata.find('users')"> -<!-- Generate configs for all users in dirvish.xml --> -<BoundPath - name='/backup/homes/${user.tag}/dirvish/default.conf' - owner='root' - group='root' - perms='0644'>client: nfs-host -tree: /export/homes/${user.text} -exclude: - *~ - .nfs* -</BoundPath> -</py:for> - -<Package name='dirvish'/> -<Path name='/etc/cron.d/dirvish'/> -<Path name='/etc/dirvish/dirvish-cronjob'/> -<Path name='/etc/dirvish/master.conf'/> -</Bundle> diff --git a/examples/TGenshi/etc/dirvish/master.conf/template.newtxt b/examples/Cfg/etc/dirvish/master.conf/master.conf.genshi index 6c9750f9d..6c9750f9d 100644 --- a/examples/TGenshi/etc/dirvish/master.conf/template.newtxt +++ b/examples/Cfg/etc/dirvish/master.conf/master.conf.genshi diff --git a/examples/TGenshi/etc/motd/template.newtxt b/examples/TGenshi/etc/motd/template.newtxt deleted file mode 100644 index ca3cc5f18..000000000 --- a/examples/TGenshi/etc/motd/template.newtxt +++ /dev/null @@ -1,31 +0,0 @@ -------------------------------------------------------------------------
- GOALS FOR SERVER MANAGED BY BCFG2
-------------------------------------------------------------------------
-Hostname is ${metadata.hostname}
-
-Groups:
-{% for group in metadata.groups %}\
- * ${group}
-{% end %}\
-
-{% if metadata.categories %}\
-Categories:
-{% for category in metadata.categories %}\
- * ${category}
-{% end %}\
-{% end %}\
-
-
-{% if metadata.Probes %}\
-Probes:
-{% for probe, value in metadata.Probes.iteritems() %}\
- * ${probe} \
- ${value}
-{% end %}\
-{% end %}\
-
-------------------------------------------------------------------------
- ITOPS MOTD
-------------------------------------------------------------------------
-Please create a Ticket for any system level changes you need from IT.
-
diff --git a/examples/TGenshi/tmp/bar/template.txt b/examples/TGenshi/tmp/bar/template.txt deleted file mode 100644 index dbf482c22..000000000 --- a/examples/TGenshi/tmp/bar/template.txt +++ /dev/null @@ -1,19 +0,0 @@ -[communication] -protocol = xmlrpc/ssl -#if metadata.uuid != None -user = $metadata.uuid -#end -#choose -#when metadata.password is not None -password = $metadata.password -#end -#when metadata.password is None -password = GlobalPassword -#end -#end - -[client] -drivers = Action,Chkconfig,POSIX,YUMng - -[components] -bcfg2 = https://config.example.com:6789 diff --git a/examples/TGenshi/tmp/foo/template.xml b/examples/TGenshi/tmp/foo/template.xml deleted file mode 100644 index 522c6e7fe..000000000 --- a/examples/TGenshi/tmp/foo/template.xml +++ /dev/null @@ -1,46 +0,0 @@ -<html xmlns:py="http://genshi.edgewall.org/"> - <head> - <title>${name}</title> - </head> - <body> - <table> - <tr><th>Name:</th><td>${name}</td></tr> - <tr><th>Hostname:</th><td>${metadata.hostname}</td></tr> - <tr><th>Toolset:</th><td>${metadata.hostname}</td></tr> - <tr><th>UUID:</th><td>${metadata.uuid}</td></tr> - <tr><th>Password:</th><td>${metadata.password}</td></tr> - <tr> - <th>Bundles:</th> - <td> - <table> - <tr py:for="bundle in metadata.bundles"><td>${bundle}</td></tr> - </table> - </td> - </tr> - <tr> - <th>Groups:</th> - <td> - <table> - <tr py:for="group in metadata.groups"><td>${group}</td></tr> - </table> - </td> - </tr> - <tr> - <th>Categories:</th> - <td> - <table> - <tr py:for="category in metadata.categories"><td>${category}</td></tr> - </table> - </td> - </tr> - <tr> - <th>Probes:</th> - <td> - <table> - <tr py:for="probe in metadata.probes"><td>${probe}</td><td>${metadata.probes[probe]}</td></tr> - </table> - </td> - </tr> - </table> - </body> -</html> diff --git a/examples/TemplateHelper/include.py b/examples/TemplateHelper/include.py index be0034f52..fcdb42ac9 100644 --- a/examples/TemplateHelper/include.py +++ b/examples/TemplateHelper/include.py @@ -5,14 +5,13 @@ Synopsis: {% python import os - include = metadata.TemplateHelper['include'] - custom = include.IncludeHelper(metadata, path).files(os.path.basename(name)) + custom = IncludeHelper(metadata, path).files(os.path.basename(name)) %}\ {% for file in custom %}\ - ########## Start ${include.describe_specificity(file)} ########## + ########## Start ${describe_specificity(file)} ########## {% include ${file} %} - ########## End ${include.describe_specificity(file)} ########## + ########## End ${describe_specificity(file)} ########## {% end %}\ This would let you include files with the same base name; e.g. in a @@ -20,7 +19,7 @@ template for ''foo.conf'', the include files would be called ''foo.conf.G_<group>.genshi_include''. If a template needs to include different files in different places, you can do that like so: - inc = metadata.TemplateHelper['include'].IncludeHelper(metadata, path) + inc = IncludeHelper(metadata, path) custom_bar = inc.files("bar") custom_baz = inc.files("baz") @@ -33,7 +32,7 @@ from ''baz.conf.G_<group>.genshi_include''. import os import re -__export__ = ["IncludeHelper", "get_specificity", "describe_specificity"] +__default__ = ["IncludeHelper", "get_specificity", "describe_specificity"] class IncludeHelper(object): diff --git a/examples/bcfg2.conf b/examples/bcfg2.conf index cac424576..1f0984f0e 100644 --- a/examples/bcfg2.conf +++ b/examples/bcfg2.conf @@ -1,5 +1,4 @@ [communication] -protocol = xmlrpc/ssl password = foobat # certificate = /etc/bcfg2.key # key = /etc/bcfg2.key diff --git a/examples/bcfg2.confHostbase b/examples/bcfg2.confHostbase deleted file mode 100644 index c9420e34a..000000000 --- a/examples/bcfg2.confHostbase +++ /dev/null @@ -1,33 +0,0 @@ -[server] -repository = /var/lib/bcfg2 -plugins = Bundler,Rules,Metadata,SSHbase,Cfg - -[statistics] -sendmailpath = /usr/sbin/sendmail - -[communication] -protocol = xmlrpc/ssl -password = foobat -key = /etc/bcfg2.key - -[components] -bcfg2 = https://localhost:6789 - -[hostbase] -# postgresql, mysql, sqlite3 or ado_mssql -database_engine = mysql -# Or path to database file if using sqlite3. -database_name = -# Not used with sqlite3. -database_user = -# Not used with sqlite3. -database_password = -# Set to empty string for localhost. Not used with sqlite3. -database_host = -# Set to empty string for default. Not used with sqlite3. -database_port = 3306 -# enter an NIS group name you'd like to give access to edit hostbase records -##authorized_group = support -# default mx record for new hosts added to the database -default_mx = mailserver.yourdomain.net -priority = 30 diff --git a/man/bcfg2-admin.8 b/man/bcfg2-admin.8 index 06cbeec0b..adc335f9b 100644 --- a/man/bcfg2-admin.8 +++ b/man/bcfg2-admin.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-ADMIN" "8" "March 18, 2013" "1.3" "Bcfg2" +.TH "BCFG2-ADMIN" "8" "April 06, 2014" "1.3" "Bcfg2" .SH NAME bcfg2-admin \- Perform repository administration tasks . @@ -78,10 +78,6 @@ Use \(aqpassword\(aq for client communication. .B backup Create an archive of the entire Bcfg2 repository. .TP -.B bundle \fIaction\fP -Display details about the available bundles (See BUNDLE OPTIONS -below). -.TP .B client \fIaction\fP \fIclient\fP [attribute=value] Add, edit, or remove clients entries in metadata (See CLIENT OPTIONS below). @@ -91,9 +87,15 @@ Compare two client configurations. Can be used to verify consistent behavior between releases. Determine differences between files or directories (See COMPARE OPTIONS below). .TP +.B dbshell +Call the Django \(aqdbshell\(aq command on the configured database. +.TP .B init Initialize a new repository (interactive). .TP +.B initreports +Initialize the Reporting database. +.TP .B minestruct \fIclient\fP [\-f xml\-file] [\-g groups] Build structure entries based on client statistics extra entries (See MINESTRUCT OPTIONS below). @@ -105,12 +107,21 @@ Query server for performance data. Install configuration information into repo based on client bad entries (See PULL OPTIONS below). .TP -.B reports [init|load_stats|purge|scrub|update] -Interact with the dynamic reporting system (See REPORTS OPTIONS -below). +.B purgereports +Purge historic and expired data from the Reporting database +.TP +.B reportssqlall +Call the Django \(aqshell\(aq command on the Reporting database. .TP -.B snapshots [init|dump|query|reports] -Interact with the Snapshots database (See SNAPSHOTS OPTIONS below). +.B reportsstats +Print Reporting database statistics. +.TP +.B scrubreports +Scrub the Reporting database for duplicate reasons and orphaned +entries. +.TP +.B shell +Call the Django \(aqshell\(aq command on the configured database. .TP .B syncdb Sync the Django ORM with the configured database. @@ -118,6 +129,12 @@ Sync the Django ORM with the configured database. .B tidy Remove unused files from repository. .TP +.B updatereports +Apply database schema updates to the Reporting database. +.TP +.B validatedb +Call the Django \(aqvalidate\(aq command on the configured database. +.TP .B viz [\-H] [\-b] [\-k] [\-o png\-file] Create a graphviz diagram of client, group and bundle information (See VIZ OPTIONS below). @@ -125,23 +142,6 @@ Create a graphviz diagram of client, group and bundle information .B xcmd Provides a XML\-RPC Command Interface to the bcfg2\-server. .UNINDENT -.SS BUNDLE OPTIONS -.INDENT 0.0 -.TP -.B mode -One of the following. -.INDENT 7.0 -.TP -.B \fIlist\-xml\fP -List all available xml bundles -.TP -.B \fIlist\-genshi\fP -List all available genshi bundles -.TP -.B \fIshow\fP -Interactive dialog to get details about the available bundles -.UNINDENT -.UNINDENT .SS CLIENT OPTIONS .INDENT 0.0 .TP @@ -170,11 +170,24 @@ are \(aqprofile\(aq, \(aquuid\(aq, \(aqpassword\(aq, \(aqlocation\(aq, \(aqsecur .SS COMPARE OPTIONS .INDENT 0.0 .TP +.B \-d \fIN\fP, \-\-diff\-lines \fIN\fP +Show only N lines of a diff +.UNINDENT +.INDENT 0.0 +.TP +.B \-c, \-\-color +Show colors even if not ryn from a TTY +.TP +.B \-q, \-\-quiet +Only show that entries differ, not how they differ +.UNINDENT +.INDENT 0.0 +.TP .B old -Specify the location of the old configuration file. +Specify the location of the old configuration(s). .TP .B new -Specify the location of the new configuration file. +Specify the location of the new configuration(s). .UNINDENT .SS MINESTRUCT OPTIONS .INDENT 0.0 @@ -200,51 +213,30 @@ Specify the type of the entry to pull. .B entry name Specify the name of the entry to pull. .UNINDENT -.SS REPORTS OPTIONS -.INDENT 0.0 -.TP -.B load_stats [\-s] [\-c] [\-03] -Load statistics data. -.TP -.B purge [\-\-client [n]] [\-\-days [n]] [\-\-expired] -Purge historic and expired data. -.TP -.B scrub -Scrub the database for duplicate reasons and orphaned entries. -.TP -.B update -Apply any updates to the reporting database. -.UNINDENT -.SS SNAPSHOTS OPTIONS -.INDENT 0.0 -.TP -.B init -Initialize the snapshots database. -.TP -.B query -Query the snapshots database. -.TP -.B dump -Dump some of the contents of the snapshots database. -.TP -.B reports [\-a] [\-b] [\-e] [\-\-date=MM\-DD\-YYYY] -Generate reports for clients in the snapshots database. -.UNINDENT .SS VIZ OPTIONS .INDENT 0.0 .TP -.B \-H +.B \-H, \-\-includehosts Include hosts in diagram. .TP -.B \-b +.B \-b, \-\-includebundles Include bundles in diagram. +.UNINDENT +.INDENT 0.0 .TP -.BI \-o \ <outfile> +.B \-o \fIoutfile\fP, \-\-outfile \fIoutfile\fP Write to outfile file instead of stdout. +.UNINDENT +.INDENT 0.0 .TP -.B \-k +.B \-k, \-\-includekey Add a shape/color key. .UNINDENT +.INDENT 0.0 +.TP +.B \-c \fIhostname\fP, \-\-only\-client \fIhostname\fP +Only show groups and bundles for the named client +.UNINDENT .SH SEE ALSO .sp \fIbcfg2\-info(8)\fP, \fIbcfg2\-server(8)\fP diff --git a/man/bcfg2-server.8 b/man/bcfg2-server.8 index dcec03252..60fe58a30 100644 --- a/man/bcfg2-server.8 +++ b/man/bcfg2-server.8 @@ -34,7 +34,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .sp \fBbcfg2\-server\fP [\-d] [\-v] [\-C \fIconfigfile\fP] [\-D \fIpidfile\fP] [\-E \fIencoding\fP] [\-Q \fIrepo path\fP] [\-S \fIserver url\fP] [\-o \fIlogfile\fP] [\-x -\fIpassword\fP] [\-\-ssl\-key=\fIssl key\fP] +\fIpassword\fP] [\-\-ssl\-key=\fIssl key\fP] [\-\-no\-fam\-blocking] .SH DESCRIPTION .sp \fBbcfg2\-server\fP is the daemon component of Bcfg2 which serves @@ -70,9 +70,12 @@ Print usage information. .TP .BI \-\-ssl\-key\fB= key Specify the path to the SSL key. +.TP +.BI \-\-no\-fam\-blocking +Synonym for fam_blocking = False in bcfg2.conf .UNINDENT .SH SEE ALSO .sp -\fIbcfg2(1)\fP, \fIbcfg2\-lint(8)\fP +\fIbcfg2(1)\fP, \fIbcfg2\-lint(8)\fP, \fIbcfg2.conf(5)\fP .\" Generated by docutils manpage writer. . diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index c503d6421..234a6c030 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,5 +1,3 @@ -.\" Man page generated from reStructuredText. -. .TH "BCFG2.CONF" "5" "November 04, 2014" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 @@ -69,7 +67,6 @@ valid: .ft C inotify gamin -fam pseudo .ft P .fi @@ -80,7 +77,7 @@ pseudo Whether the server should block at startup until the file monitor backend has processed all events. This can cause a slower startup, but ensure that all files are recognized before the first client -is handled. +is handled. Defaults to True. .TP .B ignore_files A comma\-separated list of globs that should be ignored by the file @@ -118,24 +115,22 @@ available plugins are: .sp .nf .ft C -Account -Base +ACL Bundler Bzr Cfg Cvs Darcs -DBStats Decisions +Defaults Deps -Editor FileProbes Fossil Git +GroupLogic GroupPatterns Guppy Hg -Hostbase Ldap Metadata NagiosGen @@ -150,14 +145,9 @@ Reporting Rules SEModules ServiceCompat -Snapshots SSHbase -SSLCA -Statistics Svn -TCheetah TemplateHelper -TGenshi Trigger .ft P .fi @@ -210,28 +200,9 @@ The umask to set for the server. Default is \fI0077\fP\&. .sp This section has a listing of all the plugins currently provided with Bcfg2. -.SS Account Plugin +.SS ACL Plugin .sp -The account plugin manages authentication data, including the following. -.INDENT 0.0 -.IP \(bu 2 -\fB/etc/passwd\fP -.IP \(bu 2 -\fB/etc/group\fP -.IP \(bu 2 -\fB/etc/security/limits.conf\fP -.IP \(bu 2 -\fB/etc/sudoers\fP -.IP \(bu 2 -\fB/root/.ssh/authorized_keys\fP -.UNINDENT -.SS Base Plugin -.sp -The Base plugin is a structure plugin that provides the ability -to add lists of unrelated entries into client configuration entry -inventories. Base works much like Bundler in its file format. This -structure plugin is good for the pile of independent configs needed for -most actual systems. +The ACL plugin controls which hosts can make which XML\-RPC calls. .SS Bundler Plugin .sp The Bundler plugin is used to describe groups of inter\-dependent @@ -251,19 +222,16 @@ The Cfg plugin provides a repository to describe configuration file contents for clients. In its simplest form, the Cfg repository is just a directory tree modeled off of the directory tree on your client machines. -.SS Cvs Plugin (experimental) +.SS Cvs Plugin .sp The Cvs plugin allows you to track changes to your Bcfg2 repository using a Concurrent version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -.SS Darcs Plugin (experimental) +.SS Darcs Plugin .sp The Darcs plugin allows you to track changes to your Bcfg2 repository using a Darcs version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -.SS DBStats Plugin -.sp -Direct to database statistics plugin. .SS Decisions Plugin .sp The Decisions plugin has support for a centralized set of per\-entry @@ -281,11 +249,6 @@ and only populate attributes that are not yet set. .sp The Deps plugin allows you to make a series of assertions like "Package X requires Package Y (and optionally also Package Z etc.)" -.SS Editor Plugin -.sp -The Editor plugin attempts to allow you to partially manage -configuration for a file. Its use is not recommended and not well -documented. .SS FileProbes Plugin .sp The FileProbes plugin allows you to probe a client for a file, which is @@ -302,6 +265,10 @@ get revision information out of your repository for reporting purposes. The Git plugin allows you to track changes to your Bcfg2 repository using a Git version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. +.SS GroupLogic Plugin +.sp +The GroupLogic plugin lets you flexibly assign group membership with a +Genshi template. .SS GroupPatterns Plugin .sp The GroupPatterns plugin is a connector that can assign clients group @@ -310,14 +277,11 @@ membership based on patterns in client hostnames. .sp The Guppy plugin is used to trace memory leaks within the bcfg2\-server process using Guppy. -.SS Hg Plugin (experimental) +.SS Hg Plugin .sp The Hg plugin allows you to track changes to your Bcfg2 repository using a Mercurial version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -.SS Hostbase Plugin -.sp -The Hostbase plugin is an IP management system built on top of Bcfg2. .SS Ldap Plugin .sp The Ldap plugin makes it possible to fetch data from an LDAP directory, @@ -330,7 +294,7 @@ metadata. .sp The NagiosGen plugin dynamically generates Nagios configuration files based on Bcfg2 data. -.SS Ohai Plugin (experimental) +.SS Ohai Plugin .sp The Ohai plugin is used to detect information about the client operating system. The data is reported back to the server using JSON. @@ -370,10 +334,10 @@ The Reporting plugin enables the collection of data for use with Bcfg2\(aqs dynamic reporting system. .SS Rules Plugin .sp -The Rules plugin provides literal configuration entries that resolve the -abstract configuration entries normally found in the Bundler and Base -plugins. The literal entries in Rules are suitable for consumption by -the appropriate client drivers. +The Rules plugin provides literal configuration entries that resolve +the abstract configuration entries normally found in Bundler. The +literal entries in Rules are suitable for consumption by the +appropriate client drivers. .SS SEModules Plugin .sp The SEModules plugin provides a way to distribute SELinux modules via @@ -381,37 +345,16 @@ Bcfg2. .SS ServiceCompat Plugin .sp The ServiceCompat plugin converts service entries for older clients. -.SS Snapshots Plugin -.sp -The Snapshots plugin stores various aspects of a client’s state when the -client checks in to the server. .SS SSHbase Plugin .sp The SSHbase generator plugin manages ssh host keys (both v1 and v2) for hosts. It also manages the ssh_known_hosts file. It can integrate host keys from other management domains and similarly export its keys. -.SS SSLCA Plugin -.sp -The SSLCA plugin is designed to handle creation of SSL privatekeys and -certificates on request. -.SS Statistics -.sp -The Statistics plugin is deprecated (see Reporting). .SS Svn Plugin .sp The Svn plugin allows you to track changes to your Bcfg2 repository using a Subversion backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -.SS TCheetah Plugin -.sp -The TCheetah plugin allows you to use the cheetah templating system to -create files. It also allows you to include the results of probes -executed on the client in the created files. -.SS TGenshi Plugin -.sp -The TGenshi plugin allows you to use the Genshi templating system to -create files. It also allows you to include the results of probes -executed on the client in the created files. .SS Trigger Plugin .sp The Trigger plugin provides a method for calling external scripts when @@ -503,7 +446,7 @@ the password clients need to use to communicate. On a client, sets the password to use to connect to the server. .TP .B protocol -Communication protocol to use. Defaults to xmlrpc/ssl. +Communication protocol to use. Defaults to xmlrpc/tlsv1. .TP .B retries A client\-only option. Number of times to retry network @@ -609,6 +552,10 @@ The following options are specified in the \fB[packages]\fP section. .INDENT 3.5 .INDENT 0.0 .TP +.B backends +Comma separated list of backends for the dependency resolution. +Default is "Yum,Apt,Pac,Pkgng". +.TP .B resolver Enable dependency resolution. Default is 1 (true). .TP @@ -692,28 +639,11 @@ copies will be kept. .UNINDENT .UNINDENT .UNINDENT -.SH SNAPSHOTS OPTIONS -.sp -Specified in the \fB[snapshots]\fP section. These options control the -server snapshots functionality. -.INDENT 0.0 -.INDENT 3.5 -.INDENT 0.0 -.TP -.B driver -sqlite -.TP -.B database -The name of the database to use for statistics data. -.sp -e.g.: \fB$REPOSITORY_DIR/etc/bcfg2.sqlite\fP -.UNINDENT -.UNINDENT -.UNINDENT -.SH SSLCA OPTIONS +.SH SSL CA OPTIONS .sp -These options are necessary to configure the SSLCA plugin and can be -found in the \fB[sslca_default]\fP section of the configuration file. +These options are necessary to configure the SSL CA feature of the Cfg +plugin and can be found in the \fB[sslca_default]\fP section of the +configuration file. .INDENT 0.0 .INDENT 3.5 .INDENT 0.0 @@ -745,7 +675,7 @@ control the database connection of the server. .INDENT 0.0 .TP .B engine -The database engine used by the statistics module. One of the +The database engine used by server plugins. One of the following: .INDENT 7.0 .INDENT 3.5 @@ -762,9 +692,9 @@ ado_mssql .UNINDENT .TP .B name -The name of the database to use for statistics data. If +The name of the database to use for server data. If \(aqdatabase_engine\(aq is set to \(aqsqlite3\(aq this is a file path to -the sqlite file and defaults to \fB$REPOSITORY_DIR/etc/brpt.sqlite\fP\&. +the sqlite file and defaults to \fB$REPOSITORY_DIR/etc/bcfg2.sqlite\fP. .TP .B user User for database connections. Not used for sqlite3. @@ -781,6 +711,48 @@ Port for database connections. Not used for sqlite3. .B options Various options for the database connection. The value expected is the literal value of the django OPTIONS setting. +.TP +.B reporting_engine +The database engine used by the Reporting plugin. One of the +following: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C + postgresql + mysql + sqlite3 + ado_mssql + +If reporting_engine is not specified, the Reporting plugin uses +the same database as the other server plugins. +.ft P +.fi +.UNINDENT +.UNINDENT +.TP +.B reporting_name +The name of the database to use for reporting data. If +\(aqdatabase_engine\(aq is set to \(aqsqlite3\(aq this is a file path to +the sqlite file and defaults to +\fB$REPOSITORY_DIR/etc/reporting.sqlite\fP. +.TP +.B reporting_user +User for reporting database connections. Not used for sqlite3. +.TP +.B reporting_password +Password for reporting database connections. Not used for sqlite3. +.TP +.B reporting_host +Host for reporting database connections. Not used for sqlite3. +.TP +.B reporting_port +Port for reporting database connections. Not used for sqlite3. +.TP +.B reporting_options +Various options for the database connection. The value expected +is the literal value of the django OPTIONS setting. .UNINDENT .UNINDENT .UNINDENT @@ -800,6 +772,10 @@ time zone as well). .TP .B web_debug Turn on Django debugging. +.TP +.B max_children +Maximum number of children for the reporting collector. Use 0 to +disable the limit. (default is 0) .UNINDENT .UNINDENT .UNINDENT diff --git a/misc/bcfg2-selinux.spec b/misc/bcfg2-selinux.spec index c926a1d74..e3701bbdd 100644 --- a/misc/bcfg2-selinux.spec +++ b/misc/bcfg2-selinux.spec @@ -12,11 +12,11 @@ # # Don't forget to change the Release: tag below to something like 0.1 #%%global _rc 1 -#%%global _pre 2 +%global _pre pre1 %global _pre_rc %{?_pre:.pre%{_pre}}%{?_rc:.rc%{_rc}} Name: bcfg2-selinux -Version: 1.3.5 +Version: 1.4.0 Release: 1%{?_pre_rc}%{?dist} Summary: Bcfg2 Client and Server SELinux policy @@ -33,7 +33,7 @@ Conflicts: selinux-policy = 3.11.1 License: BSD URL: http://bcfg2.org Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}%{?_pre_rc}.tar.gz -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildRoot: %{_tmppath}/%{name}-%{version}%{?_pre_rc}-%{release}-root-%(%{__id_u} -n) BuildArch: noarch BuildRequires: checkpolicy, selinux-policy-devel, hardlink diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index c2d3c3c90..357afed4e 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -19,8 +19,10 @@ # # Don't forget to change the Release: tag below to something like 0.1 #%%global _rc 1 -#%%global _pre 2 -%global _pre_rc %{?_pre:.pre%{_pre}}%{?_rc:.rc%{_rc}} +%global _pre pre1 +%global _nightly 1 +%global _date %(date +%Y%m%d) +%global _pre_rc %{?_pre:pre%{_pre}}%{?_rc:rc%{_rc}} # cherrypy 3.3 actually doesn't exist yet, but 3.2 has bugs that # prevent it from working: @@ -29,8 +31,8 @@ Name: bcfg2 -Version: 1.3.5 -Release: 1%{?_pre_rc}%{?dist} +Version: 1.4.0 +Release: 0.1.%{?_nightly:nightly.%{_date}}%{?_pre_rc}%{?dist} Summary: A configuration management system %if 0%{?suse_version} @@ -46,7 +48,7 @@ Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}%{?_pre_rc}.t Source1: http://www.w3.org/2001/XMLSchema.xsd %if %{?rhel}%{!?rhel:10} <= 5 || 0%{?suse_version} # EL5 and OpenSUSE require the BuildRoot tag -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildRoot: %{_tmppath}/%{name}-%{version}%{?_pre_rc}-%{release}-root-%(%{__id_u} -n) %endif BuildArch: noarch @@ -76,7 +78,6 @@ BuildRequires: buildsys-macros BuildRequires: python-ssl %else # rhel > 5 # EL5 lacks python-mock, so test suite is disabled -BuildRequires: python-sqlalchemy BuildRequires: python-nose BuildRequires: mock BuildRequires: m2crypto @@ -257,10 +258,8 @@ Group: System Environment/Daemons Requires: bcfg2 = %{version}-%{release} Requires: bcfg2-server = %{version}-%{release} -# cherrypy 3.3 actually doesn't exist yet, but 3.2 has bugs that -# prevent it from working: -# https://bitbucket.org/cherrypy/cherrypy/issue/1154/assertionerror-in-recv-when-ssl-is-enabled -Requires: python-cherrypy > 3.3 +# https://bitbucket.org/cherrypy/cherrypy/issue/1068/file-upload-crashes-when-using-https +Requires: python-cherrypy >= 3.2.6 %description server-cherrypy Bcfg2 helps system administrators produce a consistent, reproducible, @@ -295,6 +294,8 @@ This package includes the Bcfg2 CherryPy server backend. %package web Summary: Bcfg2 Web Reporting Interface +Requires: bcfg2-server = %{version}-%{release} +Requires: httpd %if 0%{?suse_version} Group: System/Management Requires: python-django >= 1.2 @@ -310,7 +311,6 @@ Requires: Django-south >= 0.7 %endif Requires: bcfg2-server %endif -Requires: httpd %if "%{_vendor}" == "redhat" Requires: mod_wsgi %global apache_conf %{_sysconfdir}/httpd @@ -443,7 +443,7 @@ awk ' # Get rid of extraneous shebangs for f in `find src/lib -name \*.py` do - sed -i -e '/^#!/,1d' $f + %{__sed} -i -e '/^#!/,1d' $f done sed -i "s/apache2/httpd/g" misc/apache/bcfg2.conf @@ -559,7 +559,7 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %if 0%{?suse_version} %fillup_and_insserv -f bcfg2 %else - /sbin/chkconfig --add bcfg2 + /sbin/chkconfig --add bcfg2 %endif fi %endif @@ -573,7 +573,7 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %if 0%{?suse_version} %fillup_and_insserv -f bcfg2-server %else - /sbin/chkconfig --add bcfg2-server + /sbin/chkconfig --add bcfg2-server %endif fi %endif @@ -688,8 +688,7 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %{python_sitelib}/Bcfg2/Client %{python_sitelib}/Bcfg2/Compat.py* %{python_sitelib}/Bcfg2/Logger.py* -%{python_sitelib}/Bcfg2/Options.py* -%{python_sitelib}/Bcfg2/Proxy.py* +%{python_sitelib}/Bcfg2/Options %{python_sitelib}/Bcfg2/Utils.py* %{python_sitelib}/Bcfg2/version.py* %if 0%{?suse_version} @@ -711,11 +710,7 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %config(noreplace) %{_sysconfdir}/sysconfig/bcfg2-server %{_sbindir}/bcfg2-* %dir %{_localstatedir}/lib/%{name} -%{python_sitelib}/Bcfg2/Cache.py* -%{python_sitelib}/Bcfg2/Encryption.py* -%{python_sitelib}/Bcfg2/SSLServer.py* -%{python_sitelib}/Bcfg2/Statistics.py* -%{python_sitelib}/Bcfg2/settings.py* +%{python_sitelib}/Bcfg2/DBSettings.py* %{python_sitelib}/Bcfg2/Server %{python_sitelib}/Bcfg2/Reporting %{python_sitelib}/Bcfg2/manage.py* @@ -724,7 +719,6 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %dir %{_datadir}/bcfg2 %{_datadir}/bcfg2/schemas %{_datadir}/bcfg2/xsl-transforms -%{_datadir}/bcfg2/Hostbase %if 0%{?suse_version} %{_sbindir}/rcbcfg2-server %config(noreplace) /var/adm/fillup-templates/sysconfig.bcfg2-server diff --git a/osx/Makefile b/osx/Makefile index ee7b7d565..174eb53c8 100644 --- a/osx/Makefile +++ b/osx/Makefile @@ -5,8 +5,7 @@ PREFLIGHT = preflight POSTFLIGHT = postflight PKGROOT = bcfg2pkg PKGTMP = bcfg2tmp -FILTERS = --filter Hostbase \ ---filter Reports \ +FILTERS = --filter Reports \ --filter Server \ --filter xsd \ --filter xsl \ @@ -29,9 +28,9 @@ SITELIBDIR = /Library/Python/${PYVERSION}/site-packages # an Info.plist file for packagemaker to look at for package creation # and substitute the version strings. Major/Minor versions can only be # integers (e.g. "1" and "00" for bcfg2 version 1.0.0. -BCFGVER = 1.3.5 +BCFGVER = 1.4.0pre1 MAJOR = 1 -MINOR = 35 +MINOR = 40 default: clean client diff --git a/osx/bcfg2.conf b/osx/bcfg2.conf index cac424576..1f0984f0e 100644 --- a/osx/bcfg2.conf +++ b/osx/bcfg2.conf @@ -1,5 +1,4 @@ [communication] -protocol = xmlrpc/ssl password = foobat # certificate = /etc/bcfg2.key # key = /etc/bcfg2.key diff --git a/osx/macports/Portfile b/osx/macports/Portfile index cf6f33ddc..05ce40bf0 100644 --- a/osx/macports/Portfile +++ b/osx/macports/Portfile @@ -5,7 +5,7 @@ PortSystem 1.0 PortGroup python26 1.0 name bcfg2 -version 1.3.5 +version 1.4.0pre1 categories sysutils python maintainers gmail.com:sol.jerome license BSD diff --git a/osx/macports/files/patch-setup.py.diff b/osx/macports/files/patch-setup.py.diff index f78d27e5c..01b21eadf 100644 --- a/osx/macports/files/patch-setup.py.diff +++ b/osx/macports/files/patch-setup.py.diff @@ -1,6 +1,6 @@ --- setup.py 2010-11-15 15:30:28.000000000 -0600 +++ setup.py.macports 2010-11-18 19:06:49.155292524 -0600 -@@ -11,47 +11,22 @@ +@@ -11,38 +11,21 @@ setup(cmdclass=cmdclass, name="Bcfg2", version="1.1.1", @@ -14,16 +14,12 @@ "Bcfg2.Client.Tools", - 'Bcfg2.Server', - "Bcfg2.Server.Admin", -- "Bcfg2.Server.Hostbase", -- "Bcfg2.Server.Hostbase.hostbase", - "Bcfg2.Server.Plugins", - "Bcfg2.Server.Reports", - "Bcfg2.Server.Reports.reports", - "Bcfg2.Server.Reports.reports.templatetags", -- "Bcfg2.Server.Snapshots", ], + py_modules = ["Bcfg2.Options", -+ "Bcfg2.Proxy", + "Bcfg2.Logger", + ], package_dir = {'Bcfg2':'src/lib'}, @@ -51,11 +47,5 @@ - glob('src/lib/Server/Reports/reports/templates/clients/*')), - ('share/bcfg2/Reports/templates/config_items', - glob('src/lib/Server/Reports/reports/templates/config_items/*')), -- ('share/bcfg2/Hostbase/templates', -- glob('src/lib/Server/Hostbase/hostbase/webtemplates/*.*')), -- ('share/bcfg2/Hostbase/templates/hostbase', -- glob('src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/*')), -- ('share/bcfg2/Hostbase/repo', -- glob('src/lib/Server/Hostbase/templates/*')), ] ) diff --git a/redhat/scripts/bcfg2-server.init b/redhat/scripts/bcfg2-server.init index 7fd1bd906..f14938a16 100755 --- a/redhat/scripts/bcfg2-server.init +++ b/redhat/scripts/bcfg2-server.init @@ -27,7 +27,7 @@ PATH=/sbin:/bin:/usr/bin:/usr/sbin test -f /etc/sysconfig/$prog && . /etc/sysconfig/$prog if [ "$BCFG2_SERVER_ENABLED" -eq 0 ] ; then - failure $"bcfg2-server is disabled - see /etc/sysconfig/bcfg2-server" + failure $"bcfg2-server is disabled - see /etc/sysconfig/$prog" echo exit 0 fi diff --git a/reports/reports.wsgi b/reports/reports.wsgi index 92401d763..7113b3308 100644 --- a/reports/reports.wsgi +++ b/reports/reports.wsgi @@ -1,9 +1,21 @@ import os -import Bcfg2.settings -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' +import Bcfg2.Options +import Bcfg2.DBSettings + +config_parsed = False + import django.core.handlers.wsgi + def application(environ, start_response): - if 'BCFG2_CONFIG_FILE' in environ: - Bcfg2.settings.read_config(cfile=environ['BCFG2_CONFIG_FILE']) - return django.core.handlers.wsgi.WSGIHandler()(environ, start_response) + global config_parsed + + # with wsgi, the environment isn't present in os.environ, but + # is passwd to the application function + if 'BCFG2_CONFIG_FILE' in environ: + os.environ['BCFG2_CONFIG_FILE'] = environ['BCFG2_CONFIG_FILE'] + if not config_parsed: + Bcfg2.Options.get_parser().parse() + config_parsed = True + + return django.core.handlers.wsgi.WSGIHandler()(environ, start_response) diff --git a/reports/site_media/bcfg2_base.css b/reports/site_media/bcfg2_base.css index cd5acc6c1..edec22d7c 100644 --- a/reports/site_media/bcfg2_base.css +++ b/reports/site_media/bcfg2_base.css @@ -159,6 +159,9 @@ ul.menu-level2 { .modified-lineitem { background: #FFEC8B; } +.stale-lineitem { + background: #CCCCCC; +} table.grid-view { border: solid 1px #98DBCC; diff --git a/schemas/acl-ip.xsd b/schemas/acl-ip.xsd new file mode 100644 index 000000000..1d6106c05 --- /dev/null +++ b/schemas/acl-ip.xsd @@ -0,0 +1,56 @@ +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> + <xsd:annotation> + <xsd:documentation> + Schema for IP-based client ACLs: + :ref:`server-plugins-misc-acl` ``ip.xml`` + </xsd:documentation> + </xsd:annotation> + + <xsd:complexType name="IPACLType"> + <xsd:attribute type="xsd:string" name="method" use="required"> + <xsd:annotation> + <xsd:documentation> + The name of the XML-RPC method to allow or deny. Limited + wildcards are supported. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="address"> + <xsd:annotation> + <xsd:documentation> + The IP address to match against. This is an exact match + unless :xml:attribute:`IPACLType:netmask` is defined. If + this is not defined, all addresses match the given rule. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="netmask"> + <xsd:annotation> + <xsd:documentation> + If this is defined, then it is combined with + :xml:attribute:`IPACLType:address` to produce a CIDR range, + which is used for matching instead of exact matching based + only on IP address. This can be either an integer netmask + (e.g., ``netmask="24"``) or a dotted-quad (e.g., + ``netmask="255.255.255.0"``). + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + + <xsd:complexType name="IPACLContainerType"> + <xsd:annotation> + <xsd:documentation> + Top-level tag for describing metadata-based client ACLs. + </xsd:documentation> + </xsd:annotation> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="Allow" type="IPACLType"/> + <xsd:element name="Deny" type="IPACLType"/> + <xsd:element name="Defer" type="IPACLType"/> + <xsd:element name="ACL" type="IPACLContainerType"/> + </xsd:choice> + </xsd:complexType> + + <xsd:element name="ACL" type="IPACLContainerType"/> +</xsd:schema> diff --git a/schemas/acl-metadata.xsd b/schemas/acl-metadata.xsd new file mode 100644 index 000000000..643dfec7f --- /dev/null +++ b/schemas/acl-metadata.xsd @@ -0,0 +1,87 @@ +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> + <xsd:annotation> + <xsd:documentation> + Schema for metadata-based client ACLs: + :ref:`server-plugins-misc-acl` ``metadata.xml`` + </xsd:documentation> + </xsd:annotation> + + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + + <xsd:complexType name="MetadataACLGroupType"> + <xsd:annotation> + <xsd:documentation> + An **MetadataACLGroupType** is a tag used to provide logic. + Child entries of a MetadataACLGroupType tag only apply to + machines that match the condition specified -- either + membership in a group, or a matching client name. + :xml:attribute:`MetadataACLGroupType:negate` can be set to + negate the sense of the match. + </xsd:documentation> + </xsd:annotation> + <xsd:group ref="MetadataACLElements" minOccurs="1" maxOccurs="unbounded"/> + <xsd:attribute name='name' type='xsd:string'> + <xsd:annotation> + <xsd:documentation> + The name of the client or group to match on. Child entries + will only apply to this client or group (unless + :xml:attribute:`MetadataACLGroupType:negate` is set). + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name='negate' type='xsd:boolean'> + <xsd:annotation> + <xsd:documentation> + Negate the sense of the match, so that child entries only + apply to a client if it is not a member of the given group + or does not have the given name. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name="MetadataACLType"> + <xsd:attribute type="xsd:string" name="method" use="required"> + <xsd:annotation> + <xsd:documentation> + The name of the XML-RPC method to allow or deny. Limited + wildcards are supported. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name="MetadataACLContainerType"> + <xsd:annotation> + <xsd:documentation> + Top-level tag for describing metadata-based client ACLs. + </xsd:documentation> + </xsd:annotation> + <xsd:group ref="MetadataACLElements" minOccurs="1" maxOccurs="unbounded"/> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + + <xsd:group name="MetadataACLElements"> + <xsd:choice> + <xsd:group ref="py:genshiElements"/> + <xsd:element name="Allow" type="MetadataACLType"/> + <xsd:element name="Deny" type="MetadataACLType"/> + <xsd:element name="Group" type="MetadataACLGroupType"/> + <xsd:element name="Client" type="MetadataACLGroupType"/> + <xsd:element name="ACL" type="MetadataACLContainerType"/> + </xsd:choice> + </xsd:group> + + <xsd:element name="ACL" type="MetadataACLContainerType"/> +</xsd:schema> diff --git a/schemas/acl.xsd b/schemas/acl.xsd new file mode 100644 index 000000000..ac678b6c1 --- /dev/null +++ b/schemas/acl.xsd @@ -0,0 +1,22 @@ +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> + + <xsd:annotation> + <xsd:documentation> + acl config schema for bcfg2 + Matt Schwager + </xsd:documentation> + </xsd:annotation> + + <xsd:import namespace="http://www.w3.org/XML/1998/namespace" + schemaLocation="xml.xsd"/> + + <xsd:complexType name="IPs"> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="IP" type="xsd:string" minOccurs="1" + maxOccurs="unbounded"/> + <xsd:element name="CIDR" type="xsd:string" minOccurs="0" + maxOccurs="unbounded"/> + </xsd:choice> + </xsd:complexType> + +</xsd:schema> diff --git a/schemas/authorizedkeys.xsd b/schemas/authorizedkeys.xsd index e59c964f6..c464017b4 100644 --- a/schemas/authorizedkeys.xsd +++ b/schemas/authorizedkeys.xsd @@ -1,4 +1,5 @@ -<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> <xsd:annotation> <xsd:documentation> Schema for :ref:`server-plugins-generators-cfg-sshkeys` @@ -6,6 +7,9 @@ </xsd:documentation> </xsd:annotation> + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + <xsd:complexType name="AuthorizedKeysGroupType"> <xsd:annotation> <xsd:documentation> @@ -18,9 +22,11 @@ </xsd:documentation> </xsd:annotation> <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="Allow" type="AllowType"/> <xsd:element name="Group" type="AuthorizedKeysGroupType"/> <xsd:element name="Client" type="AuthorizedKeysGroupType"/> + <xsd:element name="AuthorizedKeys" type="AuthorizedKeysType"/> </xsd:choice> <xsd:attribute name='name' type='xsd:string'> <xsd:annotation> @@ -40,6 +46,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:complexType name="OptionContainerType"> @@ -57,7 +64,6 @@ <xsd:element name="Group" type="OptionContainerType"/> <xsd:element name="Client" type="OptionContainerType"/> <xsd:element name="Option" type="AuthorizedKeysOptionType"/> - <xsd:element name="Params" type="AuthorizedKeysParamsType"/> </xsd:choice> <xsd:attribute name='name' type='xsd:string'> <xsd:annotation> @@ -87,10 +93,10 @@ </xsd:documentation> </xsd:annotation> <xsd:choice minOccurs="0" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="Group" type="OptionContainerType"/> <xsd:element name="Client" type="OptionContainerType"/> <xsd:element name="Option" type="AuthorizedKeysOptionType"/> - <xsd:element name="Params" type="AuthorizedKeysParamsType"/> </xsd:choice> <xsd:attribute name="from" type="xsd:string"> <xsd:annotation> @@ -124,6 +130,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:complexType name="AuthorizedKeysOptionType"> @@ -150,29 +157,29 @@ </xsd:attribute> </xsd:complexType> - <xsd:complexType name="AuthorizedKeysParamsType"> + <xsd:complexType name="AuthorizedKeysType"> <xsd:annotation> <xsd:documentation> - **Deprecated** way to specify options for public key - authentication and connection. See :manpage:`sshd(8)` for - details on allowable parameters. + Top-level tag for describing a generated SSH key pair. </xsd:documentation> </xsd:annotation> - <xsd:anyAttribute processContents="lax"/> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> + <xsd:element name="Allow" type="AllowType"/> + <xsd:element name="Group" type="AuthorizedKeysGroupType"/> + <xsd:element name="Client" type="AuthorizedKeysGroupType"/> + <xsd:element name="AuthorizedKeys" type="AuthorizedKeysType"/> + </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> - <xsd:element name="AuthorizedKeys"> - <xsd:annotation> - <xsd:documentation> - Top-level tag for describing a generated SSH key pair. - </xsd:documentation> - </xsd:annotation> - <xsd:complexType> - <xsd:choice minOccurs="1" maxOccurs="unbounded"> - <xsd:element name="Allow" type="AllowType"/> - <xsd:element name="Group" type="AuthorizedKeysGroupType"/> - <xsd:element name="Client" type="AuthorizedKeysGroupType"/> - </xsd:choice> - </xsd:complexType> - </xsd:element> + <xsd:element name="AuthorizedKeys" type="AuthorizedKeysType"/> </xsd:schema> diff --git a/schemas/bundle.xsd b/schemas/bundle.xsd index 337fc5ec7..aeacd0517 100644 --- a/schemas/bundle.xsd +++ b/schemas/bundle.xsd @@ -35,10 +35,7 @@ <xsd:annotation> <xsd:documentation> Abstract implementation of a Path entry. The entry will - either be handled by Cfg, TGenshi, or another - Generator plugin; or handled by Rules, in which case - the full specification of this entry will be included in - Rules. + be handled by a Generator plugin, like Cfg or Rules. </xsd:documentation> </xsd:annotation> </xsd:element> @@ -72,15 +69,6 @@ </xsd:documentation> </xsd:annotation> </xsd:element> - <xsd:element name='PostInstall' type='StructureEntry'> - <xsd:annotation> - <xsd:documentation> - PostInstall entries are deprecated in favor of Action - entries. Actions can do everything PostInstall entries can - do and more. - </xsd:documentation> - </xsd:annotation> - </xsd:element> <xsd:element name='SEBoolean' type='SELinuxStructure'> <xsd:annotation> <xsd:documentation> @@ -323,11 +311,22 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute type='xsd:string' name='name' use='required'> + <xsd:attribute type='xsd:boolean' name='independent'> + <xsd:annotation> + <xsd:documentation> + If set to ``true``, indicates that the bundle is a + collection of independent entries, and that service restarts + and modified actions should not be performed. See + :ref:`server-plugins-structures-bundler-magic` for more. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type='xsd:string' name='name'> <xsd:annotation> <xsd:documentation> - The name of the bundle. This must match the bundle - filename, minus the extension. + **Deprecated.** The name of the bundle. If present, this + must match the bundle filename, minus the extension. + Specifying the name explicitly is deprecated. </xsd:documentation> </xsd:annotation> </xsd:attribute> @@ -352,6 +351,14 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> <xsd:attribute ref="xml:base"/> </xsd:complexType> diff --git a/schemas/decisions.xsd b/schemas/decisions.xsd index 30115b367..9df4b1215 100644 --- a/schemas/decisions.xsd +++ b/schemas/decisions.xsd @@ -1,5 +1,6 @@ -<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> - +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> + <xsd:annotation> <xsd:documentation> decision list schema for bcfg2 @@ -7,16 +8,72 @@ </xsd:documentation> </xsd:annotation> - <xsd:element name='Decisions'> - <xsd:complexType> - <xsd:choice minOccurs='0' maxOccurs='unbounded'> - <xsd:element name='Decision'> - <xsd:complexType> - <xsd:attribute name='type' type='xsd:string' use='required'/> - <xsd:attribute name='name' type='xsd:string' use='required'/> - </xsd:complexType> - </xsd:element> - </xsd:choice> - </xsd:complexType> - </xsd:element> + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + + <xsd:complexType name="DecisionsGroupType"> + <xsd:annotation> + <xsd:documentation> + A **DecisionsGroupType** is a tag used to provide logic. + Child entries of a DecisionsGroupType tag only apply to + machines that match the condition specified -- either + membership in a group, or a matching client name. + :xml:attribute:`DecisionsGroupType:negate` can be set to + negate the sense of the match. + </xsd:documentation> + </xsd:annotation> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="Decisions" type="DecisionsType"/> + <xsd:element name="Decision" type="DecisionType"/> + <xsd:element name="Group" type="DecisionsGroupType"/> + <xsd:element name="Client" type="DecisionsGroupType"/> + <xsd:group ref="py:genshiElements"/> + </xsd:choice> + <xsd:attribute name='name' type='xsd:string'> + <xsd:annotation> + <xsd:documentation> + The name of the client or group to match on. Child entries + will only apply to this client or group (unless + :xml:attribute:`DecisionsGroupType:negate` is set). + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name='negate' type='xsd:boolean'> + <xsd:annotation> + <xsd:documentation> + Negate the sense of the match, so that child entries only + apply to a client if it is not a member of the given group + or does not have the given name. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name="DecisionType"> + <xsd:attribute name='type' type='xsd:string' use='required'/> + <xsd:attribute name='name' type='xsd:string' use='required'/> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name="DecisionsType"> + <xsd:choice minOccurs='0' maxOccurs='unbounded'> + <xsd:element name="Decisions" type="DecisionsType"/> + <xsd:element name="Decision" type="DecisionType"/> + <xsd:element name="Group" type="DecisionsGroupType"/> + <xsd:element name="Client" type="DecisionsGroupType"/> + <xsd:group ref="py:genshiElements"/> + </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:element name='Decisions' type="DecisionsType"/> </xsd:schema> diff --git a/schemas/defaults.xsd b/schemas/defaults.xsd index 17ae84366..f810a6269 100644 --- a/schemas/defaults.xsd +++ b/schemas/defaults.xsd @@ -35,6 +35,14 @@ <xsd:element name="Client" type="DContainerType"/> </xsd:choice> <xsd:attribute name="priority" type="xsd:integer" use="required"/> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> </xsd:element> </xsd:schema> diff --git a/schemas/fileprobes.xsd b/schemas/fileprobes.xsd index 0dfb8cdd2..64f01bf8e 100644 --- a/schemas/fileprobes.xsd +++ b/schemas/fileprobes.xsd @@ -1,4 +1,5 @@ -<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> <xsd:annotation> <xsd:documentation> FileProbes plugin config schema for bcfg2 @@ -6,29 +7,44 @@ </xsd:documentation> </xsd:annotation> + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + <xsd:complexType name="FileProbesGroupType"> <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="FileProbe" type="FileProbeType"/> <xsd:element name="Group" type="FileProbesGroupType"/> <xsd:element name="Client" type="FileProbesGroupType"/> </xsd:choice> <xsd:attribute type="xsd:string" name="name" use="required"/> <xsd:attribute type="xsd:string" name="negate"/> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:complexType name="FileProbeType"> <xsd:attribute type="xsd:string" name="name" use="required"/> <xsd:attribute type="xsd:string" name="encoding"/> <xsd:attribute type="xsd:string" name="update"/> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:element name="FileProbes"> <xsd:complexType> <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="FileProbe" type="FileProbeType"/> <xsd:element name="Group" type="FileProbesGroupType"/> <xsd:element name="Client" type="FileProbesGroupType"/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> </xsd:element> </xsd:schema> diff --git a/schemas/info.xsd b/schemas/info.xsd index 24538ffe3..5291562c1 100644 --- a/schemas/info.xsd +++ b/schemas/info.xsd @@ -92,6 +92,7 @@ </xsd:documentation> </xsd:annotation> <xsd:choice minOccurs='1' maxOccurs='1'> + <xsd:element name='FileInfo' type="FileInfoType"/> <xsd:element name='Info' type='InfoType'/> <xsd:element name='Group' type='InfoGroupType' minOccurs='0' maxOccurs='unbounded'/> @@ -121,19 +122,28 @@ </xsd:attribute> </xsd:complexType> - <xsd:element name='FileInfo'> + <xsd:complexType name="FileInfoType"> <xsd:annotation> <xsd:documentation> Top-level tag for ``info.xml``. </xsd:documentation> </xsd:annotation> - <xsd:complexType> - <xsd:choice minOccurs='0' maxOccurs='unbounded'> - <xsd:element name='Group' type='InfoGroupType'/> - <xsd:element name='Client' type='InfoGroupType'/> - <xsd:element name='Path' type='InfoGroupType'/> - <xsd:element name='Info' type='InfoType'/> - </xsd:choice> - </xsd:complexType> - </xsd:element> + <xsd:choice minOccurs='0' maxOccurs='unbounded'> + <xsd:element name='FileInfo' type="FileInfoType"/> + <xsd:element name='Group' type='InfoGroupType'/> + <xsd:element name='Client' type='InfoGroupType'/> + <xsd:element name='Path' type='InfoGroupType'/> + <xsd:element name='Info' type='InfoType'/> + </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + + <xsd:element name='FileInfo' type="FileInfoType"/> </xsd:schema> diff --git a/schemas/nagiosgen.xsd b/schemas/nagiosgen.xsd index 99d6b91c6..24c298885 100644 --- a/schemas/nagiosgen.xsd +++ b/schemas/nagiosgen.xsd @@ -1,4 +1,5 @@ -<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> <xsd:annotation> <xsd:documentation> NagiosGen config schema for bcfg2 @@ -6,26 +7,42 @@ </xsd:documentation> </xsd:annotation> + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + <xsd:complexType name="NagiosGenGroupType"> <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="Option" type="OptionType"/> <xsd:element name="Group" type="NagiosGenGroupType"/> <xsd:element name="Client" type="NagiosGenGroupType"/> </xsd:choice> <xsd:attribute type="xsd:string" name="name" use="required"/> <xsd:attribute type="xsd:string" name="negate"/> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:complexType name="OptionType" mixed="true"> <xsd:attribute type="xsd:string" name="name" use="required"/> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:element name="NagiosGen"> <xsd:complexType> <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> + <xsd:element name="Option" type="OptionType"/> <xsd:element name="Group" type="NagiosGenGroupType"/> <xsd:element name="Client" type="NagiosGenGroupType"/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> </xsd:element> </xsd:schema> diff --git a/schemas/packages.xsd b/schemas/packages.xsd index 46ae3c663..fc5a1356c 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -1,4 +1,5 @@ -<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> <xsd:annotation> <xsd:documentation> packages config schema for bcfg2 @@ -8,12 +9,15 @@ <xsd:import namespace="http://www.w3.org/XML/1998/namespace" schemaLocation="xml.xsd"/> + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> <xsd:simpleType name="SourceTypeEnum"> <xsd:restriction base="xsd:string"> <xsd:enumeration value="yum"/> <xsd:enumeration value="apt"/> <xsd:enumeration value="pac"/> + <xsd:enumeration value="pkgng"/> </xsd:restriction> </xsd:simpleType> @@ -40,6 +44,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> <xsd:anyAttribute processContents="lax"> <xsd:annotation> <xsd:documentation> @@ -58,6 +63,7 @@ </xsd:documentation> </xsd:annotation> <xsd:choice minOccurs="0" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="Component" type="xsd:string"> <xsd:annotation> <xsd:documentation> @@ -219,10 +225,12 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:complexType name="PackagesGroupType"> <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="Group" type="PackagesGroupType"/> <xsd:element name="Client" type="PackagesGroupType"/> <xsd:element name="Sources" type="SourcesType"/> @@ -230,16 +238,27 @@ </xsd:choice> <xsd:attribute type="xsd:string" name="name" use="required"/> <xsd:attribute type="xsd:boolean" name="negate"/> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:complexType name="SourcesType"> <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="Group" type="PackagesGroupType"/> <xsd:element name="Client" type="PackagesGroupType"/> <xsd:element name="Source" type="SourceType"/> <xsd:element name="Sources" type="SourcesType"/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attribute ref="xml:base"/> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:element name="Sources" type="SourcesType"/> diff --git a/schemas/pathentry.xsd b/schemas/pathentry.xsd index e5d2ef6af..44c86f9bc 100644 --- a/schemas/pathentry.xsd +++ b/schemas/pathentry.xsd @@ -12,7 +12,34 @@ schemaLocation="genshi.xsd"/> <xsd:complexType name='PathEntry'> - <xsd:attribute type='xsd:string' name='name' use='required'/> + <xsd:annotation> + <xsd:documentation> + Abstract description of a path to be installed. This can + either be a single explicit path (e.g., ``<Path + name="/etc/foo.conf"/>``) or a glob that matches a set of + paths (e.g., ``<Path glob="/etc/foo/*"/>``). Path + globbing may not work for some dynamically handled Path + entries, for instance :ref:`Packages client configs + <generating-client-configs>`. + </xsd:documentation> + </xsd:annotation> + <xsd:attribute type='xsd:string' name='name'> + <xsd:annotation> + <xsd:documentation> + Install the single named path. Either ``name`` or + :xml:attribute:`PathEntry:glob` must be specified. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="glob"> + <xsd:annotation> + <xsd:documentation> + Install all Cfg entries matching the given glob. Either + ``glob`` or :xml:attribute:`PathEntry:name` must be + specified. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attribute type='xsd:string' name='altsrc' use='optional'/> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/pkgtype.xsd b/schemas/pkgtype.xsd index 18eda88ab..7ad7606b2 100644 --- a/schemas/pkgtype.xsd +++ b/schemas/pkgtype.xsd @@ -54,6 +54,15 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute name="recommended" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Whether also the recommended packages should be installed. + This is currently only used with the :ref:`APT + <client-tools-apt>` driver. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -146,38 +155,33 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute name="installed_action" type="xsd:string" - default="install"> + <xsd:attribute name="install_missing" type="xsd:boolean" + default="true"> <xsd:annotation> <xsd:documentation> - If this is set to any value other than "install", - package installation will be suppressed with the - :ref:`YUM24 and RPM <client-tools-yum>` drivers. + Whether or not to install missing packages. This is + only honored by the the :ref:`RPM + <client-tools-yum>` driver. </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute name="version_fail_action" type="xsd:string" - default="upgrade"> + <xsd:attribute name="fix_version" type="xsd:boolean" default="true"> <xsd:annotation> <xsd:documentation> - If this is set to any value other than "upgrade", a - package that has the incorrect version installed will - not be fixed with the :ref:`YUM24 and RPM - <client-tools-yum>` drivers. Note that - "upgrade" is misleading; if a package is installed - that is newer than the desired version, it will not be - downgraded if this attribute is set to anything other - than "upgrade". + Whether or not to upgrade or downgrade packages that + are installed, but have the wrong version. This is + only honored by the :ref:`RPM + <client-tools-yum>` driver. </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute name="verify_fail_action" type="xsd:string"> + <xsd:attribute name="reinstall_broken" type="xsd:boolean" + default="true"> <xsd:annotation> <xsd:documentation> - If this is set to any value other than "reinstall", a - package that fails package verification will not be - reinstalled with the :ref:`YUM24 and RPM - <client-tools-yum>` drivers. + Whether or not to reinstall packages that fail + verification. This is only honored by the :ref:`RPM + <client-tools-yum>` driver. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/schemas/privkey.xsd b/schemas/privkey.xsd index b78fda350..30bc8a1b3 100644 --- a/schemas/privkey.xsd +++ b/schemas/privkey.xsd @@ -1,16 +1,20 @@ -<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> <xsd:annotation> <xsd:documentation> Schema for :ref:`server-plugins-generators-cfg-sshkeys` ``privkey.xml`` </xsd:documentation> </xsd:annotation> + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + <xsd:include schemaLocation="types.xsd"/> <xsd:complexType name="PrivateKeyGroupType"> <xsd:annotation> <xsd:documentation> - An **PrivateKeyGroupType** is a tag used to provide logic. + A **PrivateKeyGroupType** is a tag used to provide logic. Child entries of a PrivateKeyGroupType tag only apply to machines that match the condition specified -- either membership in a group, or a matching client name. @@ -19,6 +23,7 @@ </xsd:documentation> </xsd:annotation> <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="Passphrase" type="PassphraseType"/> <xsd:element name="Params" type="PrivateKeyParamsType"/> <xsd:element name="Group" type="PrivateKeyGroupType"/> @@ -42,6 +47,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:simpleType name="PrivateKeyTypeEnum"> @@ -72,6 +78,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:extension> </xsd:simpleContent> </xsd:complexType> @@ -97,6 +104,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:element name="PrivateKey"> @@ -107,6 +115,7 @@ </xsd:annotation> <xsd:complexType> <xsd:choice minOccurs="0" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="Passphrase" type="PassphraseType"/> <xsd:element name="Params" type="PrivateKeyParamsType"/> <xsd:element name="Group" type="PrivateKeyGroupType"/> @@ -135,14 +144,15 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute name="decrypt" type="EncryptStrictnessEnum"> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> <xsd:annotation> <xsd:documentation> - Override the global strict/lax decryption setting in + Override the global lax_decryption setting in ``bcfg2.conf``. </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> </xsd:element> </xsd:schema> diff --git a/schemas/pubkey.xsd b/schemas/pubkey.xsd index 5671a818d..f3ffc7f44 100644 --- a/schemas/pubkey.xsd +++ b/schemas/pubkey.xsd @@ -1,10 +1,14 @@ -<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> <xsd:annotation> <xsd:documentation> Schema for :ref:`server-plugins-generators-cfg-sshkeys` ``pubkey.xml`` </xsd:documentation> </xsd:annotation> + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + <xsd:element name="PublicKey"> <xsd:annotation> <xsd:documentation> diff --git a/schemas/rules.xsd b/schemas/rules.xsd index ddfb7ad0d..fb41ad9d4 100644 --- a/schemas/rules.xsd +++ b/schemas/rules.xsd @@ -13,10 +13,6 @@ <xsd:import namespace="http://genshi.edgewall.org/" schemaLocation="genshi.xsd"/> - <xsd:complexType name='PostInstallType'> - <xsd:attribute type='xsd:string' name='name' use='required'/> - </xsd:complexType> - <xsd:group name="rulesElements"> <xsd:choice> <xsd:group ref="py:genshiElements"/> @@ -126,15 +122,6 @@ </xsd:documentation> </xsd:annotation> </xsd:element> - <xsd:element name='PostInstall' type='PostInstallType'> - <xsd:annotation> - <xsd:documentation> - PostInstall entries are deprecated in favor of Action - entries. Actions can do everything PostInstall entries can - do and more. - </xsd:documentation> - </xsd:annotation> - </xsd:element> <xsd:element name='Group' type='RContainerType'> <xsd:annotation> <xsd:documentation> @@ -211,6 +198,14 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> </xsd:element> diff --git a/schemas/sslca-cert.xsd b/schemas/sslca-cert.xsd index a9c01fb29..7330ca0ff 100644 --- a/schemas/sslca-cert.xsd +++ b/schemas/sslca-cert.xsd @@ -1,10 +1,15 @@ -<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> <xsd:annotation> <xsd:documentation> - Schema for :ref:`server-plugins-generators-sslca` ``cert.xml`` + Schema for :ref:`server-plugins-generators-cfg-ssl-certificates` + ``sslcert.xml`` </xsd:documentation> </xsd:annotation> + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + <xsd:complexType name="SSLCACertGroupType"> <xsd:annotation> <xsd:documentation> @@ -17,10 +22,12 @@ </xsd:documentation> </xsd:annotation> <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="Cert" type="CertType"/> <xsd:element name="Group" type="SSLCACertGroupType"/> <xsd:element name="Client" type="SSLCACertGroupType"/> <xsd:element name="subjectAltName" type="SubjectAltNameType"/> + <xsd:element name="CertInfo" type="CertInfoType"/> </xsd:choice> <xsd:attribute name='name' type='xsd:string'> <xsd:annotation> @@ -40,6 +47,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:simpleType name="SSLCACertFormatEnum"> @@ -69,7 +77,7 @@ <xsd:documentation> The full path to the key entry to use for this certificate. This is the *client* path; e.g., for a key defined at - ``/var/lib/bcfg2/SSLCA/etc/pki/tls/private/foo.key/key.xml``, + ``/var/lib/bcfg2/SSLCA/etc/pki/tls/private/foo.key/sslkey.xml``, **key** should be ``/etc/pki/tls/private/foo.key``. </xsd:documentation> </xsd:annotation> @@ -85,8 +93,8 @@ <xsd:annotation> <xsd:documentation> The name of the CA (from :ref:`bcfg2.conf - <sslca-configuration>`) to use to generate this - certificate. + <server-plugins-generators-cfg-configuration>`) to use + to generate this certificate. </xsd:documentation> </xsd:annotation> </xsd:attribute> @@ -147,21 +155,32 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> - <xsd:element name="CertInfo"> + <xsd:complexType name="CertInfoType"> <xsd:annotation> <xsd:documentation> Top-level tag for describing an SSLCA generated certificate. </xsd:documentation> </xsd:annotation> - <xsd:complexType> - <xsd:choice minOccurs="1" maxOccurs="unbounded"> - <xsd:element name="Cert" type="CertType"/> - <xsd:element name="Group" type="SSLCACertGroupType"/> - <xsd:element name="Client" type="SSLCACertGroupType"/> - <xsd:element name="subjectAltName" type="SubjectAltNameType"/> - </xsd:choice> - </xsd:complexType> - </xsd:element> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> + <xsd:element name="Cert" type="CertType"/> + <xsd:element name="Group" type="SSLCACertGroupType"/> + <xsd:element name="Client" type="SSLCACertGroupType"/> + <xsd:element name="subjectAltName" type="SubjectAltNameType"/> + <xsd:element name="CertInfo" type="CertInfoType"/> + </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + + <xsd:element name="CertInfo" type="CertInfoType"/> </xsd:schema> diff --git a/schemas/sslca-key.xsd b/schemas/sslca-key.xsd index efd4abd58..496da859f 100644 --- a/schemas/sslca-key.xsd +++ b/schemas/sslca-key.xsd @@ -1,10 +1,15 @@ -<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> <xsd:annotation> <xsd:documentation> - Schema for :ref:`server-plugins-generators-sslca` ``key.xml`` + Schema for :ref:`server-plugins-generators-cfg-ssl-certificates` + ``sslkey.xml`` </xsd:documentation> </xsd:annotation> + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + <xsd:complexType name="SSLCAKeyGroupType"> <xsd:annotation> <xsd:documentation> @@ -17,9 +22,11 @@ </xsd:documentation> </xsd:annotation> <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> <xsd:element name="Key" type="KeyType"/> <xsd:element name="Group" type="SSLCAKeyGroupType"/> <xsd:element name="Client" type="SSLCAKeyGroupType"/> + <xsd:element name="KeyInfo" type="KeyInfoType"/> </xsd:choice> <xsd:attribute name='name' type='xsd:string'> <xsd:annotation> @@ -39,6 +46,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> <xsd:simpleType name="KeyTypeEnum"> @@ -68,20 +76,46 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> - <xsd:element name="KeyInfo"> + <xsd:complexType name="KeyInfoType"> <xsd:annotation> <xsd:documentation> Top-level tag for describing an SSLCA generated key. </xsd:documentation> </xsd:annotation> - <xsd:complexType> - <xsd:choice minOccurs="1" maxOccurs="unbounded"> - <xsd:element name="Key" type="KeyType"/> - <xsd:element name="Group" type="SSLCAKeyGroupType"/> - <xsd:element name="Client" type="SSLCAKeyGroupType"/> - </xsd:choice> - </xsd:complexType> - </xsd:element> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> + <xsd:element name="Key" type="KeyType"/> + <xsd:element name="Group" type="SSLCAKeyGroupType"/> + <xsd:element name="Client" type="SSLCAKeyGroupType"/> + <xsd:element name="KeyInfo" type="KeyInfoType"/> + </xsd:choice> + <xsd:attribute name="perhost" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Create keys on a per-host basis (rather than on a per-group + basis). + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name="category" type="xsd:string"> + <xsd:annotation> + <xsd:documentation> + Create keys specific to the given category, instead of + specific to the category given in ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name="priority" type="xsd:positiveInteger" default="50"> + <xsd:annotation> + <xsd:documentation> + Create group-specific keys with the given priority. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + + <xsd:element name="KeyInfo" type="KeyInfoType"/> </xsd:schema> diff --git a/schemas/types.xsd b/schemas/types.xsd index 52c9d59c8..0a55f6355 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -105,13 +105,6 @@ </xsd:restriction> </xsd:simpleType> - <xsd:simpleType name="EncryptStrictnessEnum"> - <xsd:restriction base="xsd:string"> - <xsd:enumeration value="strict"/> - <xsd:enumeration value="lax"/> - </xsd:restriction> - </xsd:simpleType> - <xsd:complexType name='ActionType'> <xsd:annotation> <xsd:documentation> @@ -399,6 +392,14 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:string" name="encrypted"> + <xsd:annotation> + <xsd:documentation> + The name of the encryption passphrase that the text content + of this tag is encrypted with. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attribute type="xsd:token" name="lens"> <xsd:annotation> <xsd:documentation> @@ -438,6 +439,16 @@ </xsd:simpleContent> </xsd:complexType> + <xsd:complexType name="MemberOfContainerType"> + <xsd:choice maxOccurs="unbounded"> + <xsd:element name="MemberOf" type="MemberOfType"/> + <xsd:element name="Client" type="MemberOfContainerType"/> + <xsd:element name="Group" type="MemberOfContainerType"/> + </xsd:choice> + <xsd:attribute name="name" type="xsd:string" use="required"/> + <xsd:attribute name="negate" type="xsd:boolean"/> + </xsd:complexType> + <xsd:complexType name="POSIXUserType"> <xsd:annotation> <xsd:documentation> @@ -446,6 +457,8 @@ </xsd:annotation> <xsd:choice minOccurs='0' maxOccurs='unbounded'> <xsd:element name='MemberOf' type='MemberOfType'/> + <xsd:element name='Group' type='MemberOfContainerType'/> + <xsd:element name='Client' type='MemberOfContainerType'/> </xsd:choice> <xsd:attribute type="xsd:token" name="name" use="required"> <xsd:annotation> @@ -474,9 +487,9 @@ <xsd:annotation> <xsd:documentation> This field is typically used to record general information - about the account or its user(s) such as their real name - and phone number. If this is not set, the GECOS will be - the same as the username. + about the account or its user(s) such as their real name + and phone number. If this is not set, the GECOS will be + the same as the username. </xsd:documentation> </xsd:annotation> </xsd:attribute> @@ -30,6 +30,7 @@ setup(name="Bcfg2", # nosetests test_suite='nose.collector', packages=["Bcfg2", + "Bcfg2.Options", "Bcfg2.Client", "Bcfg2.Client.Tools", "Bcfg2.Client.Tools.POSIX", @@ -39,10 +40,7 @@ setup(name="Bcfg2", "Bcfg2.Reporting.migrations", "Bcfg2.Reporting.templatetags", 'Bcfg2.Server', - "Bcfg2.Server.Admin", "Bcfg2.Server.FileMonitor", - "Bcfg2.Server.Hostbase", - "Bcfg2.Server.Hostbase.hostbase", "Bcfg2.Server.Lint", "Bcfg2.Server.Plugin", "Bcfg2.Server.Plugins", @@ -50,10 +48,9 @@ setup(name="Bcfg2", "Bcfg2.Server.Plugins.Cfg", "Bcfg2.Server.Reports", "Bcfg2.Server.Reports.reports", - "Bcfg2.Server.Snapshots", ], install_requires=inst_reqs, - tests_require=['mock', 'nose', 'sqlalchemy'], + tests_require=['mock', 'nose'], package_dir={'': 'src/lib', }, package_data={'Bcfg2.Reporting': ['templates/*.html', 'templates/*/*.html', @@ -69,12 +66,6 @@ setup(name="Bcfg2", ('share/man/man1', glob("man/bcfg2.1")), ('share/man/man5', glob("man/*.5")), ('share/man/man8', glob("man/*.8")), - ('share/bcfg2/Hostbase/templates', - glob('src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/*.*')), - ('share/bcfg2/Hostbase/templates/hostbase', - glob('src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/*')), - ('share/bcfg2/Hostbase/repo', - glob('src/lib/Bcfg2/Server/Hostbase/templates/*')), ('share/bcfg2/site_media', glob('reports/site_media/*')), ] diff --git a/solaris-ips/MANIFEST.bcfg2-server.header b/solaris-ips/MANIFEST.bcfg2-server.header index 9e6774c87..5a837b5c7 100644 --- a/solaris-ips/MANIFEST.bcfg2-server.header +++ b/solaris-ips/MANIFEST.bcfg2-server.header @@ -1,4 +1,4 @@ license ../../LICENSE license=simplified_bsd set name=description value="Configuration management server" set name=pkg.summary value="Configuration management server" -set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@1.3.5" +set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@1.4.0pre1" diff --git a/solaris-ips/MANIFEST.bcfg2.header b/solaris-ips/MANIFEST.bcfg2.header index 97c63107e..5fd5c0c5d 100644 --- a/solaris-ips/MANIFEST.bcfg2.header +++ b/solaris-ips/MANIFEST.bcfg2.header @@ -1,5 +1,5 @@ license ../../LICENSE license=simplified_bsd set name=description value="Configuration management client" set name=pkg.summary value="Configuration management client" -set name=pkg.fmri value="pkg://bcfg2/bcfg2@1.3.5" +set name=pkg.fmri value="pkg://bcfg2/bcfg2@1.4.0pre1" file usr/bin/bcfg2 group=bin mode=0755 owner=root path=usr/bin/bcfg2 diff --git a/solaris-ips/Makefile b/solaris-ips/Makefile index ec85dccc2..63a1ac61e 100644 --- a/solaris-ips/Makefile +++ b/solaris-ips/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/gmake -VERS=1.3.5-1 +VERS=1.4.0pre1-1 PYVERSION := $(shell python -c "import sys; print sys.version[0:3]") default: clean package diff --git a/solaris-ips/pkginfo.bcfg2 b/solaris-ips/pkginfo.bcfg2 index 57ffaffb8..236009c8f 100644 --- a/solaris-ips/pkginfo.bcfg2 +++ b/solaris-ips/pkginfo.bcfg2 @@ -1,7 +1,7 @@ PKG="SCbcfg2" NAME="bcfg2" ARCH="sparc" -VERSION="1.3.5" +VERSION="1.4.0pre1" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris-ips/pkginfo.bcfg2-server b/solaris-ips/pkginfo.bcfg2-server index 50812c4a5..af1ab36fc 100644 --- a/solaris-ips/pkginfo.bcfg2-server +++ b/solaris-ips/pkginfo.bcfg2-server @@ -1,7 +1,7 @@ PKG="SCbcfg2-server" NAME="bcfg2-server" ARCH="sparc" -VERSION="1.3.5" +VERSION="1.4.0pre1" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris/Makefile b/solaris/Makefile index 7719d7017..a8a673e3e 100644 --- a/solaris/Makefile +++ b/solaris/Makefile @@ -1,7 +1,7 @@ #!/usr/sfw/bin/gmake PYTHON="/usr/local/bin/python" -VERS=1.3.5-1 +VERS=1.4.0pre1-1 export PYVERSION := $(shell $(PYTHON) -c "import sys; print sys.version[0:3]") default: clean package diff --git a/solaris/pkginfo.bcfg2 b/solaris/pkginfo.bcfg2 index 57ffaffb8..236009c8f 100644 --- a/solaris/pkginfo.bcfg2 +++ b/solaris/pkginfo.bcfg2 @@ -1,7 +1,7 @@ PKG="SCbcfg2" NAME="bcfg2" ARCH="sparc" -VERSION="1.3.5" +VERSION="1.4.0pre1" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris/pkginfo.bcfg2-server b/solaris/pkginfo.bcfg2-server index 50812c4a5..af1ab36fc 100644 --- a/solaris/pkginfo.bcfg2-server +++ b/solaris/pkginfo.bcfg2-server @@ -1,7 +1,7 @@ PKG="SCbcfg2-server" NAME="bcfg2-server" ARCH="sparc" -VERSION="1.3.5" +VERSION="1.4.0pre1" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris/prototype.bcfg2 b/solaris/prototype.bcfg2 index 21b6cb564..9e92c84b4 100644 --- a/solaris/prototype.bcfg2 +++ b/solaris/prototype.bcfg2 @@ -4,8 +4,6 @@ d none lib/PYVERSION 0755 root bin d none lib/PYVERSION/site-packages 0755 root bin d none lib/PYVERSION/site-packages/Bcfg2 0755 bin bin f none lib/PYVERSION/site-packages/Bcfg2/__init__.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Statistics.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/SSLServer.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Component.py 0644 bin bin d none lib/PYVERSION/site-packages/Bcfg2/Client 0755 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/XML.py 0644 bin bin @@ -13,7 +11,7 @@ d none lib/PYVERSION/site-packages/Bcfg2/Client/Tools 0755 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/Action.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/IPS.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/FreeBSDInit.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/RPMng.py 0644 bin bin +f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/RPM.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/Chkconfig.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/RcUpdate.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/APT.py 0644 bin bin @@ -24,15 +22,15 @@ f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/rpmtools.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/launchd.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/FreeBSDPackage.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/Blast.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/YUMng.py 0644 bin bin +f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/YUM.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/Portage.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/DebInit.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/Encap.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Tools/SMF.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/__init__.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Client/Frame.py 0644 bin bin +f none lib/PYVERSION/site-packages/Bcfg2/Client/Proxy.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Logger.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Proxy.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Options.py 0644 bin bin d none bin 0755 root bin f none bin/bcfg2 0755 bin bin diff --git a/solaris/prototype.bcfg2-server b/solaris/prototype.bcfg2-server index 590175329..91fa83dca 100644 --- a/solaris/prototype.bcfg2-server +++ b/solaris/prototype.bcfg2-server @@ -8,7 +8,6 @@ d none lib/PYVERSION/site-packages/Bcfg2/Server/Admin 0755 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Admin/Tidy.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Admin/Minestruct.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Admin/__init__.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Server/Admin/Snapshots.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Admin/Init.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Admin/Group.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Admin/Compare.py 0644 bin bin @@ -28,35 +27,25 @@ f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/GroupPatterns.py 0644 bi f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/SSHbase.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Trigger.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/__init__.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Snapshots.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/TCheetah.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Account.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Cfg.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Statistics.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Metadata.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Base.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Pkgmgr.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Ohai.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Properties.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Editor.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Bundler.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/NagiosGen.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Deps.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Svn.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/DBStats.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/TGenshi.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Plugins/Git.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/FileMonitor.py 0644 bin bin -d none lib/PYVERSION/site-packages/Bcfg2/Server/Snapshots 0755 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Server/Snapshots/model.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Server/Snapshots/__init__.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Server/Core.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/__init__.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Statistics.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/SSLServer.py 0644 bin bin +f none lib/PYVERSION/site-packages/Bcfg2/Server/Statistics.py 0644 bin bin +f none lib/PYVERSION/site-packages/Bcfg2/Server/SSLServer.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Component.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Logger.py 0644 bin bin -f none lib/PYVERSION/site-packages/Bcfg2/Proxy.py 0644 bin bin f none lib/PYVERSION/site-packages/Bcfg2/Options.py 0644 bin bin d none bin 0755 bin bin f none bin/bcfg2-server 0755 bin bin diff --git a/src/lib/Bcfg2/Cache.py b/src/lib/Bcfg2/Cache.py deleted file mode 100644 index 842098eda..000000000 --- a/src/lib/Bcfg2/Cache.py +++ /dev/null @@ -1,14 +0,0 @@ -""" An implementation of a simple memory-backed cache. Right now this -doesn't provide many features, but more (time-based expiration, etc.) -can be added as necessary. """ - - -class Cache(dict): - """ an implementation of a simple memory-backed cache """ - - def expire(self, key=None): - """ expire all items, or a specific item, from the cache """ - if key is None: - self.clear() - elif key in self: - del self[key] diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py deleted file mode 100644 index 090921ab2..000000000 --- a/src/lib/Bcfg2/Client/Client.py +++ /dev/null @@ -1,341 +0,0 @@ -""" The main Bcfg2 client class """ - -import os -import sys -import stat -import time -import fcntl -import socket -import logging -import tempfile -import Bcfg2.Proxy -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Client.XML -import Bcfg2.Client.Frame -import Bcfg2.Client.Tools -from Bcfg2.Utils import locked, Executor -from Bcfg2.Compat import xmlrpclib -from Bcfg2.version import __version__ - - -class Client(object): - """ The main Bcfg2 client class """ - - def __init__(self, setup): - self.toolset = None - self.tools = None - self.config = None - self._proxy = None - self.setup = setup - - if self.setup['debug']: - level = logging.DEBUG - elif self.setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2', - to_syslog=self.setup['syslog'], - level=level, - to_file=self.setup['logging']) - self.logger = logging.getLogger('bcfg2') - self.logger.debug(self.setup) - - self.cmd = Executor(self.setup['command_timeout']) - - if self.setup['bundle_quick']: - if not self.setup['bundle'] and not self.setup['skipbundle']: - self.logger.error("-Q option requires -b or -B") - raise SystemExit(1) - elif self.setup['remove']: - self.logger.error("-Q option incompatible with -r") - raise SystemExit(1) - if 'drivers' in self.setup and self.setup['drivers'] == 'help': - self.logger.info("The following drivers are available:") - self.logger.info(Bcfg2.Client.Tools.drivers) - raise SystemExit(0) - if self.setup['remove'] and 'services' in self.setup['remove'].lower(): - self.logger.error("Service removal is nonsensical; " - "removed services will only be disabled") - if (self.setup['remove'] and - self.setup['remove'].lower() not in ['all', 'services', 'packages', - 'users']): - self.logger.error("Got unknown argument %s for -r" % - self.setup['remove']) - if self.setup["file"] and self.setup["cache"]: - print("cannot use -f and -c together") - raise SystemExit(1) - if not self.setup['server'].startswith('https://'): - self.setup['server'] = 'https://' + self.setup['server'] - - def _probe_failure(self, probename, msg): - """ handle failure of a probe in the way the user wants us to - (exit or continue) """ - message = "Failed to execute probe %s: %s" % (probename, msg) - if self.setup['probe_exit']: - self.fatal_error(message) - else: - self.logger.error(message) - - def run_probe(self, probe): - """Execute probe.""" - name = probe.get('name') - self.logger.info("Running probe %s" % name) - ret = Bcfg2.Client.XML.Element("probe-data", - name=name, - source=probe.get('source')) - try: - scripthandle, scriptname = tempfile.mkstemp() - script = os.fdopen(scripthandle, 'w') - try: - script.write("#!%s\n" % - (probe.attrib.get('interpreter', '/bin/sh'))) - if sys.hexversion >= 0x03000000: - script.write(probe.text) - else: - script.write(probe.text.encode('utf-8')) - script.close() - os.chmod(scriptname, - stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | - stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | - stat.S_IWUSR) # 0755 - rv = self.cmd.run(scriptname, timeout=self.setup['timeout']) - if rv.stderr: - self.logger.warning("Probe %s has error output: %s" % - (name, rv.stderr)) - if not rv.success: - self._probe_failure(name, "Return value %s" % rv) - self.logger.info("Probe %s has result:" % name) - self.logger.info(rv.stdout) - if sys.hexversion >= 0x03000000: - ret.text = rv.stdout - else: - ret.text = rv.stdout.decode('utf-8') - finally: - os.unlink(scriptname) - except SystemExit: - raise - except: - self._probe_failure(name, sys.exc_info()[1]) - return ret - - def fatal_error(self, message): - """Signal a fatal error.""" - self.logger.error("Fatal error: %s" % (message)) - raise SystemExit(1) - - @property - def proxy(self): - """ get an XML-RPC proxy to the server """ - if self._proxy is None: - self._proxy = Bcfg2.Proxy.ComponentProxy( - self.setup['server'], - self.setup['user'], - self.setup['password'], - key=self.setup['key'], - cert=self.setup['certificate'], - ca=self.setup['ca'], - allowedServerCNs=self.setup['serverCN'], - timeout=self.setup['timeout'], - retries=int(self.setup['retries']), - delay=int(self.setup['retry_delay']), - protocol=self.setup['protocol']) - return self._proxy - - def run_probes(self, times=None): - """ run probes and upload probe data """ - if times is None: - times = dict() - - try: - probes = Bcfg2.Client.XML.XML(str(self.proxy.GetProbes())) - except (Bcfg2.Proxy.ProxyError, - Bcfg2.Proxy.CertificateError, - socket.gaierror, - socket.error): - err = sys.exc_info()[1] - self.fatal_error("Failed to download probes from bcfg2: %s" % err) - except Bcfg2.Client.XML.ParseError: - err = sys.exc_info()[1] - self.fatal_error("Server returned invalid probe requests: %s" % - err) - - times['probe_download'] = time.time() - - # execute probes - probedata = Bcfg2.Client.XML.Element("ProbeData") - for probe in probes.findall(".//probe"): - probedata.append(self.run_probe(probe)) - - if len(probes.findall(".//probe")) > 0: - try: - # upload probe responses - self.proxy.RecvProbeData( - Bcfg2.Client.XML.tostring( - probedata, - xml_declaration=False).decode('utf-8')) - except Bcfg2.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to upload probe data: %s" % err) - - times['probe_upload'] = time.time() - - def get_config(self, times=None): - """ load the configuration, either from the cached - configuration file (-f), or from the server """ - if times is None: - times = dict() - - if self.setup['file']: - # read config from file - try: - self.logger.debug("Reading cached configuration from %s" % - self.setup['file']) - return open(self.setup['file'], 'r').read() - except IOError: - self.fatal_error("Failed to read cached configuration from: %s" - % (self.setup['file'])) - else: - # retrieve config from server - if self.setup['profile']: - try: - self.proxy.AssertProfile(self.setup['profile']) - except Bcfg2.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to set client profile: %s" % err) - - try: - self.proxy.DeclareVersion(__version__) - except xmlrpclib.Fault: - err = sys.exc_info()[1] - if (err.faultCode == xmlrpclib.METHOD_NOT_FOUND or - (err.faultCode == 7 and - err.faultString.startswith("Unknown method"))): - self.logger.debug("Server does not support declaring " - "client version") - else: - self.logger.error("Failed to declare version: %s" % err) - except (Bcfg2.Proxy.ProxyError, - Bcfg2.Proxy.CertificateError, - socket.gaierror, - socket.error): - err = sys.exc_info()[1] - self.logger.error("Failed to declare version: %s" % err) - - self.run_probes(times=times) - - if self.setup['decision'] in ['whitelist', 'blacklist']: - try: - self.setup['decision_list'] = \ - self.proxy.GetDecisionList(self.setup['decision']) - self.logger.info("Got decision list from server:") - self.logger.info(self.setup['decision_list']) - except Bcfg2.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to get decision list: %s" % err) - - try: - rawconfig = self.proxy.GetConfig().encode('utf-8') - except Bcfg2.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to download configuration from " - "Bcfg2: %s" % err) - - times['config_download'] = time.time() - return rawconfig - - def run(self): - """Perform client execution phase.""" - times = {} - - # begin configuration - times['start'] = time.time() - - self.logger.info("Starting Bcfg2 client run at %s" % times['start']) - - rawconfig = self.get_config(times=times).decode('utf-8') - - if self.setup['cache']: - try: - open(self.setup['cache'], 'w').write(rawconfig) - os.chmod(self.setup['cache'], 33152) - except IOError: - self.logger.warning("Failed to write config cache file %s" % - (self.setup['cache'])) - times['caching'] = time.time() - - try: - self.config = Bcfg2.Client.XML.XML(rawconfig) - except Bcfg2.Client.XML.ParseError: - syntax_error = sys.exc_info()[1] - self.fatal_error("The configuration could not be parsed: %s" % - syntax_error) - - times['config_parse'] = time.time() - - if self.config.tag == 'error': - self.fatal_error("Server error: %s" % (self.config.text)) - return(1) - - if self.setup['bundle_quick']: - newconfig = Bcfg2.Client.XML.XML('<Configuration/>') - for bundle in self.config.getchildren(): - if (bundle.tag == 'Bundle' and - ((self.setup['bundle'] and - bundle.get('name') in self.setup['bundle']) or - (self.setup['skipbundle'] and - bundle.get('name') not in self.setup['skipbundle']))): - newconfig.append(bundle) - self.config = newconfig - - self.tools = Bcfg2.Client.Frame.Frame(self.config, - self.setup, - times, self.setup['drivers'], - self.setup['dryrun']) - - if not self.setup['omit_lock_check']: - # check lock here - try: - lockfile = open(self.setup['lockfile'], 'w') - if locked(lockfile.fileno()): - self.fatal_error("Another instance of Bcfg2 is running. " - "If you want to bypass the check, run " - "with the %s option" % - Bcfg2.Options.OMIT_LOCK_CHECK.cmd) - except SystemExit: - raise - except: - lockfile = None - self.logger.error("Failed to open lockfile %s: %s" % - (self.setup['lockfile'], sys.exc_info()[1])) - - # execute the configuration - self.tools.Execute() - - if not self.setup['omit_lock_check']: - # unlock here - if lockfile: - try: - fcntl.lockf(lockfile.fileno(), fcntl.LOCK_UN) - os.remove(self.setup['lockfile']) - except OSError: - self.logger.error("Failed to unlock lockfile %s" % - lockfile.name) - - if not self.setup['file'] and not self.setup['bundle_quick']: - # upload statistics - feedback = self.tools.GenerateStats() - - try: - self.proxy.RecvStats( - Bcfg2.Client.XML.tostring( - feedback, - xml_declaration=False).decode('utf-8')) - except Bcfg2.Proxy.ProxyError: - err = sys.exc_info()[1] - self.logger.error("Failed to upload configuration statistics: " - "%s" % err) - raise SystemExit(2) - - self.logger.info("Finished Bcfg2 client run at %s" % time.time()) diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py deleted file mode 100644 index ad718749e..000000000 --- a/src/lib/Bcfg2/Client/Frame.py +++ /dev/null @@ -1,536 +0,0 @@ -""" Frame is the Client Framework that verifies and installs entries, -and generates statistics. """ - -import copy -import time -import fnmatch -import logging -import Bcfg2.Client.Tools -from Bcfg2.Client import prompt -from Bcfg2.Compat import any, all # pylint: disable=W0622 - - -def matches_entry(entryspec, entry): - """ Determine if the Decisions-style entry specification matches - the entry. Both are tuples of (tag, name). The entryspec can - handle the wildcard * in either position. """ - if entryspec == entry: - return True - return all(fnmatch.fnmatch(entry[i], entryspec[i]) for i in [0, 1]) - - -def matches_white_list(entry, whitelist): - """ Return True if (<entry tag>, <entry name>) is in the given - whitelist. """ - return any(matches_entry(we, (entry.tag, entry.get('name'))) - for we in whitelist) - - -def passes_black_list(entry, blacklist): - """ Return True if (<entry tag>, <entry name>) is not in the given - blacklist. """ - return not any(matches_entry(be, (entry.tag, entry.get('name'))) - for be in blacklist) - - -# pylint: disable=W0702 -# in frame we frequently want to catch all exceptions, regardless of -# type, so disable the pylint rule that catches that. - - -class Frame(object): - """Frame is the container for all Tool objects and state information.""" - - def __init__(self, config, setup, times, drivers, dryrun): - self.config = config - self.times = times - self.dryrun = dryrun - self.times['initialization'] = time.time() - self.setup = setup - self.tools = [] - self.states = {} - self.whitelist = [] - self.blacklist = [] - self.removal = [] - self.logger = logging.getLogger(__name__) - for driver in drivers[:]: - if (driver not in Bcfg2.Client.Tools.drivers and - isinstance(driver, str)): - self.logger.error("Tool driver %s is not available" % driver) - drivers.remove(driver) - - tclass = {} - for tool in drivers: - if not isinstance(tool, str): - tclass[time.time()] = tool - tool_class = "Bcfg2.Client.Tools.%s" % tool - try: - tclass[tool] = getattr(__import__(tool_class, globals(), - locals(), ['*']), - tool) - except ImportError: - continue - except: - self.logger.error("Tool %s unexpectedly failed to load" % tool, - exc_info=1) - - for tool in list(tclass.values()): - try: - self.tools.append(tool(self.logger, setup, config)) - except Bcfg2.Client.Tools.ToolInstantiationError: - continue - except: - self.logger.error("Failed to instantiate tool %s" % tool, - exc_info=1) - - for tool in self.tools[:]: - for conflict in getattr(tool, 'conflicts', []): - for item in self.tools: - if item.name == conflict: - self.tools.remove(item) - - self.logger.info("Loaded tool drivers:") - self.logger.info([tool.name for tool in self.tools]) - - deprecated = [tool.name for tool in self.tools if tool.deprecated] - if deprecated: - self.logger.warning("Loaded deprecated tool drivers:") - self.logger.warning(deprecated) - experimental = [tool.name for tool in self.tools if tool.experimental] - if experimental: - self.logger.info("Loaded experimental tool drivers:") - self.logger.info(experimental) - - # find entries not handled by any tools - self.unhandled = [entry for struct in config - for entry in struct - if entry not in self.handled] - - if self.unhandled: - self.logger.error("The following entries are not handled by any " - "tool:") - for entry in self.unhandled: - self.logger.error("%s:%s:%s" % (entry.tag, entry.get('type'), - entry.get('name'))) - - self.find_dups(config) - - pkgs = [(entry.get('name'), entry.get('origin')) - for struct in config - for entry in struct - if entry.tag == 'Package'] - if pkgs: - self.logger.debug("The following packages are specified in bcfg2:") - self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] is None]) - self.logger.debug("The following packages are prereqs added by " - "Packages:") - self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages']) - - def find_dups(self, config): - """ Find duplicate entries and warn about them """ - entries = dict() - for struct in config: - for entry in struct: - for tool in self.tools: - if tool.handlesEntry(entry): - pkey = tool.primarykey(entry) - if pkey in entries: - entries[pkey] += 1 - else: - entries[pkey] = 1 - multi = [e for e, c in entries.items() if c > 1] - if multi: - self.logger.debug("The following entries are included multiple " - "times:") - for entry in multi: - self.logger.debug(entry) - - def promptFilter(self, msg, entries): - """Filter a supplied list based on user input.""" - ret = [] - entries.sort(key=lambda e: e.tag + ":" + e.get('name')) - for entry in entries[:]: - if entry in self.unhandled: - # don't prompt for entries that can't be installed - continue - if 'qtext' in entry.attrib: - iprompt = entry.get('qtext') - else: - iprompt = msg % (entry.tag, entry.get('name')) - if prompt(iprompt): - ret.append(entry) - return ret - - def __getattr__(self, name): - if name in ['extra', 'handled', 'modified', '__important__']: - ret = [] - for tool in self.tools: - ret += getattr(tool, name) - return ret - elif name in self.__dict__: - return self.__dict__[name] - raise AttributeError(name) - - def InstallImportant(self): - """Install important entries - - We also process the decision mode stuff here because we want to prevent - non-whitelisted/blacklisted 'important' entries from being installed - prior to determining the decision mode on the client. - """ - # Need to process decision stuff early so that dryrun mode - # works with it - self.whitelist = [entry for entry in self.states - if not self.states[entry]] - if not self.setup['file']: - if self.setup['decision'] == 'whitelist': - dwl = self.setup['decision_list'] - w_to_rem = [e for e in self.whitelist - if not matches_white_list(e, dwl)] - if w_to_rem: - self.logger.info("In whitelist mode: " - "suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) - for e in w_to_rem]) - self.whitelist = [x for x in self.whitelist - if x not in w_to_rem] - elif self.setup['decision'] == 'blacklist': - b_to_rem = \ - [e for e in self.whitelist - if not passes_black_list(e, self.setup['decision_list'])] - if b_to_rem: - self.logger.info("In blacklist mode: " - "suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) - for e in b_to_rem]) - self.whitelist = [x for x in self.whitelist - if x not in b_to_rem] - - # take care of important entries first - if not self.dryrun: - parent_map = dict((c, p) - for p in self.config.getiterator() - for c in p) - for cfile in self.config.findall(".//Path"): - if (cfile.get('name') not in self.__important__ or - cfile.get('type') != 'file' or - cfile not in self.whitelist): - continue - parent = parent_map[cfile] - if ((parent.tag == "Bundle" and - ((self.setup['bundle'] and - parent.get("name") not in self.setup['bundle']) or - (self.setup['skipbundle'] and - parent.get("name") in self.setup['skipbundle']))) or - (parent.tag == "Independent" and - (self.setup['bundle'] or self.setup['skipindep']))): - continue - tools = [t for t in self.tools - if t.handlesEntry(cfile) and t.canVerify(cfile)] - if tools: - if (self.setup['interactive'] and not - self.promptFilter("Install %s: %s? (y/N):", [cfile])): - self.whitelist.remove(cfile) - continue - try: - self.states[cfile] = tools[0].InstallPath(cfile) - if self.states[cfile]: - tools[0].modified.append(cfile) - except: - self.logger.error("Unexpected tool failure", - exc_info=1) - cfile.set('qtext', '') - if tools[0].VerifyPath(cfile, []): - self.whitelist.remove(cfile) - - def Inventory(self): - """ - Verify all entries, - find extra entries, - and build up workqueues - - """ - # initialize all states - for struct in self.config.getchildren(): - for entry in struct.getchildren(): - self.states[entry] = False - for tool in self.tools: - try: - tool.Inventory(self.states) - except: - self.logger.error("%s.Inventory() call failed:" % tool.name, - exc_info=1) - - def Decide(self): # pylint: disable=R0912 - """Set self.whitelist based on user interaction.""" - iprompt = "Install %s: %s? (y/N): " - rprompt = "Remove %s: %s? (y/N): " - if self.setup['remove']: - if self.setup['remove'] == 'all': - self.removal = self.extra - elif self.setup['remove'].lower() == 'services': - self.removal = [entry for entry in self.extra - if entry.tag == 'Service'] - elif self.setup['remove'].lower() == 'packages': - self.removal = [entry for entry in self.extra - if entry.tag == 'Package'] - elif self.setup['remove'].lower() == 'users': - self.removal = [entry for entry in self.extra - if entry.tag in ['POSIXUser', 'POSIXGroup']] - - candidates = [entry for entry in self.states - if not self.states[entry]] - - if self.dryrun: - if self.whitelist: - self.logger.info("In dryrun mode: " - "suppressing entry installation for:") - self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) - for entry in self.whitelist]) - self.whitelist = [] - if self.removal: - self.logger.info("In dryrun mode: " - "suppressing entry removal for:") - self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) - for entry in self.removal]) - self.removal = [] - - # Here is where most of the work goes - # first perform bundle filtering - all_bundle_names = [b.get('name') - for b in self.config.findall('./Bundle')] - bundles = self.config.getchildren() - if self.setup['bundle']: - # warn if non-existent bundle given - for bundle in self.setup['bundle']: - if bundle not in all_bundle_names: - self.logger.info("Warning: Bundle %s not found" % bundle) - bundles = [b for b in bundles - if b.get('name') in self.setup['bundle']] - elif self.setup['indep']: - bundles = [b for b in bundles if b.tag != 'Bundle'] - if self.setup['skipbundle']: - # warn if non-existent bundle given - if not self.setup['bundle_quick']: - for bundle in self.setup['skipbundle']: - if bundle not in all_bundle_names: - self.logger.info("Warning: Bundle %s not found" % - bundle) - bundles = [b for b in bundles - if b.get('name') not in self.setup['skipbundle']] - if self.setup['skipindep']: - bundles = [b for b in bundles if b.tag == 'Bundle'] - - self.whitelist = [e for e in self.whitelist - if any(e in b for b in bundles)] - - # first process prereq actions - for bundle in bundles[:]: - if bundle.tag != 'Bundle': - continue - bmodified = len([item for item in bundle - if item in self.whitelist or - item in self.modified]) - actions = [a for a in bundle.findall('./Action') - if (a.get('timing') != 'post' and - (bmodified or a.get('when') == 'always'))] - # now we process all "pre" and "both" actions that are either - # always or the bundle has been modified - if self.setup['interactive']: - self.promptFilter(iprompt, actions) - self.DispatchInstallCalls(actions) - - # need to test to fail entries in whitelist - if False in [self.states[a] for a in actions]: - # then display bundles forced off with entries - self.logger.info("Bundle %s failed prerequisite action" % - (bundle.get('name'))) - bundles.remove(bundle) - b_to_remv = [ent for ent in self.whitelist if ent in bundle] - if b_to_remv: - self.logger.info("Not installing entries from Bundle %s" % - (bundle.get('name'))) - self.logger.info(["%s:%s" % (e.tag, e.get('name')) - for e in b_to_remv]) - for ent in b_to_remv: - self.whitelist.remove(ent) - - self.logger.debug("Installing entries in the following bundle(s):") - self.logger.debug(" %s" % ", ".join(b.get("name") for b in bundles - if b.get("name"))) - - if self.setup['interactive']: - self.whitelist = self.promptFilter(iprompt, self.whitelist) - self.removal = self.promptFilter(rprompt, self.removal) - - for entry in candidates: - if entry not in self.whitelist: - self.blacklist.append(entry) - - def DispatchInstallCalls(self, entries): - """Dispatch install calls to underlying tools.""" - for tool in self.tools: - handled = [entry for entry in entries if tool.canInstall(entry)] - if not handled: - continue - try: - tool.Install(handled, self.states) - except: - self.logger.error("%s.Install() call failed:" % tool.name, - exc_info=1) - - def Install(self): - """Install all entries.""" - self.DispatchInstallCalls(self.whitelist) - mods = self.modified - mbundles = [struct for struct in self.config.findall('Bundle') - if any(True for mod in mods if mod in struct)] - - if self.modified: - # Handle Bundle interdeps - if mbundles: - self.logger.info("The Following Bundles have been modified:") - self.logger.info([mbun.get('name') for mbun in mbundles]) - tbm = [(t, b) for t in self.tools for b in mbundles] - for tool, bundle in tbm: - try: - tool.Inventory(self.states, [bundle]) - except: - self.logger.error("%s.Inventory() call failed:" % - tool.name, - exc_info=1) - clobbered = [entry for bundle in mbundles for entry in bundle - if (not self.states[entry] and - entry not in self.blacklist)] - if clobbered: - self.logger.debug("Found clobbered entries:") - self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) - for entry in clobbered]) - if not self.setup['interactive']: - self.DispatchInstallCalls(clobbered) - - for bundle in self.config.findall('.//Bundle'): - if (self.setup['bundle'] and - bundle.get('name') not in self.setup['bundle']): - # prune out unspecified bundles when running with -b - continue - if bundle in mbundles: - self.logger.debug("Bundle %s was modified" % - bundle.get('name')) - func = "BundleUpdated" - else: - self.logger.debug("Bundle %s was not modified" % - bundle.get('name')) - func = "BundleNotUpdated" - for tool in self.tools: - try: - getattr(tool, func)(bundle, self.states) - except: - self.logger.error("%s.%s() call failed:" % - (tool.name, func), exc_info=1) - - def Remove(self): - """Remove extra entries.""" - for tool in self.tools: - extras = [entry for entry in self.removal - if tool.handlesEntry(entry)] - if extras: - try: - tool.Remove(extras) - except: - self.logger.error("%s.Remove() failed" % tool.name, - exc_info=1) - - def CondDisplayState(self, phase): - """Conditionally print tracing information.""" - self.logger.info('Phase: %s' % phase) - self.logger.info('Correct entries: %d' % - list(self.states.values()).count(True)) - self.logger.info('Incorrect entries: %d' % - list(self.states.values()).count(False)) - if phase == 'final' and list(self.states.values()).count(False): - for entry in sorted(self.states.keys(), key=lambda e: e.tag + ":" + - e.get('name')): - if not self.states[entry]: - etype = entry.get('type') - if etype: - self.logger.info("%s:%s:%s" % (entry.tag, etype, - entry.get('name'))) - else: - self.logger.info("%s:%s" % (entry.tag, - entry.get('name'))) - self.logger.info('Total managed entries: %d' % - len(list(self.states.values()))) - self.logger.info('Unmanaged entries: %d' % len(self.extra)) - if phase == 'final' and self.setup['extra']: - for entry in sorted(self.extra, key=lambda e: e.tag + ":" + - e.get('name')): - etype = entry.get('type') - if etype: - self.logger.info("%s:%s:%s" % (entry.tag, etype, - entry.get('name'))) - else: - self.logger.info("%s:%s" % (entry.tag, - entry.get('name'))) - - if ((list(self.states.values()).count(False) == 0) and not self.extra): - self.logger.info('All entries correct.') - - def ReInventory(self): - """Recheck everything.""" - if not self.dryrun and self.setup['kevlar']: - self.logger.info("Rechecking system inventory") - self.Inventory() - - def Execute(self): - """Run all methods.""" - self.Inventory() - self.times['inventory'] = time.time() - self.CondDisplayState('initial') - self.InstallImportant() - self.Decide() - self.Install() - self.times['install'] = time.time() - self.Remove() - self.times['remove'] = time.time() - if self.modified: - self.ReInventory() - self.times['reinventory'] = time.time() - self.times['finished'] = time.time() - self.CondDisplayState('final') - - def GenerateStats(self): - """Generate XML summary of execution statistics.""" - feedback = Bcfg2.Client.XML.Element("upload-statistics") - stats = Bcfg2.Client.XML.SubElement( - feedback, - 'Statistics', - total=str(len(self.states)), - version='2.0', - revision=self.config.get('revision', '-1')) - good_entries = [key for key, val in list(self.states.items()) if val] - good = len(good_entries) - stats.set('good', str(good)) - if any(not val for val in list(self.states.values())): - stats.set('state', 'dirty') - else: - stats.set('state', 'clean') - - # List bad elements of the configuration - for (data, ename) in [(self.modified, 'Modified'), - (self.extra, "Extra"), - (good_entries, "Good"), - ([entry for entry in self.states - if not self.states[entry]], "Bad")]: - container = Bcfg2.Client.XML.SubElement(stats, ename) - for item in data: - item.set('qtext', '') - container.append(copy.deepcopy(item)) - item.text = None - - timeinfo = Bcfg2.Client.XML.Element("OpStamps") - feedback.append(stats) - for (event, timestamp) in list(self.times.items()): - timeinfo.set(event, str(timestamp)) - stats.append(timeinfo) - return feedback diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py index 736325eab..679b4c52b 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Client/Proxy.py @@ -1,6 +1,10 @@ import re +import sys +import time import socket import logging +import Bcfg2.Options +from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus # The ssl module is provided by either Python 2.6 or a separate ssl # package that works on older versions of Python (see @@ -8,19 +12,10 @@ import logging # M2Crypto instead. try: import ssl - SSL_LIB = 'py26_ssl' SSL_ERROR = ssl.SSLError except ImportError: - from M2Crypto import SSL - import M2Crypto.SSL.Checker - SSL_LIB = 'm2crypto' - SSL_ERROR = SSL.SSLError - -import sys -import time + raise Exception("No SSL module support") -# Compatibility imports -from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus version = sys.version_info[:2] has_py26 = version >= (2, 6) @@ -64,6 +59,7 @@ class CertificateError(Exception): _orig_Method = xmlrpclib._Method + class RetryMethod(xmlrpclib._Method): """Method with error handling and retries built in.""" log = logging.getLogger('xmlrpc') @@ -122,10 +118,8 @@ class SSLHTTPConnection(httplib.HTTPConnection): implements SSL and related behaviors. """ - logger = logging.getLogger('Bcfg2.Proxy.SSLHTTPConnection') - def __init__(self, host, port=None, strict=None, timeout=90, key=None, - cert=None, ca=None, scns=None, protocol='xmlrpc/ssl'): + cert=None, ca=None, scns=None, protocol='xmlrpc/tlsv1'): """Initializes the `httplib.HTTPConnection` object and stores security parameters @@ -150,15 +144,15 @@ class SSLHTTPConnection(httplib.HTTPConnection): specify the same file as `cert` if using a file that contains both. See http://docs.python.org/library/ssl.html#ssl-certificates - for details. Required if using xmlrpc/ssl with client - certificate authentication. + for details. Required if using client certificate + authentication. cert : string, optional The file system path to the local endpoint's SSL certificate. May specify the same file as `cert` if using a file that contains both. See http://docs.python.org/library/ssl.html#ssl-certificates - for details. Required if using xmlrpc/ssl with client - certificate authentication. + for details. Required if using client certificate + authentication. ca : string, optional The file system path to a set of concatenated certificate authority certs, which are used to validate certificates @@ -179,6 +173,8 @@ class SSLHTTPConnection(httplib.HTTPConnection): # the strict parameter is deprecated. # HTTP 0.9-style "Simple Responses" are not supported anymore. httplib.HTTPConnection.__init__(self, host, port, timeout=timeout) + self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__, + self.__class__.__name__)) self.key = key self.cert = cert self.ca = ca @@ -187,15 +183,6 @@ class SSLHTTPConnection(httplib.HTTPConnection): self.timeout = timeout def connect(self): - """Initiates a connection using previously set attributes.""" - if SSL_LIB == 'py26_ssl': - self._connect_py26ssl() - elif SSL_LIB == 'm2crypto': - self._connect_m2crypto() - else: - raise Exception("No SSL module support") - - def _connect_py26ssl(self): """Initiates a connection using the ssl module.""" # check for IPv6 hostip = socket.getaddrinfo(self.host, @@ -217,12 +204,15 @@ class SSLHTTPConnection(httplib.HTTPConnection): other_side_required = ssl.CERT_REQUIRED else: other_side_required = ssl.CERT_NONE - self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") + self.logger.warning("No ca is specified. Cannot authenticate the " + "server with SSL.") if self.cert and not self.key: - self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot " + "authenticate this client with SSL.") self.cert = None if self.key and not self.cert: - self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") + self.logger.warning("SSL key specfied, but no cert. Cannot " + "authenticate this client with SSL.") self.key = None rawsock.settimeout(self.timeout) @@ -233,61 +223,17 @@ class SSLHTTPConnection(httplib.HTTPConnection): self.sock.connect((self.host, self.port)) peer_cert = self.sock.getpeercert() if peer_cert and self.scns: - scn = [x[0][1] for x in peer_cert['subject'] if x[0][0] == 'commonName'][0] + scn = [x[0][1] for x in peer_cert['subject'] + if x[0][0] == 'commonName'][0] if scn not in self.scns: raise CertificateError(scn) self.sock.closeSocket = True - def _connect_m2crypto(self): - """Initiates a connection using the M2Crypto module.""" - - if self.protocol == 'xmlrpc/ssl': - ctx = SSL.Context('sslv23') - elif self.protocol == 'xmlrpc/tlsv1': - ctx = SSL.Context('tlsv1') - else: - self.logger.error("Unknown protocol %s" % (self.protocol)) - raise Exception("unknown protocol %s" % self.protocol) - - if self.ca: - # Use the certificate authority to validate the cert - # presented by the server - ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, depth=9) - if ctx.load_verify_locations(self.ca) != 1: - raise Exception('No CA certs') - else: - self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") - - if self.cert and self.key: - # A cert/key is defined, use them to support client - # authentication to the server - ctx.load_cert(self.cert, self.key) - elif self.cert: - self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") - elif self.key: - self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") - - self.sock = SSL.Connection(ctx) - if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', self.host): - # host is ip address - try: - hostname = socket.gethostbyaddr(self.host)[0] - except: - # fall back to ip address - hostname = self.host - else: - hostname = self.host - try: - self.sock.connect((hostname, self.port)) - # automatically checks cert matches host - except M2Crypto.SSL.Checker.WrongHost: - wr = sys.exc_info()[1] - raise CertificateError(wr) - class XMLRPCTransport(xmlrpclib.Transport): - def __init__(self, key=None, cert=None, ca=None, protocol=None, - scns=None, use_datetime=0, timeout=90): + def __init__(self, key=None, cert=None, ca=None, + scns=None, use_datetime=0, timeout=90, + protocol='xmlrpc/tlsv1'): if hasattr(xmlrpclib.Transport, '__init__'): xmlrpclib.Transport.__init__(self, use_datetime) self.key = key @@ -344,27 +290,54 @@ class XMLRPCTransport(xmlrpclib.Transport): # pylint: enable=E1101 -def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, - allowedServerCNs=None, timeout=90, retries=3, delay=1, - protocol=None): - - """Constructs proxies to components. - - Arguments: - component_name -- name of the component to connect to - - Additional arguments are passed to the ServerProxy constructor. - - """ - xmlrpclib._Method.max_retries = retries - xmlrpclib._Method.retry_delay = delay - - if user and password: - method, path = urlparse(url)[:2] - newurl = "%s://%s:%s@%s" % (method, quote_plus(user, ''), - quote_plus(password, ''), path) - else: - newurl = url - ssl_trans = XMLRPCTransport(key, cert, ca, protocol, - allowedServerCNs, timeout=float(timeout)) - return xmlrpclib.ServerProxy(newurl, allow_none=True, transport=ssl_trans) +class ComponentProxy(xmlrpclib.ServerProxy): + """Constructs proxies to components. """ + + options = [ + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.password, Bcfg2.Options.Common.client_timeout, + Bcfg2.Options.Common.protocol, + Bcfg2.Options.PathOption( + '--ssl-key', cf=('communication', 'key'), dest="key", + help='Path to SSL key'), + Bcfg2.Options.PathOption( + cf=('communication', 'certificate'), dest="cert", + help='Path to SSL certificate'), + Bcfg2.Options.Option( + "-u", "--user", default="root", cf=('communication', 'user'), + help='The user to provide for authentication'), + Bcfg2.Options.Option( + "-R", "--retries", type=int, default=3, + cf=('communication', 'retries'), + help='The number of times to retry network communication'), + Bcfg2.Options.Option( + "-y", "--retry-delay", type=int, default=1, + cf=('communication', 'retry_delay'), + help='The time in seconds to wait between retries'), + Bcfg2.Options.Option( + '--ssl-cns', cf=('communication', 'serverCommonNames'), + type=Bcfg2.Options.Types.colon_list, + help='List of server commonNames')] + + def __init__(self): + RetryMethod.max_retries = Bcfg2.Options.setup.retries + RetryMethod.retry_delay = Bcfg2.Options.setup.retry_delay + + if Bcfg2.Options.setup.user and Bcfg2.Options.setup.password: + method, path = urlparse(Bcfg2.Options.setup.server)[:2] + url = "%s://%s:%s@%s" % ( + method, + quote_plus(Bcfg2.Options.setup.user, ''), + quote_plus(Bcfg2.Options.setup.password, ''), + path) + else: + url = Bcfg2.Options.setup.server + ssl_trans = XMLRPCTransport( + key=Bcfg2.Options.setup.key, + cert=Bcfg2.Options.setup.cert, + ca=Bcfg2.Options.setup.ca, + scns=Bcfg2.Options.setup.ssl_cns, + timeout=Bcfg2.Options.setup.client_timeout, + protocol=Bcfg2.Options.setup.protocol) + xmlrpclib.ServerProxy.__init__(self, url, + allow_none=True, transport=ssl_trans) diff --git a/src/lib/Bcfg2/Client/Tools/APK.py b/src/lib/Bcfg2/Client/Tools/APK.py index eb70fa676..7313f6fcc 100644 --- a/src/lib/Bcfg2/Client/Tools/APK.py +++ b/src/lib/Bcfg2/Client/Tools/APK.py @@ -12,11 +12,6 @@ class APK(Bcfg2.Client.Tools.PkgTool): pkgtype = 'apk' pkgtool = ("/sbin/apk add %s", ("%s", ["name"])) - def __init__(self, logger, setup, config): - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) - self.installed = {} - self.RefreshPackages() - def RefreshPackages(self): """Refresh memory hashes of packages.""" names = self.cmd.run("/sbin/apk info").stdout.splitlines() @@ -38,8 +33,6 @@ class APK(Bcfg2.Client.Tools.PkgTool): if entry.attrib['name'] in self.installed: if entry.attrib['version'] in \ ['auto', self.installed[entry.attrib['name']]]: - # if (not self.setup['quick'] and - # entry.get('verify', 'true') == 'true'): # FIXME: Does APK have any sort of verification mechanism? return True else: diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index 300c9bc51..1003ab842 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -4,31 +4,39 @@ import warnings warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) -import apt.cache import os import sys +import apt.cache +import Bcfg2.Options import Bcfg2.Client.Tools class APT(Bcfg2.Client.Tools.Tool): - """The Debian toolset implements package and service operations and - inherits the rest from Toolset.Toolset. + """The Debian toolset implements package and service operations + and inherits the rest from Toolset.Toolset.""" + + options = Bcfg2.Client.Tools.Tool.options + [ + Bcfg2.Options.PathOption( + cf=('APT', 'install_path'), + default='/usr', dest='apt_install_path', + help='Apt tools install path'), + Bcfg2.Options.PathOption( + cf=('APT', 'var_path'), default='/var', dest='apt_var_path', + help='Apt tools var path'), + Bcfg2.Options.PathOption( + cf=('APT', 'etc_path'), default='/etc', dest='apt_etc_path', + help='System etc path')] - """ - name = 'APT' __execs__ = [] __handles__ = [('Package', 'deb'), ('Path', 'ignore')] __req__ = {'Package': ['name', 'version'], 'Path': ['type']} - def __init__(self, logger, setup, config): - Bcfg2.Client.Tools.Tool.__init__(self, logger, setup, config) + def __init__(self, config): + Bcfg2.Client.Tools.Tool.__init__(self, config) - self.install_path = setup.get('apt_install_path', '/usr') - self.var_path = setup.get('apt_var_path', '/var') - self.etc_path = setup.get('apt_etc_path', '/etc') - self.debsums = '%s/bin/debsums' % self.install_path - self.aptget = '%s/bin/apt-get' % self.install_path - self.dpkg = '%s/bin/dpkg' % self.install_path + self.debsums = '%s/bin/debsums' % Bcfg2.Options.setup.apt_install_path + self.aptget = '%s/bin/apt-get' % Bcfg2.Options.setup.apt_install_path + self.dpkg = '%s/bin/dpkg' % Bcfg2.Options.setup.apt_install_path self.__execs__ = [self.debsums, self.aptget, self.dpkg] path_entries = os.environ['PATH'].split(':') @@ -40,30 +48,32 @@ class APT(Bcfg2.Client.Tools.Tool): '-o DPkg::Options::=--force-confmiss ' + \ '--reinstall ' + \ '--force-yes ' - if not self.setup['debug']: + if not Bcfg2.Options.setup.debug: self.pkgcmd += '-q=2 ' self.pkgcmd += '-y install %s' self.ignores = [entry.get('name') for struct in config for entry in struct if entry.tag == 'Path' and entry.get('type') == 'ignore'] - self.__important__ = self.__important__ + \ - ["%s/cache/debconf/config.dat" % self.var_path, - "%s/cache/debconf/templates.dat" % self.var_path, - '/etc/passwd', '/etc/group', - '%s/apt/apt.conf' % self.etc_path, - '%s/dpkg/dpkg.cfg' % self.etc_path] + \ - [entry.get('name') for struct in config for entry in struct - if entry.tag == 'Path' and - entry.get('name').startswith( - '%s/apt/sources.list' % self.etc_path)] + self.__important__ = self.__important__ + [ + "%s/cache/debconf/config.dat" % Bcfg2.Options.setup.apt_var_path, + "%s/cache/debconf/templates.dat" % + Bcfg2.Options.setup.apt_var_path, + '/etc/passwd', '/etc/group', + '%s/apt/apt.conf' % Bcfg2.Options.setup.apt_etc_path, + '%s/dpkg/dpkg.cfg' % Bcfg2.Options.setup.apt_etc_path] + \ + [entry.get('name') for struct in config + for entry in struct + if (entry.tag == 'Path' and + entry.get('name').startswith( + '%s/apt/sources.list' % + Bcfg2.Options.setup.apt_etc_path))] self.nonexistent = [entry.get('name') for struct in config - for entry in struct - if entry.tag == 'Path' and - entry.get('type') == 'nonexistent'] + for entry in struct if entry.tag == 'Path' + and entry.get('type') == 'nonexistent'] os.environ["DEBIAN_FRONTEND"] = 'noninteractive' self.actions = {} - if self.setup['kevlar'] and not self.setup['dryrun']: + if Bcfg2.Options.setup.kevlar and not Bcfg2.Options.setup.dry_run: self.cmd.run("%s --force-confold --configure --pending" % self.dpkg) self.cmd.run("%s clean" % self.aptget) @@ -199,7 +209,7 @@ class APT(Bcfg2.Client.Tools.Tool): return False else: # version matches - if not self.setup['quick'] \ + if not Bcfg2.Options.setup.quick \ and entry.get('verify', 'true') == 'true' \ and checksums: pkgsums = self.VerifyDebsums(entry, modlist) @@ -233,7 +243,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.modified += packages self.extra = self.FindExtra() - def Install(self, packages, states): + def Install(self, packages): # it looks like you can't install arbitrary versions of software # out of the pkg cache, we will still need to call apt-get ipkgs = [] @@ -285,10 +295,12 @@ class APT(Bcfg2.Client.Tools.Tool): self.logger.error("APT command failed") self.pkg_cache = apt.cache.Cache() self.extra = self.FindExtra() + states = dict() for package in packages: states[package] = self.VerifyPackage(package, [], checksums=False) if states[package]: self.modified.append(package) + return states def VerifyPath(self, entry, _): # pylint: disable=W0613 """Do nothing here since we only verify Path type=ignore.""" diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index 1edf4609b..ca0502b75 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -1,18 +1,14 @@ """Action driver""" -import os -import sys -import select import Bcfg2.Client.Tools -from Bcfg2.Compat import input # pylint: disable=W0622 +from Bcfg2.Utils import safe_input class Action(Bcfg2.Client.Tools.Tool): """Implement Actions""" name = 'Action' - __handles__ = [('PostInstall', None), ('Action', None)] - __req__ = {'PostInstall': ['name'], - 'Action': ['name', 'timing', 'when', 'command', 'status']} + __handles__ = [('Action', None)] + __req__ = {'Action': ['name', 'timing', 'when', 'command', 'status']} def RunAction(self, entry): """This method handles command execution and status return.""" @@ -22,19 +18,15 @@ class Action(Bcfg2.Client.Tools.Tool): shell = True shell_string = '(in shell) ' - if not self.setup['dryrun']: - if self.setup['interactive']: + if not Bcfg2.Options.setup.dry_run: + if Bcfg2.Options.setup.interactive: prompt = ('Run Action %s%s, %s: (y/N): ' % (shell_string, entry.get('name'), entry.get('command'))) - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], - 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - ans = input(prompt) + ans = safe_input(prompt) if ans not in ['y', 'Y']: return False - if self.setup['servicemode'] == 'build': + if Bcfg2.Options.setup.service_mode == 'build': if entry.get('build', 'true') == 'false': self.logger.debug("Action: Deferring execution of %s due " "to build mode" % entry.get('command')) @@ -55,39 +47,29 @@ class Action(Bcfg2.Client.Tools.Tool): """Actions always verify true.""" return True - def VerifyPostInstall(self, dummy, _): - """Actions always verify true.""" - return True - def InstallAction(self, entry): """Run actions as pre-checks for bundle installation.""" if entry.get('timing') != 'post': return self.RunAction(entry) return True - def InstallPostInstall(self, entry): - """ Install a deprecated PostInstall entry """ - self.logger.warning("Installing deprecated PostInstall entry %s" % - entry.get("name")) - return self.InstallAction(entry) - - def BundleUpdated(self, bundle, states): + def BundleUpdated(self, bundle): """Run postinstalls when bundles have been updated.""" - for postinst in bundle.findall("PostInstall"): - if not self._install_allowed(postinst): - continue - self.cmd.run(postinst.get('name')) + states = dict() for action in bundle.findall("Action"): if action.get('timing') in ['post', 'both']: if not self._install_allowed(action): continue states[action] = self.RunAction(action) + return states - def BundleNotUpdated(self, bundle, states): + def BundleNotUpdated(self, bundle): """Run Actions when bundles have not been updated.""" + states = dict() for action in bundle.findall("Action"): - if action.get('timing') in ['post', 'both'] and \ - action.get('when') != 'modified': + if (action.get('timing') in ['post', 'both'] and + action.get('when') != 'modified'): if not self._install_allowed(action): continue states[action] = self.RunAction(action) + return states diff --git a/src/lib/Bcfg2/Client/Tools/Blast.py b/src/lib/Bcfg2/Client/Tools/Blast.py index 2627c42fe..fd594b4f4 100644 --- a/src/lib/Bcfg2/Client/Tools/Blast.py +++ b/src/lib/Bcfg2/Client/Tools/Blast.py @@ -13,9 +13,9 @@ class Blast(Bcfg2.Client.Tools.SYSV.SYSV): __handles__ = [('Package', 'blast')] __req__ = {'Package': ['name', 'version', 'bname']} - def __init__(self, logger, setup, config): + def __init__(self, config): # dont use the sysv constructor - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) + Bcfg2.Client.Tools.PkgTool.__init__(self, config) noaskfile = tempfile.NamedTemporaryFile() self.noaskname = noaskfile.name try: diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index 4833f3f68..fab142a7c 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -3,7 +3,6 @@ """This is chkconfig support.""" import os - import Bcfg2.Client.Tools import Bcfg2.Client.XML @@ -96,15 +95,15 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): bootcmd = '/sbin/chkconfig %s %s' % (entry.get('name'), bootstatus) bootcmdrv = self.cmd.run(bootcmd).success - if self.setup['servicemode'] == 'disabled': + if Bcfg2.Options.setup.servicemode == 'disabled': # 'disabled' means we don't attempt to modify running svcs return bootcmdrv - buildmode = self.setup['servicemode'] == 'build' - if (entry.get('status') == 'on' and not buildmode) and \ - entry.get('current_status') == 'off': + buildmode = Bcfg2.Options.setup.servicemode == 'build' + if ((entry.get('status') == 'on' and not buildmode) and + entry.get('current_status') == 'off'): svccmdrv = self.start_service(entry) - elif (entry.get('status') == 'off' or buildmode) and \ - entry.get('current_status') == 'on': + elif ((entry.get('status') == 'off' or buildmode) and + entry.get('current_status') == 'on'): svccmdrv = self.stop_service(entry) else: svccmdrv = True # ignore status attribute diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index b544e44d4..53e5e7ec6 100644 --- a/src/lib/Bcfg2/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -3,6 +3,7 @@ import glob import os import re +import Bcfg2.Options import Bcfg2.Client.Tools # Debian squeeze and beyond uses a dependecy based boot sequence @@ -33,8 +34,8 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): if entry.get('sequence'): if (deb_version in DEBIAN_OLD_STYLE_BOOT_SEQUENCE or - deb_version.startswith('5') or - os.path.exists('/etc/init.d/.legacy-bootordering')): + deb_version.startswith('5') or + os.path.exists('/etc/init.d/.legacy-bootordering')): start_sequence = int(entry.get('sequence')) kill_sequence = 100 - start_sequence else: @@ -137,10 +138,10 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): bootcmd = '/usr/sbin/update-rc.d -f %s remove' % \ entry.get('name') bootcmdrv = self.cmd.run(bootcmd) - if self.setup['servicemode'] == 'disabled': + if Bcfg2.Options.setup.service_mode == 'disabled': # 'disabled' means we don't attempt to modify running svcs return bootcmdrv and seqcmdrv - buildmode = self.setup['servicemode'] == 'build' + buildmode = Bcfg2.Options.setup.service_mode == 'build' if (entry.get('status') == 'on' and not buildmode) and \ entry.get('current_status') == 'off': svccmdrv = self.start_service(entry) diff --git a/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py index 8ff26d8f3..24bc4cf36 100644 --- a/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py @@ -1,27 +1,143 @@ """FreeBSD Init Support for Bcfg2.""" -__revision__ = '$Rev$' - -# TODO -# - hardcoded path to ports rc.d -# - doesn't know about /etc/rc.d/ import os +import re +import Bcfg2.Options import Bcfg2.Client.Tools class FreeBSDInit(Bcfg2.Client.Tools.SvcTool): """FreeBSD service support for Bcfg2.""" name = 'FreeBSDInit' + __execs__ = ['/usr/sbin/service', '/usr/sbin/sysrc'] __handles__ = [('Service', 'freebsd')] __req__ = {'Service': ['name', 'status']} + rcvar_re = re.compile(r'^(?P<var>[a-z_]+_enable)="[A-Z]+"$') - def __init__(self, logger, cfg, setup): - Bcfg2.Client.Tools.Tool.__init__(self, logger, cfg, setup) - if os.uname()[0] != 'FreeBSD': - raise Bcfg2.Client.Tools.ToolInstantiationError + def get_svc_command(self, service, action): + return '/usr/sbin/service %s %s' % (service.get('name'), action) - def VerifyService(self, entry, _): + def verify_bootstatus(self, entry, bootstatus): + """Verify bootstatus for entry.""" + cmd = self.get_svc_command(entry, 'enabled') + current_bootstatus = bool(self.cmd.run(cmd)) + + if bootstatus == 'off': + if current_bootstatus: + entry.set('current_bootstatus', 'on') + return False + return True + elif not current_bootstatus: + entry.set('current_bootstatus', 'off') + return False return True - def get_svc_command(self, service, action): - return "/usr/local/etc/rc.d/%s %s" % (service.get('name'), action) + def check_service(self, entry): + # use 'onestatus' to enable status reporting for disabled services + cmd = self.get_svc_command(entry, 'onestatus') + return bool(self.cmd.run(cmd)) + + def stop_service(self, service): + # use 'onestop' to enable stopping of disabled services + self.logger.debug('Stopping service %s' % service.get('name')) + return self.cmd.run(self.get_svc_command(service, 'onestop')) + + + def VerifyService(self, entry, _): + """Verify Service status for entry.""" + entry.set('target_status', entry.get('status')) # for reporting + bootstatus = self.get_bootstatus(entry) + if bootstatus is None: + return True + current_bootstatus = self.verify_bootstatus(entry, bootstatus) + + if entry.get('status') == 'ignore': + # 'ignore' should verify + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True + + if svcstatus: + entry.set('current_status', 'on') + else: + entry.set('current_status', 'off') + + return current_bootstatus and current_svcstatus + + def InstallService(self, entry): + """Install Service entry.""" + self.logger.info("Installing Service %s" % (entry.get('name'))) + bootstatus = self.get_bootstatus(entry) + + # check if service exists + all_services_cmd = '/usr/sbin/service -l' + all_services = self.cmd.run(all_services_cmd).stdout.splitlines() + if entry.get('name') not in all_services: + self.logger.debug("Service %s does not exist" % entry.get('name')) + return False + + # get rcvar for service + vars = set() + rcvar_cmd = self.get_svc_command(entry, 'rcvar') + for line in self.cmd.run(rcvar_cmd).stdout.splitlines(): + match = self.rcvar_re.match(line) + if match: + vars.add(match.group('var')) + + if bootstatus is not None: + bootcmdrv = True + sysrcstatus = None + if bootstatus == 'on': + sysrcstatus = 'YES' + elif bootstatus == 'off': + sysrcstatus = 'NO' + if sysrcstatus is not None: + for var in vars: + if not self.cmd.run('/usr/sbin/sysrc %s="%s"' % (var, sysrcstatus)): + bootcmdrv = False + break + + if Bcfg2.Options.setup.service_mode == 'disabled': + # 'disabled' means we don't attempt to modify running svcs + return bootcmdrv + buildmode = Bcfg2.Options.setup.service_mode == 'build' + if (entry.get('status') == 'on' and not buildmode) and \ + entry.get('current_status') == 'off': + svccmdrv = self.start_service(entry) + elif (entry.get('status') == 'off' or buildmode) and \ + entry.get('current_status') == 'on': + svccmdrv = self.stop_service(entry) + else: + svccmdrv = True # ignore status attribute + return bootcmdrv and svccmdrv + else: + # when bootstatus is 'None', status == 'ignore' + return True + + def FindExtra(self): + """Find Extra FreeBSD Service entries.""" + specified = [entry.get('name') for entry in self.getSupportedEntries()] + extra = set() + for path in self.cmd.run("/usr/sbin/service -e").stdout.splitlines(): + name = os.path.basename(path) + if name not in specified: + extra.add(name) + return [Bcfg2.Client.XML.Element('Service', name=name, type='freebsd') + for name in list(extra)] + + def Remove(self, _): + """Remove extra service entries.""" + # Extra service removal is nonsensical + # Extra services need to be reflected in the config + return diff --git a/src/lib/Bcfg2/Client/Tools/IPS.py b/src/lib/Bcfg2/Client/Tools/IPS.py index 5be8527bf..0f82b1bc1 100644 --- a/src/lib/Bcfg2/Client/Tools/IPS.py +++ b/src/lib/Bcfg2/Client/Tools/IPS.py @@ -15,14 +15,13 @@ class IPS(Bcfg2.Client.Tools.PkgTool): __req__ = {'Package': ['name', 'version']} pkgtool = ('pkg install --no-refresh %s', ('%s', ['name'])) - def __init__(self, logger, setup, cfg): + def __init__(self, config): self.installed = {} self.pending_upgrades = set() self.image = image.Image() self.image.find_root('/', False) self.image.load_config() - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, cfg) - self.cfg = cfg + Bcfg2.Client.Tools.PkgTool.__init__(self, config) def RefreshPackages(self): self.installed = dict() diff --git a/src/lib/Bcfg2/Client/Tools/MacPorts.py b/src/lib/Bcfg2/Client/Tools/MacPorts.py index 40d90eec9..1e9847c42 100644 --- a/src/lib/Bcfg2/Client/Tools/MacPorts.py +++ b/src/lib/Bcfg2/Client/Tools/MacPorts.py @@ -12,11 +12,6 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool): pkgtype = 'macport' pkgtool = ('/opt/local/bin/port install %s', ('%s', ['name'])) - def __init__(self, logger, setup, config): - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) - self.installed = {} - self.RefreshPackages() - def RefreshPackages(self): """Refresh memory hashes of packages.""" pkgcache = self.cmd.run(["/opt/local/bin/port", @@ -44,8 +39,6 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool): if entry.attrib['name'] in self.installed: if (self.installed[entry.attrib['name']] == entry.attrib['version'] or entry.attrib['version'] == 'any'): - # if (not self.setup['quick'] and - # entry.get('verify', 'true') == 'true'): # FIXME: We should be able to check this once # http://trac.macports.org/ticket/15709 is implemented return True diff --git a/src/lib/Bcfg2/Client/Tools/OpenCSW.py b/src/lib/Bcfg2/Client/Tools/OpenCSW.py index 60e362e64..3ea9d835e 100644 --- a/src/lib/Bcfg2/Client/Tools/OpenCSW.py +++ b/src/lib/Bcfg2/Client/Tools/OpenCSW.py @@ -14,9 +14,9 @@ class OpenCSW(Bcfg2.Client.Tools.SYSV.SYSV): __handles__ = [('Package', 'opencsw')] __req__ = {'Package': ['name', 'version', 'bname']} - def __init__(self, logger, setup, config): + def __init__(self, config): # dont use the sysv constructor - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) + Bcfg2.Client.Tools.PkgTool.__init__(self, config) noaskfile = tempfile.NamedTemporaryFile() self.noaskname = noaskfile.name try: diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py index 8506f4bc7..fc4e16904 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py @@ -190,12 +190,12 @@ class POSIXAugeas(POSIXTool): :ref:`client-tools-augeas`. """ __req__ = ['name', 'mode', 'owner', 'group'] - def __init__(self, logger, setup, config): - POSIXTool.__init__(self, logger, setup, config) + def __init__(self, config): + POSIXTool.__init__(self, config) self._augeas = dict() # file tool for setting initial values of files that don't # exist - self.filetool = POSIXFile(logger, setup, config) + self.filetool = POSIXFile(config) def get_augeas(self, entry): """ Get an augeas object for the given entry. """ diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py index 9b84adad0..6237ccce2 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py @@ -13,7 +13,7 @@ class POSIXDevice(POSIXTool): if entry.get('dev_type') in ['block', 'char']: # check if major/minor are properly specified if (entry.get('major') is None or - entry.get('minor') is None): + entry.get('minor') is None): return False return True diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index b1bde1057..fc445e07c 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -3,9 +3,9 @@ import os import sys import stat -import time import difflib import tempfile +import Bcfg2.Options from Bcfg2.Client.Tools.POSIX.base import POSIXTool from Bcfg2.Compat import unicode, b64encode, b64decode # pylint: disable=W0622 @@ -43,7 +43,7 @@ class POSIXFile(POSIXTool): tempdata = entry.text if isinstance(tempdata, unicode) and unicode != str: try: - tempdata = tempdata.encode(self.setup['encoding']) + tempdata = tempdata.encode(Bcfg2.Options.setup.encoding) except UnicodeEncodeError: err = sys.exc_info()[1] self.logger.error("POSIX: Error encoding file %s: %s" % @@ -56,7 +56,7 @@ class POSIXFile(POSIXTool): if isinstance(tempdata, str) and str != unicode: tempdatasize = len(tempdata) else: - tempdatasize = len(tempdata.encode(self.setup['encoding'])) + tempdatasize = len(tempdata.encode(Bcfg2.Options.setup.encoding)) different = False content = None @@ -78,7 +78,7 @@ class POSIXFile(POSIXTool): content = open(entry.get('name')).read() except UnicodeDecodeError: content = open(entry.get('name'), - encoding=self.setup['encoding']).read() + encoding=Bcfg2.Options.setup.encoding).read() except IOError: self.logger.error("POSIX: Failed to read %s: %s" % (entry.get("name"), sys.exc_info()[1])) @@ -89,7 +89,7 @@ class POSIXFile(POSIXTool): self.logger.debug("POSIX: %s has incorrect contents" % entry.get("name")) self._get_diffs( - entry, interactive=self.setup['interactive'], + entry, interactive=Bcfg2.Options.setup.interactive, sensitive=entry.get('sensitive', 'false').lower() == 'true', is_binary=is_binary, content=content) return POSIXTool.verify(self, entry, modlist) and not different @@ -116,7 +116,7 @@ class POSIXFile(POSIXTool): os.fdopen(newfd, 'w').write(filedata) else: os.fdopen(newfd, 'wb').write( - filedata.encode(self.setup['encoding'])) + filedata.encode(Bcfg2.Options.setup.encoding)) except (OSError, IOError): err = sys.exc_info()[1] self.logger.error("POSIX: Failed to open temp file %s for writing " @@ -181,20 +181,20 @@ class POSIXFile(POSIXTool): (entry.get("name"), sys.exc_info()[1])) return False if not is_binary: - is_binary |= not self._is_string(content, self.setup['encoding']) + is_binary |= not self._is_string(content, + Bcfg2.Options.setup.encoding) if is_binary: # don't compute diffs if the file is binary prompt.append('Binary file, no printable diff') attrs['current_bfile'] = b64encode(content) else: + diff = self._diff(content, self._get_data(entry)[0], + filename=entry.get("name")) if interactive: - diff = self._diff(content, self._get_data(entry)[0], - difflib.unified_diff, - filename=entry.get("name")) if diff: - udiff = '\n'.join(l.rstrip('\n') for l in diff) + udiff = '\n'.join(diff) if hasattr(udiff, "decode"): - udiff = udiff.decode(self.setup['encoding']) + udiff = udiff.decode(Bcfg2.Options.setup.encoding) try: prompt.append(udiff) except UnicodeEncodeError: @@ -207,8 +207,6 @@ class POSIXFile(POSIXTool): prompt.append("Diff took too long to compute, no " "printable diff") if not sensitive: - diff = self._diff(content, self._get_data(entry)[0], - difflib.ndiff, filename=entry.get("name")) if diff: attrs["current_bdiff"] = b64encode("\n".join(diff)) else: @@ -219,28 +217,16 @@ class POSIXFile(POSIXTool): for attr, val in attrs.items(): entry.set(attr, val) - def _diff(self, content1, content2, difffunc, filename=None): - """ Return a diff of the two strings, as produced by difffunc. - warns after 5 seconds and times out after 30 seconds. """ - rv = [] - start = time.time() - longtime = False - for diffline in difffunc(content1.split('\n'), - content2.split('\n')): - now = time.time() - rv.append(diffline) - if now - start > 5 and not longtime: - if filename: - self.logger.info("POSIX: Diff of %s taking a long time" % - filename) - else: - self.logger.info("POSIX: Diff taking a long time") - longtime = True - elif now - start > 30: - if filename: - self.logger.error("POSIX: Diff of %s took too long; " - "giving up" % filename) - else: - self.logger.error("POSIX: Diff took too long; giving up") - return False - return rv + def _diff(self, content1, content2, filename=None): + """ Return a unified diff of the two strings """ + + if filename: + fromfile = "%s (on disk)" % filename + tofile = "%s (from bcfg2)" % filename + else: + fromfile = "" + tofile = "" + return difflib.unified_diff(content1.split('\n'), + content2.split('\n'), + fromfile=fromfile, + tofile=tofile) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py index f7251ca50..d67a68c8b 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py @@ -24,8 +24,8 @@ class POSIXNonexistent(POSIXTool): for struct in self.config.getchildren(): for el in struct.getchildren(): if (el.tag == 'Path' and - el.get('type') != 'nonexistent' and - el.get('name').startswith(ename)): + el.get('type') != 'nonexistent' and + el.get('name').startswith(ename)): self.logger.error('POSIX: Not removing %s. One or ' 'more files in this directory are ' 'specified in your configuration.' % diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py index 8d64cf84d..c27c7559d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py @@ -4,20 +4,31 @@ import os import re import sys import shutil -from datetime import datetime +import Bcfg2.Options import Bcfg2.Client.Tools +from datetime import datetime from Bcfg2.Compat import walk_packages from Bcfg2.Client.Tools.POSIX.base import POSIXTool class POSIX(Bcfg2.Client.Tools.Tool): """POSIX File support code.""" - name = 'POSIX' - def __init__(self, logger, setup, config): - Bcfg2.Client.Tools.Tool.__init__(self, logger, setup, config) - self.ppath = setup['ppath'] - self.max_copies = setup['max_copies'] + options = Bcfg2.Client.Tools.Tool.options + [ + Bcfg2.Options.PathOption( + cf=('paranoid', 'path'), default='/var/cache/bcfg2', + dest='paranoid_path', + help='Specify path for paranoid file backups'), + Bcfg2.Options.Option( + cf=('paranoid', 'max_copies'), default=1, type=int, + dest='paranoid_copies', + help='Specify the number of paranoid copies you want'), + Bcfg2.Options.BooleanOption( + '-P', '--paranoid', cf=('client', 'paranoid'), + help='Make automatic backups of config files')] + + def __init__(self, config): + Bcfg2.Client.Tools.Tool.__init__(self, config) self._handlers = self._load_handlers() self.logger.debug("POSIX: Handlers loaded: %s" % (", ".join(self._handlers.keys()))) @@ -56,7 +67,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): if POSIXTool in hdlr.__mro__: # figure out what entry type this handler handles etype = hdlr.__name__[5:].lower() - rv[etype] = hdlr(self.logger, self.setup, self.config) + rv[etype] = hdlr(self.config) return rv def canVerify(self, entry): @@ -92,7 +103,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): self.logger.debug("POSIX: Verifying entry %s:%s:%s" % (entry.tag, entry.get("type"), entry.get("name"))) ret = self._handlers[entry.get("type")].verify(entry, modlist) - if self.setup['interactive'] and not ret: + if Bcfg2.Options.setup.interactive and not ret: entry.set('qtext', '%s\nInstall %s %s: (y/N) ' % (entry.get('qtext', ''), @@ -106,35 +117,39 @@ class POSIX(Bcfg2.Client.Tools.Tool): bkupnam + r'_\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}$') # current list of backups for this file try: - bkuplist = [f for f in os.listdir(self.ppath) if - bkup_re.match(f)] + bkuplist = [f + for f in os.listdir(Bcfg2.Options.setup.paranoid_path) + if bkup_re.match(f)] except OSError: err = sys.exc_info()[1] self.logger.error("POSIX: Failed to create backup list in %s: %s" % - (self.ppath, err)) + (Bcfg2.Options.setup.paranoid_path, err)) return bkuplist.sort() - while len(bkuplist) >= int(self.max_copies): + while len(bkuplist) >= int(Bcfg2.Options.setup.paranoid_copies): # remove the oldest backup available oldest = bkuplist.pop(0) self.logger.info("POSIX: Removing old backup %s" % oldest) try: - os.remove(os.path.join(self.ppath, oldest)) + os.remove(os.path.join(Bcfg2.Options.setup.paranoid_path, + oldest)) except OSError: err = sys.exc_info()[1] - self.logger.error("POSIX: Failed to remove old backup %s: %s" % - (os.path.join(self.ppath, oldest), err)) + self.logger.error( + "POSIX: Failed to remove old backup %s: %s" % + (os.path.join(Bcfg2.Options.setup.paranoid_path, oldest), + err)) def _paranoid_backup(self, entry): """ Take a backup of the specified entry for paranoid mode """ if (entry.get("paranoid", 'false').lower() == 'true' and - self.setup.get("paranoid", False) and - entry.get('current_exists', 'true') == 'true' and - not os.path.isdir(entry.get("name"))): + Bcfg2.Options.setup.paranoid and + entry.get('current_exists', 'true') == 'true' and + not os.path.isdir(entry.get("name"))): self._prune_old_backups(entry) bkupnam = "%s_%s" % (entry.get('name').replace('/', '_'), datetime.isoformat(datetime.now())) - bfile = os.path.join(self.ppath, bkupnam) + bfile = os.path.join(Bcfg2.Options.setup.paranoid_path, bkupnam) try: shutil.copy(entry.get('name'), bfile) self.logger.info("POSIX: Backup of %s saved to %s" % diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 3d1358ce0..8895eaae1 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -105,23 +105,23 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): path = entry.get("name") rv = True - if entry.get("owner") and entry.get("group"): - try: - self.logger.debug("POSIX: Setting ownership of %s to %s:%s" % - (path, - self._norm_entry_uid(entry), - self._norm_entry_gid(entry))) - os.chown(path, self._norm_entry_uid(entry), - self._norm_entry_gid(entry)) - except KeyError: - self.logger.error('POSIX: Failed to change ownership of %s' % - path) - rv = False - os.chown(path, 0, 0) - except OSError: - self.logger.error('POSIX: Failed to change ownership of %s' % - path) - rv = False + if os.geteuid() == 0: + if entry.get("owner") and entry.get("group"): + try: + self.logger.debug("POSIX: Setting ownership of %s to %s:%s" + % (path, + self._norm_entry_uid(entry), + self._norm_entry_gid(entry))) + os.chown(path, self._norm_entry_uid(entry), + self._norm_entry_gid(entry)) + except (OSError, KeyError): + self.logger.error('POSIX: Failed to change ownership of %s' + % path) + rv = False + if sys.exc_info()[0] == KeyError: + os.chown(path, 0, 0) + else: + self.logger.debug("POSIX: Run as non-root, not setting ownership") if entry.get("mode"): wanted_mode = int(entry.get('mode'), 8) @@ -518,7 +518,8 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): (path, attrib['current_group'], entry.get('group'))) if (wanted_mode and - oct_mode(int(attrib['current_mode'], 8)) != oct_mode(wanted_mode)): + oct_mode(int(attrib['current_mode'], 8)) != + oct_mode(wanted_mode)): errors.append("Permissions for path %s are incorrect. " "Current permissions are %s but should be %s" % (path, attrib['current_mode'], entry.get('mode'))) @@ -543,7 +544,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: wanted_secontext = entry.get("secontext") if (wanted_secontext and - attrib['current_secontext'] != wanted_secontext): + attrib['current_secontext'] != wanted_secontext): errors.append("SELinux context for path %s is incorrect. " "Current context is %s but should be %s" % (path, attrib['current_secontext'], diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index bbae7abcc..7200b0fc2 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -3,14 +3,38 @@ and groupadd/mod/del """ import pwd import grp +import Bcfg2.Options import Bcfg2.Client.XML import Bcfg2.Client.Tools from Bcfg2.Utils import PackedDigitRange +def uid_range_type(val): + """ Option type to unpack a list of numerical ranges """ + return PackedDigitRange(*Bcfg2.Options.Types.comma_list(val)) + + class POSIXUsers(Bcfg2.Client.Tools.Tool): """ A tool to handle creating users and groups with useradd/mod/del and groupadd/mod/del """ + options = Bcfg2.Client.Tools.Tool.options + [ + Bcfg2.Options.Option( + cf=('POSIXUsers', 'uid_whitelist'), default=[], + type=uid_range_type, + help="UID ranges the POSIXUsers tool will manage"), + Bcfg2.Options.Option( + cf=('POSIXUsers', 'gid_whitelist'), default=[], + type=uid_range_type, + help="GID ranges the POSIXUsers tool will manage"), + Bcfg2.Options.Option( + cf=('POSIXUsers', 'uid_blacklist'), default=[], + type=uid_range_type, + help="UID ranges the POSIXUsers tool will not manage"), + Bcfg2.Options.Option( + cf=('POSIXUsers', 'gid_blacklist'), default=[], + type=uid_range_type, + help="GID ranges the POSIXUsers tool will not manage")] + __execs__ = ['/usr/sbin/useradd', '/usr/sbin/usermod', '/usr/sbin/userdel', '/usr/sbin/groupadd', '/usr/sbin/groupmod', '/usr/sbin/groupdel'] @@ -18,7 +42,6 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): ('POSIXGroup', None)] __req__ = dict(POSIXUser=['name'], POSIXGroup=['name']) - experimental = True #: A mapping of XML entry attributes to the indexes of #: corresponding values in the get{pw|gr}all data structures @@ -30,25 +53,15 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): #: user or group id_mapping = dict(POSIXUser="uid", POSIXGroup="gid") - def __init__(self, logger, setup, config): - Bcfg2.Client.Tools.Tool.__init__(self, logger, setup, config) + def __init__(self, config): + Bcfg2.Client.Tools.Tool.__init__(self, config) self.set_defaults = dict(POSIXUser=self.populate_user_entry, POSIXGroup=lambda g: g) self._existing = None - self._whitelist = dict(POSIXUser=None, POSIXGroup=None) - self._blacklist = dict(POSIXUser=None, POSIXGroup=None) - if self.setup['posix_uid_whitelist']: - self._whitelist['POSIXUser'] = \ - PackedDigitRange(*self.setup['posix_uid_whitelist']) - else: - self._blacklist['POSIXUser'] = \ - PackedDigitRange(*self.setup['posix_uid_blacklist']) - if self.setup['posix_gid_whitelist']: - self._whitelist['POSIXGroup'] = \ - PackedDigitRange(*self.setup['posix_gid_whitelist']) - else: - self._blacklist['POSIXGroup'] = \ - PackedDigitRange(*self.setup['posix_gid_blacklist']) + self._whitelist = dict(POSIXUser=Bcfg2.Options.setup.uid_whitelist, + POSIXGroup=Bcfg2.Options.setup.gid_whitelist) + self._blacklist = dict(POSIXUser=Bcfg2.Options.setup.uid_blacklist, + POSIXGroup=Bcfg2.Options.setup.gid_blacklist) @property def existing(self): @@ -66,7 +79,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): defined, and the uid/gid is in that whitelist; or b) no whitelist is defined, and the uid/gid is not in the blacklist. """ - if self._whitelist[tag] is None: + if not self._whitelist[tag]: return eid not in self._blacklist[tag] else: return eid in self._whitelist[tag] @@ -87,7 +100,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): return False return True - def Inventory(self, states, structures=None): + def Inventory(self, structures=None): if not structures: structures = self.config.getchildren() # we calculate a list of all POSIXUser and POSIXGroup entries, @@ -107,7 +120,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): (group, entry.get("name"))) struct.append(Bcfg2.Client.XML.Element("POSIXGroup", name=group)) - return Bcfg2.Client.Tools.Tool.Inventory(self, states, structures) + return Bcfg2.Client.Tools.Tool.Inventory(self, structures) + Inventory.__doc__ = Bcfg2.Client.Tools.Tool.Inventory.__doc__ def FindExtra(self): extra = [] @@ -165,7 +179,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): % (entry.tag, entry.get("name"), actual, expected)])) rv = False - if self.setup['interactive'] and not rv: + if Bcfg2.Options.setup.interactive and not rv: entry.set('qtext', '%s\nInstall %s %s: (y/N) ' % (entry.get('qtext', ''), entry.tag, entry.get('name'))) @@ -174,7 +188,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): def VerifyPOSIXGroup(self, entry, _): """ Verify a POSIXGroup entry """ rv = self._verify(entry) - if self.setup['interactive'] and not rv: + if Bcfg2.Options.setup.interactive and not rv: entry.set('qtext', '%s\nInstall %s %s: (y/N) ' % (entry.get('qtext', ''), entry.tag, entry.get('name'))) @@ -191,7 +205,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): for attr, idx in self.attr_mapping[entry.tag].items(): val = str(self.existing[entry.tag][entry.get("name")][idx]) entry.set("current_%s" % - attr, val.decode(self.setup['encoding'])) + attr, val.decode(Bcfg2.Options.setup.encoding)) if attr in ["uid", "gid"]: if entry.get(attr) is None: # no uid/gid specified, so we let the tool @@ -213,7 +227,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): entry.set('qtext', "\n".join([entry.get('qtext', '')] + errors)) return len(errors) == 0 - def Install(self, entries, states): + def Install(self, entries): + states = dict() for entry in entries: # install groups first, so that all groups exist for # users that might need them @@ -223,6 +238,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.tag == 'POSIXUser': states[entry] = self._install(entry) self._existing = None + return states def _install(self, entry): """ add or modify a user or group using the appropriate command """ diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index b931fe028..b82b905e7 100644 --- a/src/lib/Bcfg2/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py @@ -13,11 +13,6 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): pkgtype = 'pacman' pkgtool = ("/usr/bin/pacman --needed --noconfirm --noprogressbar") - def __init__(self, logger, setup, config): - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) - self.installed = {} - self.RefreshPackages() - def RefreshPackages(self): '''Refresh memory hashes of packages''' self.installed = {} @@ -42,8 +37,6 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): return True elif self.installed[entry.attrib['name']] == \ entry.attrib['version']: - # if (not self.setup['quick'] and - # entry.get('verify', 'true') == 'true'): # FIXME: need to figure out if pacman # allows you to verify packages return True @@ -65,7 +58,7 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): self.RefreshPackages() self.extra = self.FindExtra() - def Install(self, packages, states): + def Install(self, packages): ''' Pacman Install ''' diff --git a/src/lib/Bcfg2/Client/Tools/Pkgng.py b/src/lib/Bcfg2/Client/Tools/Pkgng.py new file mode 100644 index 000000000..cd70d662d --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/Pkgng.py @@ -0,0 +1,226 @@ +"""This is the Bcfg2 support for pkg.""" + +import os +import Bcfg2.Options +import Bcfg2.Client.Tools + + +class Pkgng(Bcfg2.Client.Tools.Tool): + """Support for pkgng packages on FreeBSD.""" + + options = Bcfg2.Client.Tools.Tool.options + [ + Bcfg2.Options.PathOption( + cf=('Pkgng', 'path'), + default='/usr/sbin/pkg', dest='pkg_path', + help='Pkgng tool path')] + + name = 'Pkgng' + __execs__ = [] + __handles__ = [('Package', 'pkgng'), ('Path', 'ignore')] + __req__ = {'Package': ['name', 'version'], 'Path': ['type']} + + def __init__(self, config): + Bcfg2.Client.Tools.Tool.__init__(self, config) + + self.pkg = Bcfg2.Options.setup.pkg_path + self.__execs__ = [self.pkg] + + self.pkgcmd = self.pkg + ' install -fy' + if not Bcfg2.Options.setup.debug: + self.pkgcmd += ' -q' + self.pkgcmd += ' %s' + + self.ignores = [entry.get('name') for struct in config + for entry in struct + if entry.tag == 'Path' and + entry.get('type') == 'ignore'] + + self.__important__ = self.__important__ + \ + [entry.get('name') for struct in config + for entry in struct + if (entry.tag == 'Path' and + entry.get('name').startswith('/etc/pkg/'))] + self.nonexistent = [entry.get('name') for struct in config + for entry in struct if entry.tag == 'Path' + and entry.get('type') == 'nonexistent'] + self.actions = {} + self.pkg_cache = {} + + try: + self._load_pkg_cache() + except OSError: + raise Bcfg2.Client.Tools.ToolInstantiationError + + def _load_pkg_cache(self): + """Cache the version of all currently installed packages.""" + self.pkg_cache = {} + output = self.cmd.run([self.pkg, 'query', '-a', '%n %v']).stdout + for line in output.splitlines(): + parts = line.split(' ') + name = ' '.join(parts[:-1]) + self.pkg_cache[name] = parts[-1] + + def FindExtra(self): + """Find extra packages.""" + packages = [entry.get('name') for entry in self.getSupportedEntries()] + extras = [(name, value) for (name, value) in self.pkg_cache.items() + if name not in packages] + return [Bcfg2.Client.XML.Element('Package', name=name, + type='pkgng', version=version) + for (name, version) in extras] + + def VerifyChecksums(self, entry, modlist): + """Verify the checksum of the files, owned by a package.""" + output = self.cmd.run([self.pkg, 'check', '-s', + entry.get('name')]).stdout.splitlines() + files = [] + for item in output: + if "checksum mismatch" in item: + files.append(item.split()[-1]) + elif "No such file or directory" in item: + continue + else: + self.logger.error("Got Unsupported pattern %s " + "from pkg check" % item) + + files = list(set(files) - set(self.ignores)) + # We check if there is file in the checksum to do + if files: + # if files are found there we try to be sure our modlist is sane + # with erroneous symlinks + modlist = [os.path.realpath(filename) for filename in modlist] + bad = [filename for filename in files if filename not in modlist] + if bad: + self.logger.debug("It is suggested that you either manage " + "these files, revert the changes, or ignore " + "false failures:") + self.logger.info("Package %s failed validation. Bad files " + "are:" % entry.get('name')) + self.logger.info(bad) + entry.set('qtext', + "Reinstall Package %s-%s to fix failing files? " + "(y/N) " % (entry.get('name'), entry.get('version'))) + return False + return True + + def _get_candidate_versions(self, name): + """ + Get versions of the specified package name available for + installation from the configured remote repositories. + """ + output = self.cmd.run([self.pkg, 'search', '-Qversion', '-q', + '-Sname', '-e', name]).stdout.splitlines() + versions = [] + for line in output: + versions.append(line) + + if len(versions) == 0: + return None + + return sorted(versions) + + def VerifyPackage(self, entry, modlist, checksums=True): + """Verify package for entry.""" + if 'version' not in entry.attrib: + self.logger.info("Cannot verify unversioned package %s" % + (entry.attrib['name'])) + return False + + pkgname = entry.get('name') + if pkgname not in self.pkg_cache: + self.logger.info("Package %s not installed" % (entry.get('name'))) + entry.set('current_exists', 'false') + return False + + installed_version = self.pkg_cache[pkgname] + candidate_versions = self._get_candidate_versions(pkgname) + if candidate_versions is not None: + candidate_version = candidate_versions[0] + else: + self.logger.error("Package %s is installed but no candidate" + "version was found." % (entry.get('name'))) + return False + + if entry.get('version').startswith('auto'): + desired_version = candidate_version + entry.set('version', "auto: %s" % desired_version) + elif entry.get('version').startswith('any'): + desired_version = installed_version + entry.set('version', "any: %s" % desired_version) + else: + desired_version = entry.get('version') + + if desired_version != installed_version: + entry.set('current_version', installed_version) + entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % + (entry.get('name'), entry.get('current_version'), + desired_version)) + return False + else: + # version matches + if (not Bcfg2.Options.setup.quick and + entry.get('verify', 'true') == 'true' + and checksums): + pkgsums = self.VerifyChecksums(entry, modlist) + return pkgsums + return True + + def Remove(self, packages): + """Deal with extra configuration detected.""" + pkgnames = " ".join([pkg.get('name') for pkg in packages]) + if len(packages) > 0: + self.logger.info('Removing packages:') + self.logger.info(pkgnames) + self.cmd.run([self.pkg, 'delete', '-y', pkgnames]) + self._load_pkg_cache() + self.modified += packages + self.extra = self.FindExtra() + + def Install(self, packages): + ipkgs = [] + bad_pkgs = [] + for pkg in packages: + versions = self._get_candidate_versions(pkg.get('name')) + if versions is None: + self.logger.error("pkg has no information about package %s" % + (pkg.get('name'))) + continue + + if pkg.get('version').startswith('auto') or \ + pkg.get('version').startswith('any'): + ipkgs.append("%s-%s" % (pkg.get('name'), versions[0])) + continue + + if pkg.get('version') in versions: + ipkgs.append("%s-%s" % (pkg.get('name'), pkg.get('version'))) + continue + else: + self.logger.error("Package %s: desired version %s not in %s" % + (pkg.get('name'), pkg.get('version'), + versions)) + bad_pkgs.append(pkg.get('name')) + + if bad_pkgs: + self.logger.error("Cannot find correct versions of packages:") + self.logger.error(bad_pkgs) + if not ipkgs: + return + if not self.cmd.run(self.pkgcmd % (" ".join(ipkgs))): + self.logger.error("pkg command failed") + self._load_pkg_cache() + self.extra = self.FindExtra() + mark = [] + states = dict() + for package in packages: + states[package] = self.VerifyPackage(package, [], checksums=False) + if states[package]: + self.modified.append(package) + if package.get('origin') == 'Packages': + mark.append(package.get('name')) + if mark: + self.cmd.run([self.pkg, 'set', '-A1', '-y'] + mark) + return states + + def VerifyPath(self, _entry, _): + """Do nothing here since we only verify Path type=ignore.""" + return True diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py index 78ccb2d37..5c092f46b 100644 --- a/src/lib/Bcfg2/Client/Tools/Portage.py +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -5,9 +5,13 @@ import Bcfg2.Client.Tools class Portage(Bcfg2.Client.Tools.PkgTool): - """The Gentoo toolset implements package and service operations and - inherits the rest from Toolset.Toolset.""" - name = 'Portage' + """The Gentoo toolset implements package and service operations + and inherits the rest from Tools.Tool.""" + + options = Bcfg2.Client.Tools.PkgTool.options + [ + Bcfg2.Options.BooleanOption( + cf=('Portage', 'binpkgonly'), help='Portage binary packages only')] + __execs__ = ['/usr/bin/emerge', '/usr/bin/equery'] __handles__ = [('Package', 'ebuild')] __req__ = {'Package': ['name', 'version']} @@ -17,17 +21,15 @@ class Portage(Bcfg2.Client.Tools.PkgTool): 'version'])) pkgtool = ('emerge %s', ('=%s-%s', ['name', 'version'])) - def __init__(self, logger, cfg, setup): + def __init__(self, config): self._initialised = False - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, cfg, setup) + Bcfg2.Client.Tools.PkgTool.__init__(self, config) self._initialised = True self.__important__ = self.__important__ + ['/etc/make.conf'] self._pkg_pattern = re.compile(r'(.*)-(\d.*)') self._ebuild_pattern = re.compile('(ebuild|binary)') - self.cfg = cfg self.installed = {} - self._binpkgonly = self.setup.get('portage_binpkgonly', False) - if self._binpkgonly: + if Bcfg2.Options.setup.binpkgonly: self.pkgtool = self._binpkgtool self.RefreshPackages() @@ -62,9 +64,9 @@ class Portage(Bcfg2.Client.Tools.PkgTool): version = self.installed[entry.get('name')] entry.set('current_version', version) - if not self.setup['quick']: + if not Bcfg2.Options.setup.quick: if ('verify' not in entry.attrib or - entry.get('verify').lower() == 'true'): + entry.get('verify').lower() == 'true'): # Check the package if: # - Not running in quick mode diff --git a/src/lib/Bcfg2/Client/Tools/RPM.py b/src/lib/Bcfg2/Client/Tools/RPM.py index a4dd2b730..464b7e389 100644 --- a/src/lib/Bcfg2/Client/Tools/RPM.py +++ b/src/lib/Bcfg2/Client/Tools/RPM.py @@ -1,12 +1,1140 @@ """Bcfg2 Support for RPMS""" -import os.path +import os import rpm -import rpmtools import Bcfg2.Client.Tools +import grp +import optparse +import pwd +import stat +import sys +try: + import hashlib + py24compat = False +except ImportError: + # FIXME: Remove when client python dep is 2.5 or greater + py24compat = True + import md5 + +# Determine what prelink tools we have available. +# The isprelink module is a python extension that examines the ELF headers +# to see if the file has been prelinked. If it is not present a lot of files +# are unnecessarily run through the prelink command. +try: + from isprelink import * + isprelink_imported = True +except ImportError: + isprelink_imported = False + +# If the prelink command is installed on the system then we need to do +# prelink -y on files. +if os.access('/usr/sbin/prelink', os.X_OK): + prelink_exists = True +else: + prelink_exists = False + +# If we don't have isprelink then we will use the prelink configuration file to +# filter what we have to put through prelink -y. +import re +blacklist = [] +whitelist = [] +try: + f = open('/etc/prelink.conf', mode='r') + for line in f: + if line.startswith('#'): + continue + option, pattern = line.split() + if pattern.startswith('*.'): + pattern = pattern.replace('*.', '\.') + pattern += '$' + elif pattern.startswith('/'): + pattern = '^' + pattern + if option == '-b': + blacklist.append(pattern) + elif option == '-l': + whitelist.append(pattern) + f.close() +except IOError: + pass + +blacklist_re = re.compile('|'.join(blacklist)) +whitelist_re = re.compile('|'.join(whitelist)) + +# Flags that are not defined in rpm-python. +# They are defined in lib/rpmcli.h +# Bit(s) for verifyFile() attributes. +# +RPMVERIFY_NONE = 0 +RPMVERIFY_MD5 = 1 # 1 << 0 # from %verify(md5) +RPMVERIFY_FILESIZE = 2 # 1 << 1 # from %verify(size) +RPMVERIFY_LINKTO = 4 # 1 << 2 # from %verify(link) +RPMVERIFY_USER = 8 # 1 << 3 # from %verify(user) +RPMVERIFY_GROUP = 16 # 1 << 4 # from %verify(group) +RPMVERIFY_MTIME = 32 # 1 << 5 # from %verify(mtime) +RPMVERIFY_MODE = 64 # 1 << 6 # from %verify(mode) +RPMVERIFY_RDEV = 128 # 1 << 7 # from %verify(rdev) +RPMVERIFY_CONTEXTS = 32768 # (1 << 15) # from --nocontexts +RPMVERIFY_READLINKFAIL = 268435456 # (1 << 28) # readlink failed +RPMVERIFY_READFAIL = 536870912 # (1 << 29) # file read failed +RPMVERIFY_LSTATFAIL = 1073741824 # (1 << 30) # lstat failed +RPMVERIFY_LGETFILECONFAIL = 2147483648 # (1 << 31) # lgetfilecon failed + +RPMVERIFY_FAILURES = \ + (RPMVERIFY_LSTATFAIL | RPMVERIFY_READFAIL | + RPMVERIFY_READLINKFAIL | RPMVERIFY_LGETFILECONFAIL) + +# Bit(s) to control rpm_verify() operation. +# +VERIFY_DEFAULT = 0, # /*!< */ +VERIFY_MD5 = 1 << 0 # /*!< from --nomd5 */ +VERIFY_SIZE = 1 << 1 # /*!< from --nosize */ +VERIFY_LINKTO = 1 << 2 # /*!< from --nolinkto */ +VERIFY_USER = 1 << 3 # /*!< from --nouser */ +VERIFY_GROUP = 1 << 4 # /*!< from --nogroup */ +VERIFY_MTIME = 1 << 5 # /*!< from --nomtime */ +VERIFY_MODE = 1 << 6 # /*!< from --nomode */ +VERIFY_RDEV = 1 << 7 # /*!< from --nodev */ +# /* bits 8-14 unused, reserved for rpmVerifyAttrs */ +VERIFY_CONTEXTS = 1 << 15 # /*!< verify: from --nocontexts */ +VERIFY_FILES = 1 << 16 # /*!< verify: from --nofiles */ +VERIFY_DEPS = 1 << 17 # /*!< verify: from --nodeps */ +VERIFY_SCRIPT = 1 << 18 # /*!< verify: from --noscripts */ +VERIFY_DIGEST = 1 << 19 # /*!< verify: from --nodigest */ +VERIFY_SIGNATURE = 1 << 20 # /*!< verify: from --nosignature */ +VERIFY_PATCHES = 1 << 21 # /*!< verify: from --nopatches */ +VERIFY_HDRCHK = 1 << 22 # /*!< verify: from --nohdrchk */ +VERIFY_FOR_LIST = 1 << 23 # /*!< query: from --list */ +VERIFY_FOR_STATE = 1 << 24 # /*!< query: from --state */ +VERIFY_FOR_DOCS = 1 << 25 # /*!< query: from --docfiles */ +VERIFY_FOR_CONFIG = 1 << 26 # /*!< query: from --configfiles */ +VERIFY_FOR_DUMPFILES = 1 << 27 # /*!< query: from --dump */ +# /* bits 28-31 used in rpmVerifyAttrs */ + +# Comes from C cource. lib/rpmcli.h +VERIFY_ATTRS = \ + (VERIFY_MD5 | VERIFY_SIZE | VERIFY_LINKTO | VERIFY_USER | VERIFY_GROUP | + VERIFY_MTIME | VERIFY_MODE | VERIFY_RDEV | VERIFY_CONTEXTS) + +VERIFY_ALL = \ + (VERIFY_ATTRS | VERIFY_FILES | VERIFY_DEPS | VERIFY_SCRIPT | + VERIFY_DIGEST | VERIFY_SIGNATURE | VERIFY_HDRCHK) + + +# Some masks for what checks to NOT do on these file types. +# The C code actiually resets these up for every file. +DIR_FLAGS = ~(RPMVERIFY_MD5 | RPMVERIFY_FILESIZE | RPMVERIFY_MTIME | + RPMVERIFY_LINKTO) + +# These file types all have the same mask, but hopefully this will make the +# code more readable. +FIFO_FLAGS = CHR_FLAGS = BLK_FLAGS = GHOST_FLAGS = DIR_FLAGS + +LINK_FLAGS = ~(RPMVERIFY_MD5 | RPMVERIFY_FILESIZE | RPMVERIFY_MTIME | + RPMVERIFY_MODE | RPMVERIFY_USER | RPMVERIFY_GROUP) + +REG_FLAGS = ~(RPMVERIFY_LINKTO) + + +def s_isdev(mode): + """ + Check to see if a file is a device. + + """ + return stat.S_ISBLK(mode) | stat.S_ISCHR(mode) + + +def rpmpackagelist(rts): + """ + Equivalent of rpm -qa. Intended for RefreshPackages() in the RPM Driver. + Requires rpmtransactionset() to be run first to get a ts. + Returns a list of pkgspec dicts. + + e.g. [{'name':'foo', 'epoch':'20', 'version':'1.2', + 'release':'5', 'arch':'x86_64' }, + {'name':'bar', 'epoch':'10', 'version':'5.2', + 'release':'2', 'arch':'x86_64' }] + + """ + return [ + {'name': header[rpm.RPMTAG_NAME], + 'epoch': header[rpm.RPMTAG_EPOCH], + 'version': header[rpm.RPMTAG_VERSION], + 'release': header[rpm.RPMTAG_RELEASE], + 'arch': header[rpm.RPMTAG_ARCH], + 'gpgkeyid': + header.sprintf("%|SIGGPG?{%{SIGGPG:pgpsig}}:{None}|").split()[-1]} + for header in rts.dbMatch()] + + +def getindexbykeyword(index_ts, **kwargs): + """ + Return list of indexs from the rpmdb matching keywords + ex: getHeadersByKeyword(name='foo', version='1', release='1') + + Can be passed any structure that can be indexed by the pkgspec + keyswords as other keys are filtered out. + + """ + lst = [] + name = kwargs.get('name') + if name: + index_mi = index_ts.dbMatch(rpm.RPMTAG_NAME, name) + else: + index_mi = index_ts.dbMatch() + + if 'epoch' in kwargs: + if kwargs['epoch'] is not None and kwargs['epoch'] != 'None': + kwargs['epoch'] = int(kwargs['epoch']) + else: + del(kwargs['epoch']) + + keywords = [key for key in list(kwargs.keys()) + if key in ('name', 'epoch', 'version', 'release', 'arch')] + keywords_len = len(keywords) + for hdr in index_mi: + match = 0 + for keyword in keywords: + if hdr[keyword] == kwargs[keyword]: + match += 1 + if match == keywords_len: + lst.append(index_mi.instance()) + del index_mi + return lst + + +def getheadersbykeyword(header_ts, **kwargs): + """ + Borrowed parts of this from from Yum. Need to fix it though. + Epoch is not handled right. + + Return list of headers from the rpmdb matching keywords + ex: getHeadersByKeyword(name='foo', version='1', release='1') + + Can be passed any structure that can be indexed by the pkgspec + keyswords as other keys are filtered out. + + """ + lst = [] + name = kwargs.get('name') + if name: + header_mi = header_ts.dbMatch(rpm.RPMTAG_NAME, name) + else: + header_mi = header_ts.dbMatch() + + if 'epoch' in kwargs: + if kwargs['epoch'] is not None and kwargs['epoch'] != 'None': + kwargs['epoch'] = int(kwargs['epoch']) + else: + del(kwargs['epoch']) + + keywords = [key for key in list(kwargs.keys()) + if key in ('name', 'epoch', 'version', 'release', 'arch')] + keywords_len = len(keywords) + for hdr in header_mi: + match = 0 + for keyword in keywords: + if hdr[keyword] == kwargs[keyword]: + match += 1 + if match == keywords_len: + lst.append(hdr) + del header_mi + return lst + + +def prelink_md5_check(filename): + """ + Checks if a file is prelinked. If it is run it through prelink -y + to get the unprelinked md5 and file size. + + Return 0 if the file was not prelinked, otherwise return the file size. + Always return the md5. + + """ + prelink = False + try: + plf = open(filename, "rb") + except IOError: + return False, 0 + + if prelink_exists: + if isprelink_imported: + plfd = plf.fileno() + if isprelink(plfd): + plf.close() + cmd = '/usr/sbin/prelink -y %s 2> /dev/null' \ + % (re.escape(filename)) + plf = os.popen(cmd, 'rb') + prelink = True + elif (whitelist_re.search(filename) and not + blacklist_re.search(filename)): + plf.close() + cmd = '/usr/sbin/prelink -y %s 2> /dev/null' \ + % (re.escape(filename)) + plf = os.popen(cmd, 'rb') + prelink = True + + fsize = 0 + if py24compat: + chksum = md5.new() + else: + chksum = hashlib.md5() + while 1: + data = plf.read() + if not data: + break + fsize += len(data) + chksum.update(data) + plf.close() + file_md5 = chksum.hexdigest() + if prelink: + return file_md5, fsize + else: + return file_md5, 0 + + +def prelink_size_check(filename): + """ + This check is only done if the prelink_md5_check() is not done first. + + Checks if a file is prelinked. If it is run it through prelink -y + to get the unprelinked file size. + + Return 0 if the file was not prelinked, otherwise return the file size. + + """ + fsize = 0 + try: + plf = open(filename, "rb") + except IOError: + return False + + if prelink_exists: + if isprelink_imported: + plfd = plf.fileno() + if isprelink(plfd): + plf.close() + cmd = '/usr/sbin/prelink -y %s 2> /dev/null' \ + % (re.escape(filename)) + plf = os.popen(cmd, 'rb') + + while 1: + data = plf.read() + if not data: + break + fsize += len(data) + + elif (whitelist_re.search(filename) and not + blacklist_re.search(filename)): + plf.close() + cmd = '/usr/sbin/prelink -y %s 2> /dev/null' \ + % (re.escape(filename)) + plf = os.popen(cmd, 'rb') + + while 1: + data = plf.read() + if not data: + break + fsize += len(data) + + plf.close() + + return fsize + + +def debug_verify_flags(vflags): + """ + Decodes the verify flags bits. + """ + if vflags & RPMVERIFY_MD5: + print('RPMVERIFY_MD5') + if vflags & RPMVERIFY_FILESIZE: + print('RPMVERIFY_FILESIZE') + if vflags & RPMVERIFY_LINKTO: + print('RPMVERIFY_LINKTO') + if vflags & RPMVERIFY_USER: + print('RPMVERIFY_USER') + if vflags & RPMVERIFY_GROUP: + print('RPMVERIFY_GROUP') + if vflags & RPMVERIFY_MTIME: + print('RPMVERIFY_MTIME') + if vflags & RPMVERIFY_MODE: + print('RPMVERIFY_MODE') + if vflags & RPMVERIFY_RDEV: + print('RPMVERIFY_RDEV') + if vflags & RPMVERIFY_CONTEXTS: + print('RPMVERIFY_CONTEXTS') + if vflags & RPMVERIFY_READLINKFAIL: + print('RPMVERIFY_READLINKFAIL') + if vflags & RPMVERIFY_READFAIL: + print('RPMVERIFY_READFAIL') + if vflags & RPMVERIFY_LSTATFAIL: + print('RPMVERIFY_LSTATFAIL') + if vflags & RPMVERIFY_LGETFILECONFAIL: + print('RPMVERIFY_LGETFILECONFAIL') + + +def debug_file_flags(fflags): + """ + Decodes the file flags bits. + """ + if fflags & rpm.RPMFILE_CONFIG: + print('rpm.RPMFILE_CONFIG') + + if fflags & rpm.RPMFILE_DOC: + print('rpm.RPMFILE_DOC') + + if fflags & rpm.RPMFILE_ICON: + print('rpm.RPMFILE_ICON') + + if fflags & rpm.RPMFILE_MISSINGOK: + print('rpm.RPMFILE_MISSINGOK') + + if fflags & rpm.RPMFILE_NOREPLACE: + print('rpm.RPMFILE_NOREPLACE') + + if fflags & rpm.RPMFILE_GHOST: + print('rpm.RPMFILE_GHOST') + + if fflags & rpm.RPMFILE_LICENSE: + print('rpm.RPMFILE_LICENSE') + + if fflags & rpm.RPMFILE_README: + print('rpm.RPMFILE_README') + + if fflags & rpm.RPMFILE_EXCLUDE: + print('rpm.RPMFILE_EXLUDE') + + if fflags & rpm.RPMFILE_UNPATCHED: + print('rpm.RPMFILE_UNPATCHED') + + if fflags & rpm.RPMFILE_PUBKEY: + print('rpm.RPMFILE_PUBKEY') + + +def rpm_verify_file(fileinfo, rpmlinktos, omitmask): + """ + Verify all the files in a package. + + Returns a list of error flags, the file type and file name. The list + entries are strings that are the same as the labels for the bitwise + flags used in the C code. + + """ + (fname, fsize, fmode, fmtime, fflags, frdev, finode, fnlink, fstate, + vflags, fuser, fgroup, fmd5) = fileinfo + + # 1. rpmtsRootDir stuff. What does it do and where to I get it from? + + file_results = [] + flags = vflags + + # Check to see if the file was installed - if not pretend all is ok. + # This is what the rpm C code does! + if fstate != rpm.RPMFILE_STATE_NORMAL: + return file_results + + # Get the installed files stats + try: + lstat = os.lstat(fname) + except OSError: + if not (fflags & (rpm.RPMFILE_MISSINGOK | rpm.RPMFILE_GHOST)): + file_results.append('RPMVERIFY_LSTATFAIL') + #file_results.append(fname) + return file_results + + # 5. Contexts? SELinux stuff? + + # Setup what checks to do. This is straight out of the C code. + if stat.S_ISDIR(lstat.st_mode): + flags &= DIR_FLAGS + elif stat.S_ISLNK(lstat.st_mode): + flags &= LINK_FLAGS + elif stat.S_ISFIFO(lstat.st_mode): + flags &= FIFO_FLAGS + elif stat.S_ISCHR(lstat.st_mode): + flags &= CHR_FLAGS + elif stat.S_ISBLK(lstat.st_mode): + flags &= BLK_FLAGS + else: + flags &= REG_FLAGS + + if (fflags & rpm.RPMFILE_GHOST): + flags &= GHOST_FLAGS + + flags &= ~(omitmask | RPMVERIFY_FAILURES) + + # 8. SELinux stuff. + + prelink_size = 0 + if flags & RPMVERIFY_MD5: + prelink_md5, prelink_size = prelink_md5_check(fname) + if prelink_md5 is False: + file_results.append('RPMVERIFY_MD5') + file_results.append('RPMVERIFY_READFAIL') + elif prelink_md5 != fmd5: + file_results.append('RPMVERIFY_MD5') + + if flags & RPMVERIFY_LINKTO: + linkto = os.readlink(fname) + if not linkto: + file_results.append('RPMVERIFY_READLINKFAIL') + file_results.append('RPMVERIFY_LINKTO') + else: + if len(rpmlinktos) == 0 or linkto != rpmlinktos: + file_results.append('RPMVERIFY_LINKTO') + + if flags & RPMVERIFY_FILESIZE: + if not (flags & RPMVERIFY_MD5): # prelink check hasn't been done. + prelink_size = prelink_size_check(fname) + if (prelink_size != 0): # This is a prelinked file. + if (prelink_size != fsize): + file_results.append('RPMVERIFY_FILESIZE') + elif lstat.st_size != fsize: # It wasn't a prelinked file. + file_results.append('RPMVERIFY_FILESIZE') + + if flags & RPMVERIFY_MODE: + metamode = fmode + filemode = lstat.st_mode + + # Comparing the type of %ghost files is meaningless, but perms are ok. + if fflags & rpm.RPMFILE_GHOST: + metamode &= ~0xf000 + filemode &= ~0xf000 + + if (stat.S_IFMT(metamode) != stat.S_IFMT(filemode)) or \ + (stat.S_IMODE(metamode) != stat.S_IMODE(filemode)): + file_results.append('RPMVERIFY_MODE') + + if flags & RPMVERIFY_RDEV: + if (stat.S_ISCHR(fmode) != stat.S_ISCHR(lstat.st_mode) or + stat.S_ISBLK(fmode) != stat.S_ISBLK(lstat.st_mode)): + file_results.append('RPMVERIFY_RDEV') + elif (s_isdev(fmode) & s_isdev(lstat.st_mode)): + st_rdev = lstat.st_rdev + if frdev != st_rdev: + file_results.append('RPMVERIFY_RDEV') + + if flags & RPMVERIFY_MTIME: + if lstat.st_mtime != fmtime: + file_results.append('RPMVERIFY_MTIME') + + if flags & RPMVERIFY_USER: + try: + user = pwd.getpwuid(lstat.st_uid)[0] + except KeyError: + user = None + if not user or not fuser or (user != fuser): + file_results.append('RPMVERIFY_USER') + + if flags & RPMVERIFY_GROUP: + try: + group = grp.getgrgid(lstat.st_gid)[0] + except KeyError: + group = None + if not group or not fgroup or (group != fgroup): + file_results.append('RPMVERIFY_GROUP') + + return file_results + + +def rpm_verify_dependencies(header): + """ + Check package dependencies. Header is an rpm.hdr. + + Don't like opening another ts to do this, but + it was the only way I could find of clearing the ts + out. + + Have asked on the rpm-maint list on how to do + this the right way (28 Feb 2007). + + ts.check() returns: + + ((name, version, release), (reqname, reqversion), \ + flags, suggest, sense) + + """ + _ts1 = rpmtransactionset() + _ts1.addInstall(header, 'Dep Check', 'i') + dep_errors = _ts1.check() + _ts1.closeDB() + return dep_errors + + +def rpm_verify_package(vp_ts, header, verify_options): + """ + Verify a single package specified by header. Header is an rpm.hdr. + + If errors are found it returns a dictionary of errors. + + """ + # Set some transaction level flags. + vsflags = 0 + if 'nodigest' in verify_options: + vsflags |= rpm._RPMVSF_NODIGESTS + if 'nosignature' in verify_options: + vsflags |= rpm._RPMVSF_NOSIGNATURES + ovsflags = vp_ts.setVSFlags(vsflags) + + # Map from the Python options to the rpm bitwise flags. + omitmask = 0 + + if 'nolinkto' in verify_options: + omitmask |= VERIFY_LINKTO + if 'nomd5' in verify_options: + omitmask |= VERIFY_MD5 + if 'nosize' in verify_options: + omitmask |= VERIFY_SIZE + if 'nouser' in verify_options: + omitmask |= VERIFY_USER + if 'nogroup' in verify_options: + omitmask |= VERIFY_GROUP + if 'nomtime' in verify_options: + omitmask |= VERIFY_MTIME + if 'nomode' in verify_options: + omitmask |= VERIFY_MODE + if 'nordev' in verify_options: + omitmask |= VERIFY_RDEV + + omitmask = ((~omitmask & VERIFY_ATTRS) ^ VERIFY_ATTRS) + + package_results = {} + + # Check Signatures and Digests. + # No idea what this might return. Need to break something to see. + # Setting the vsflags above determines what gets checked in the header. + hdr_stat = vp_ts.hdrCheck(header.unload()) + if hdr_stat: + package_results['hdr'] = hdr_stat + + # Check Package Depencies. + if 'nodeps' not in verify_options: + dep_stat = rpm_verify_dependencies(header) + if dep_stat: + package_results['deps'] = dep_stat + + # Check all the package files. + if 'nofiles' not in verify_options: + vp_fi = header.fiFromHeader() + for fileinfo in vp_fi: + # Do not bother doing anything with ghost files. + # This is what RPM does. + if fileinfo[4] & rpm.RPMFILE_GHOST: + continue + + # This is only needed because of an inconsistency in the + # rpm.fi interface. + linktos = vp_fi.FLink() + + file_stat = rpm_verify_file(fileinfo, linktos, omitmask) + + #if len(file_stat) > 0 or options.verbose: + if len(file_stat) > 0: + fflags = fileinfo[4] + if fflags & rpm.RPMFILE_CONFIG: + file_stat.append('c') + elif fflags & rpm.RPMFILE_DOC: + file_stat.append('d') + elif fflags & rpm.RPMFILE_GHOST: + file_stat.append('g') + elif fflags & rpm.RPMFILE_LICENSE: + file_stat.append('l') + elif fflags & rpm.RPMFILE_PUBKEY: + file_stat.append('P') + elif fflags & rpm.RPMFILE_README: + file_stat.append('r') + else: + file_stat.append(' ') + + file_stat.append(fileinfo[0]) # The filename. + package_results.setdefault('files', []).append(file_stat) + + # Run the verify script if there is one. + # Do we want this? + #if 'noscripts' not in verify_options: + # script_stat = rpmVerifyscript() + # if script_stat: + # package_results['script'] = script_stat + + # If there have been any errors, add the package nevra to the result. + if len(package_results) > 0: + package_results.setdefault('nevra', (header[rpm.RPMTAG_NAME], + header[rpm.RPMTAG_EPOCH], + header[rpm.RPMTAG_VERSION], + header[rpm.RPMTAG_RELEASE], + header[rpm.RPMTAG_ARCH])) + else: + package_results = None + + # Put things back the way we found them. + vsflags = vp_ts.setVSFlags(ovsflags) + + return package_results + + +def rpm_verify(verify_ts, verify_pkgspec, verify_options=[]): + """ + Requires rpmtransactionset() to be run first to get a ts. + + pkgspec is a dict specifying the package + e.g.: + For a single package + { name='foo', epoch='20', version='1', release='1', arch='x86_64'} + + For all packages + {} + + Or any combination of keywords to select one or more packages to verify. + + options is a list of 'rpm --verify' options. + Default is to check everything. + e.g.: + [ 'nodeps', 'nodigest', 'nofiles', 'noscripts', 'nosignature', + 'nolinkto' 'nomd5', 'nosize', 'nouser', 'nogroup', 'nomtime', + 'nomode', 'nordev' ] + + Returns a list. One list entry per package. Each list entry is a + dictionary. Dict keys are 'files', 'deps', 'nevra' and 'hdr'. + Entries only get added for the failures. If nothing failed, None is + returned. + + Its all a bit messy and probably needs reviewing. + + [ { 'hdr': [???], + 'deps: [((name, version, release), (reqname, reqversion), + flags, suggest, sense), .... ] + 'files': [ ['filename1', 'RPMVERIFY_GROUP', 'RPMVERIFY_USER' ], + ['filename2', 'RPMVERFIY_LSTATFAIL']] + 'nevra': ['name1', 'epoch1', 'version1', 'release1', 'arch1'] } + { 'hdr': [???], + 'deps: [((name, version, release), (reqname, reqversion), + flags, suggest, sense), .... ] + 'files': [ ['filename', 'RPMVERIFY_GROUP', 'RPMVERIFY_USER" ], + ['filename2', 'RPMVERFIY_LSTATFAIL']] + 'nevra': ['name2', 'epoch2', 'version2', 'release2', 'arch2'] } ] + + """ + verify_results = [] + headers = getheadersbykeyword(verify_ts, **verify_pkgspec) + for header in headers: + result = rpm_verify_package(verify_ts, header, verify_options) + if result: + verify_results.append(result) + + return verify_results + + +def rpmtransactionset(): + """ + A simple wrapper for rpm.TransactionSet() to keep everthiing together. + Might use it to set some ts level flags later. + + """ + ts = rpm.TransactionSet() + return ts + + +class Rpmtscallback(object): + """ + Callback for ts.run(). Used for adding, upgrading and removing packages. + Starting with all possible reasons codes, but bcfg2 will probably only + make use of a few of them. + + Mostly just printing stuff at the moment to understand how the callback + is used. + """ + def __init__(self): + self.fdnos = {} + + def callback(self, reason, amount, total, key, client_data): + """ + Generic rpmts call back. + """ + if reason == rpm.RPMCALLBACK_INST_OPEN_FILE: + pass + elif reason == rpm.RPMCALLBACK_INST_CLOSE_FILE: + pass + elif reason == rpm.RPMCALLBACK_INST_START: + pass + elif reason == rpm.RPMCALLBACK_TRANS_PROGRESS or \ + reason == rpm.RPMCALLBACK_INST_PROGRESS: + pass + # rpm.RPMCALLBACK_INST_PROGRESS' + elif reason == rpm.RPMCALLBACK_TRANS_START: + pass + elif reason == rpm.RPMCALLBACK_TRANS_STOP: + pass + elif reason == rpm.RPMCALLBACK_REPACKAGE_START: + pass + elif reason == rpm.RPMCALLBACK_REPACKAGE_PROGRESS: + pass + elif reason == rpm.RPMCALLBACK_REPACKAGE_STOP: + pass + elif reason == rpm.RPMCALLBACK_UNINST_PROGRESS: + pass + elif reason == rpm.RPMCALLBACK_UNINST_START: + pass + elif reason == rpm.RPMCALLBACK_UNINST_STOP: + pass + # How do we get at this? + # RPM.modified += key + elif reason == rpm.RPMCALLBACK_UNPACK_ERROR: + pass + elif reason == rpm.RPMCALLBACK_CPIO_ERROR: + pass + elif reason == rpm.RPMCALLBACK_UNKNOWN: + pass + else: + print('ERROR - Fell through callBack') + + +def rpm_erase(erase_pkgspecs, erase_flags): + """ + pkgspecs is a list of pkgspec dicts specifying packages + e.g.: + For a single package + { name='foo', epoch='20', version='1', release='1', arch='x86_64'} + + """ + erase_ts_flags = 0 + if 'noscripts' in erase_flags: + erase_ts_flags |= rpm.RPMTRANS_FLAG_NOSCRIPTS + if 'notriggers' in erase_flags: + erase_ts_flags |= rpm.RPMTRANS_FLAG_NOTRIGGERS + if 'repackage' in erase_flags: + erase_ts_flags |= rpm.RPMTRANS_FLAG_REPACKAGE + + erase_ts = rpmtransactionset() + erase_ts.setFlags(erase_ts_flags) + + for pkgspec in erase_pkgspecs: + idx_list = getindexbykeyword(erase_ts, **pkgspec) + if len(idx_list) > 1 and not 'allmatches' in erase_flags: + #pass + print('ERROR - Multiple package match for erase', pkgspec) + else: + for idx in idx_list: + erase_ts.addErase(idx) + + #for te in erase_ts: + + erase_problems = [] + if 'nodeps' not in erase_flags: + erase_problems = erase_ts.check() + + if erase_problems == []: + erase_ts.order() + erase_callback = Rpmtscallback() + erase_ts.run(erase_callback.callback, 'Erase') + #else: + + erase_ts.closeDB() + del erase_ts + return erase_problems + + +def display_verify_file(file_results): + ''' + Display file results similar to rpm --verify. + ''' + filename = file_results[-1] + filetype = file_results[-2] + + result_string = '' + + if 'RPMVERIFY_LSTATFAIL' in file_results: + result_string = 'missing ' + else: + if 'RPMVERIFY_FILESIZE' in file_results: + result_string = result_string + 'S' + else: + result_string = result_string + '.' + + if 'RPMVERIFY_MODE' in file_results: + result_string = result_string + 'M' + else: + result_string = result_string + '.' + + if 'RPMVERIFY_MD5' in file_results: + if 'RPMVERIFY_READFAIL' in file_results: + result_string = result_string + '?' + else: + result_string = result_string + '5' + else: + result_string = result_string + '.' + + if 'RPMVERIFY_RDEV' in file_results: + result_string = result_string + 'D' + else: + result_string = result_string + '.' + + if 'RPMVERIFY_LINKTO' in file_results: + if 'RPMVERIFY_READLINKFAIL' in file_results: + result_string = result_string + '?' + else: + result_string = result_string + 'L' + else: + result_string = result_string + '.' + + if 'RPMVERIFY_USER' in file_results: + result_string = result_string + 'U' + else: + result_string = result_string + '.' + + if 'RPMVERIFY_GROUP' in file_results: + result_string = result_string + 'G' + else: + result_string = result_string + '.' + + if 'RPMVERIFY_MTIME' in file_results: + result_string = result_string + 'T' + else: + result_string = result_string + '.' + + print(result_string + ' ' + filetype + ' ' + filename) + sys.stdout.flush() + +#============================================================================= +# Some options and output to assist with development and testing. +# These are not intended for normal use. +if __name__ == "__main__": + + p = optparse.OptionParser() + + p.add_option('--name', action='store', + default=None, + help='''Package name to verify. + + ****************************************** + NOT SPECIFYING A NAME MEANS 'ALL' PACKAGES. + ****************************************** + + The specified operation will be carried out on all + instances of packages that match the package + specification + (name, epoch, version, release, arch).''') + + p.add_option('--epoch', action='store', + default=None, + help='''Package epoch.''') + + p.add_option('--version', action='store', + default=None, + help='''Package version.''') + + p.add_option('--release', action='store', + default=None, + help='''Package release.''') + + p.add_option('--arch', action='store', + default=None, + help='''Package arch.''') + + p.add_option('--erase', '-e', action='store_true', + default=None, + help= + '''**************************************************** + REMOVE PACKAGES. THERE ARE NO WARNINGS. MULTIPLE + PACKAGES WILL BE REMOVED IF A FULL PACKAGE SPEC IS NOT + GIVEN. E.G. IF JUST A NAME IS GIVEN ALL INSTALLED + INSTANCES OF THAT PACKAGE WILL BE REMOVED PROVIDED + DEPENDENCY CHECKS PASS. IF JUST AN EPOCH IS GIVEN + ALL PACKAGE INSTANCES WITH THAT EPOCH WILL BE REMOVED. + ****************************************************''') + + p.add_option('--list', '-l', action='store_true', + help='''List package identity info. rpm -qa ish equivalent + intended for use in RefreshPackages().''') + + p.add_option('--verify', action='store_true', + help='''Verify Package(s). Output is only produced after all + packages has been verified. Be patient.''') + + p.add_option('--verbose', '-v', action='store_true', + help='''Verbose output for --verify option. Output is the + same as rpm -v --verify.''') + + p.add_option('--nodeps', action='store_true', + default=False, + help='Do not do dependency testing.') + + p.add_option('--nodigest', action='store_true', + help='Do not check package digests.') + + p.add_option('--nofiles', action='store_true', + help='Do not do file checks.') + + p.add_option('--noscripts', action='store_true', + help='Do not run verification scripts.') + + p.add_option('--nosignature', action='store_true', + help='Do not do package signature verification.') + + p.add_option('--nolinkto', action='store_true', + help='Do not do symlink tests.') + + p.add_option('--nomd5', action='store_true', + help='''Do not do MD5 checksums on files. Note that this does + not work for prelink files yet.''') + + p.add_option('--nosize', action='store_true', + help='''Do not do file size tests. Note that this does not + work for prelink files yet.''') + + p.add_option('--nouser', action='store_true', + help='Do not check file user ownership.') + + p.add_option('--nogroup', action='store_true', + help='Do not check file group ownership.') + + p.add_option('--nomtime', action='store_true', + help='Do not check file modification times.') + + p.add_option('--nomode', action='store_true', + help='Do not check file modes (permissions).') + + p.add_option('--nordev', action='store_true', + help='Do not check device node.') + + p.add_option('--notriggers', action='store_true', + help='Do not do not generate triggers on erase.') + + p.add_option('--repackage', action='store_true', + help='''Do repackage on erase.i Packages are put + in /var/spool/repackage.''') + + p.add_option('--allmatches', action='store_true', + help= + '''Remove all package instances that match the + pkgspec. + + *************************************************** + NO WARNINGS ARE GIVEN. IF THERE IS NO PACKAGE SPEC + THAT MEANS ALL PACKAGES!!!! + ***************************************************''') + + options, arguments = p.parse_args() + + pkgspec = {} + rpm_options = [] + + if options.nodeps: + rpm_options.append('nodeps') + + if options.nodigest: + rpm_options.append('nodigest') + + if options.nofiles: + rpm_options.append('nofiles') + + if options.noscripts: + rpm_options.append('noscripts') + + if options.nosignature: + rpm_options.append('nosignature') + + if options.nolinkto: + rpm_options.append('nolinkto') + + if options.nomd5: + rpm_options.append('nomd5') + + if options.nosize: + rpm_options.append('nosize') + + if options.nouser: + rpm_options.append('nouser') + + if options.nogroup: + rpm_options.append('nogroup') + + if options.nomtime: + rpm_options.append('nomtime') + + if options.nomode: + rpm_options.append('nomode') + + if options.nordev: + rpm_options.append('nordev') + + if options.repackage: + rpm_options.append('repackage') + + if options.allmatches: + rpm_options.append('allmatches') + + main_ts = rpmtransactionset() + + cmdline_pkgspec = {} + if options.name != 'all': + if options.name: + cmdline_pkgspec['name'] = str(options.name) + if options.epoch: + cmdline_pkgspec['epoch'] = str(options.epoch) + if options.version: + cmdline_pkgspec['version'] = str(options.version) + if options.release: + cmdline_pkgspec['release'] = str(options.release) + if options.arch: + cmdline_pkgspec['arch'] = str(options.arch) + + if options.verify: + results = rpm_verify(main_ts, cmdline_pkgspec, rpm_options) + for r in results: + files = r.get('files', '') + for f in files: + display_verify_file(f) + + elif options.list: + for p in rpmpackagelist(main_ts): + print(p) + + elif options.erase: + if options.name: + rpm_erase([cmdline_pkgspec], rpm_options) + else: + print('You must specify the "--name" option') + class RPM(Bcfg2.Client.Tools.PkgTool): """Support for RPM packages.""" + options = Bcfg2.Client.Tools.PkgTool.options + [ + Bcfg2.Options.Option( + cf=('RPM', 'installonlypackages'), dest="rpm_installonly", + type=Bcfg2.Options.Types.comma_list, + default=['kernel', 'kernel-bigmem', 'kernel-enterprise', + 'kernel-smp', 'kernel-modules', 'kernel-debug', + 'kernel-unsupported', 'kernel-devel', 'kernel-source', + 'kernel-default', 'kernel-largesmp-devel', + 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], + help='RPM install-only packages'), + Bcfg2.Options.BooleanOption( + cf=('RPM', 'pkg_checks'), default=True, dest="rpm_pkg_checks", + help="Perform RPM package checks"), + Bcfg2.Options.BooleanOption( + cf=('RPM', 'pkg_verify'), default=True, dest="rpm_pkg_verify", + help="Perform RPM package verify"), + Bcfg2.Options.BooleanOption( + cf=('RPM', 'install_missing'), default=True, + dest="rpm_install_missing", + help="Install missing packages"), + Bcfg2.Options.Option( + cf=('RPM', 'erase_flags'), default=["allmatches"], + dest="rpm_erase_flags", type=Bcfg2.Options.Types.comma_list, + help="RPM erase flags"), + Bcfg2.Options.BooleanOption( + cf=('RPM', 'fix_version'), default=True, + dest="rpm_fix_version", + help="Fix (upgrade or downgrade) packages with the wrong version"), + Bcfg2.Options.BooleanOption( + cf=('RPM', 'reinstall_broken'), default=True, + dest="rpm_reinstall_broken", + help="Reinstall packages that fail to verify"), + Bcfg2.Options.Option( + cf=('RPM', 'verify_flags'), default=[], + dest="rpm_verify_flags", type=Bcfg2.Options.Types.comma_list, + help="RPM verify flags")] + __execs__ = ['/bin/rpm', '/var/lib/rpm'] __handles__ = [('Package', 'rpm')] @@ -15,7 +1143,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): __new_req__ = {'Package': ['name'], 'Instance': ['version', 'release', 'arch']} - __new_ireq__ = {'Package': ['uri'], \ + __new_ireq__ = {'Package': ['uri'], 'Instance': ['simplefile']} __gpg_req__ = {'Package': ['name', 'version']} @@ -26,60 +1154,51 @@ class RPM(Bcfg2.Client.Tools.PkgTool): __new_gpg_ireq__ = {'Package': ['name'], 'Instance': ['version', 'release']} - conflicts = ['RPMng'] - pkgtype = 'rpm' pkgtool = ("rpm --oldpackage --replacepkgs --quiet -U %s", ("%s", ["url"])) - def __init__(self, logger, setup, config): - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) + def __init__(self, config): + Bcfg2.Client.Tools.PkgTool.__init__(self, config) # create a global ignore list used when ignoring particular # files during package verification - self.ignores = [entry.get('name') for struct in config for entry in struct \ - if entry.get('type') == 'ignore'] + self.ignores = [entry.get('name') for struct in config + for entry in struct if entry.get('type') == 'ignore'] self.instance_status = {} self.extra_instances = [] self.modlists = {} self.gpg_keyids = self.getinstalledgpg() - opt_prefix = self.name.lower() - self.installOnlyPkgs = self.setup["%s_installonly" % opt_prefix] + self.installOnlyPkgs = Bcfg2.Options.setup.rpm_installonly if 'gpg-pubkey' not in self.installOnlyPkgs: self.installOnlyPkgs.append('gpg-pubkey') - self.erase_flags = self.setup['%s_erase_flags' % opt_prefix] - self.pkg_checks = self.setup['%s_pkg_checks' % opt_prefix] - self.pkg_verify = self.setup['%s_pkg_verify' % opt_prefix] - self.installed_action = self.setup['%s_installed_action' % opt_prefix] - self.version_fail_action = self.setup['%s_version_fail_action' % - opt_prefix] - self.verify_fail_action = self.setup['%s_verify_fail_action' % - opt_prefix] - self.verify_flags = self.setup['%s_verify_flags' % opt_prefix] + self.verify_flags = Bcfg2.Options.setup.rpm_verify_flags if '' in self.verify_flags: self.verify_flags.remove('') self.logger.debug('%s: installOnlyPackages = %s' % (self.name, self.installOnlyPkgs)) self.logger.debug('%s: erase_flags = %s' % - (self.name, self.erase_flags)) + (self.name, Bcfg2.Options.setup.rpm_erase_flags)) self.logger.debug('%s: pkg_checks = %s' % - (self.name, self.pkg_checks)) + (self.name, Bcfg2.Options.setup.rpm_pkg_checks)) self.logger.debug('%s: pkg_verify = %s' % - (self.name, self.pkg_verify)) - self.logger.debug('%s: installed_action = %s' % - (self.name, self.installed_action)) - self.logger.debug('%s: version_fail_action = %s' % - (self.name, self.version_fail_action)) - self.logger.debug('%s: verify_fail_action = %s' % - (self.name, self.verify_fail_action)) + (self.name, Bcfg2.Options.setup.rpm_pkg_verify)) + self.logger.debug('%s: install_missing = %s' % + (self.name, Bcfg2.Options.setup.install_missing)) + self.logger.debug('%s: fix_version = %s' % + (self.name, Bcfg2.Options.setup.rpm_fix_version)) + self.logger.debug('%s: reinstall_broken = %s' % + (self.name, + Bcfg2.Options.setup.rpm_reinstall_broken)) self.logger.debug('%s: verify_flags = %s' % (self.name, self.verify_flags)) # Force a re- prelink of all packages if prelink exists. # Many, if not most package verifies can be caused by out of # date prelinking. - if os.path.isfile('/usr/sbin/prelink') and not self.setup['dryrun']: + if (os.path.isfile('/usr/sbin/prelink') and + not Bcfg2.Options.setup.dry_run): rv = self.cmd.run('/usr/sbin/prelink -a -mR') if rv.success: self.logger.debug('Pre-emptive prelink succeeded') @@ -104,18 +1223,18 @@ class RPM(Bcfg2.Client.Tools.PkgTool): 'arch':'x86_64'} ] """ self.installed = {} - refresh_ts = rpmtools.rpmtransactionset() + refresh_ts = rpmtransactionset() # Don't bother with signature checks at this stage. The GPG keys might # not be installed. - refresh_ts.setVSFlags(rpm._RPMVSF_NODIGESTS|rpm._RPMVSF_NOSIGNATURES) - for nevra in rpmtools.rpmpackagelist(refresh_ts): + refresh_ts.setVSFlags(rpm._RPMVSF_NODIGESTS | rpm._RPMVSF_NOSIGNATURES) + for nevra in rpmpackagelist(refresh_ts): self.installed.setdefault(nevra['name'], []).append(nevra) - if self.setup['debug']: + if Bcfg2.Options.setup.debug: print("The following package instances are installed:") for name, instances in list(self.installed.items()): self.logger.debug(" " + name) for inst in instances: - self.logger.debug(" %s" %self.str_evra(inst)) + self.logger.debug(" %s" % self.str_evra(inst)) refresh_ts.closeDB() del refresh_ts @@ -145,18 +1264,19 @@ class RPM(Bcfg2.Client.Tools.PkgTool): Constructs the text prompts for interactive mode. """ - instances = [inst for inst in entry if inst.tag == 'Instance' or inst.tag == 'Package'] + instances = [inst for inst in entry if inst.tag == 'Instance' or + inst.tag == 'Package'] if instances == []: # We have an old style no Instance entry. Convert it to new style. instance = Bcfg2.Client.XML.SubElement(entry, 'Package') for attrib in list(entry.attrib.keys()): instance.attrib[attrib] = entry.attrib[attrib] - if (self.pkg_checks and - entry.get('pkg_checks', 'true').lower() == 'true'): + if (Bcfg2.Options.setup.rpm_pkg_checks and + entry.get('pkg_checks', 'true').lower() == 'true'): if 'any' in [entry.get('version'), pinned_version]: version, release = 'any', 'any' elif entry.get('version') == 'auto': - if pinned_version != None: + if pinned_version is not None: version, release = pinned_version.split('-') else: return False @@ -166,242 +1286,315 @@ class RPM(Bcfg2.Client.Tools.PkgTool): instance.set('release', release) if entry.get('verify', 'true') == 'false': instance.set('verify', 'false') - instances = [ instance ] + instances = [instance] - self.logger.debug("Verifying package instances for %s" % entry.get('name')) + self.logger.debug("Verifying package instances for %s" % + entry.get('name')) package_fail = False qtext_versions = '' if entry.get('name') in self.installed: # There is at least one instance installed. - if (self.pkg_checks and - entry.get('pkg_checks', 'true').lower() == 'true'): + if (Bcfg2.Options.setup.rpm_pkg_checks and + entry.get('pkg_checks', 'true').lower() == 'true'): rpmTs = rpm.TransactionSet() rpmHeader = None for h in rpmTs.dbMatch(rpm.RPMTAG_NAME, entry.get('name')): - if rpmHeader is None or rpm.versionCompare(h, rpmHeader) > 0: + if rpmHeader is None or \ + rpm.versionCompare(h, rpmHeader) > 0: rpmHeader = h - rpmProvides = [ h['provides'] for h in \ - rpmTs.dbMatch(rpm.RPMTAG_NAME, entry.get('name')) ] + rpmProvides = [h['provides'] for h in + rpmTs.dbMatch(rpm.RPMTAG_NAME, + entry.get('name'))] rpmIntersection = set(rpmHeader['provides']) & \ - set(self.installOnlyPkgs) + set(self.installOnlyPkgs) if len(rpmIntersection) > 0: # Packages that should only be installed or removed. # e.g. kernels. self.logger.debug(" Install only package.") for inst in instances: - self.instance_status.setdefault(inst, {})['installed'] = False + self.instance_status.setdefault(inst, {})['installed']\ + = False self.instance_status[inst]['version_fail'] = False - if inst.tag == 'Package' and len(self.installed[entry.get('name')]) > 1: - self.logger.error("WARNING: Multiple instances of package %s are installed." % \ + if inst.tag == 'Package' and \ + len(self.installed[entry.get('name')]) > 1: + self.logger.error("WARNING: Multiple instances of " + "package %s are installed." % (entry.get('name'))) for pkg in self.installed[entry.get('name')]: - if inst.get('version') == 'any' or self.pkg_vr_equal(inst, pkg) \ - or self.inst_evra_equal(inst, pkg): + if inst.get('version') == 'any' or \ + self.pkg_vr_equal(inst, pkg) or \ + self.inst_evra_equal(inst, pkg): if inst.get('version') == 'any': self.logger.error("got any version") - self.logger.debug(" %s" % self.str_evra(inst)) + self.logger.debug(" %s" % + self.str_evra(inst)) self.instance_status[inst]['installed'] = True - if (self.pkg_verify and - inst.get('pkg_verify', 'true').lower() == 'true'): - flags = inst.get('verify_flags', '').split(',') + self.verify_flags + if (Bcfg2.Options.setup.rpm_pkg_verify and + inst.get('pkg_verify', + 'true').lower() == 'true'): + flags = inst.get('verify_flags', + '').split(',') + \ + self.verify_flags if pkg.get('gpgkeyid', '')[-8:] not in self.gpg_keyids and \ entry.get('name') != 'gpg-pubkey': flags += ['nosignature', 'nodigest'] - self.logger.debug('WARNING: Package %s %s requires GPG Public key with ID %s'\ - % (pkg.get('name'), self.str_evra(pkg), \ - pkg.get('gpgkeyid', ''))) - self.logger.debug(' Disabling signature check.') - - if self.setup.get('quick', False): - if rpmtools.prelink_exists: + self.logger.debug('WARNING: Package ' + '%s %s requires GPG ' + 'Public key with ID ' + '%s' % + (pkg.get('name'), + self.str_evra(pkg), + pkg.get('gpgkeyid', + ''))) + self.logger.debug(' Disabling ' + 'signature check.') + + if Bcfg2.Options.setup.quick: + if prelink_exists: flags += ['nomd5', 'nosize'] else: flags += ['nomd5'] - self.logger.debug(" verify_flags = %s" % flags) + self.logger.debug(" verify_flags = " + "%s" % flags) if inst.get('verify', 'true') == 'false': - self.instance_status[inst]['verify'] = None + self.instance_status[inst]['verify'] =\ + None else: - vp_ts = rpmtools.rpmtransactionset() - self.instance_status[inst]['verify'] = \ - rpmtools.rpm_verify( vp_ts, pkg, flags) + vp_ts = rpmtransactionset() + self.instance_status[inst]['verify'] =\ + rpm_verify(vp_ts, pkg, flags) vp_ts.closeDB() del vp_ts - if self.instance_status[inst]['installed'] == False: - self.logger.info(" Package %s %s not installed." % \ - (entry.get('name'), self.str_evra(inst))) + if not self.instance_status[inst]['installed']: + self.logger.info(" Package %s %s not " + "installed." % + (entry.get('name'), + self.str_evra(inst))) - qtext_versions = qtext_versions + 'I(%s) ' % self.str_evra(inst) + qtext_versions = qtext_versions + 'I(%s) ' % \ + self.str_evra(inst) entry.set('current_exists', 'false') else: # Normal Packages that can be upgraded. for inst in instances: - self.instance_status.setdefault(inst, {})['installed'] = False + self.instance_status.setdefault(inst, {})['installed']\ + = False self.instance_status[inst]['version_fail'] = False - # Only installed packages with the same architecture are - # relevant. - if inst.get('arch', None) == None: + # only installed packages with the same architecture + # are relevant. + if inst.get('arch', None) is None: arch_match = self.installed[entry.get('name')] else: - arch_match = [pkg for pkg in self.installed[entry.get('name')] \ - if pkg.get('arch', None) == inst.get('arch', None)] + arch_match = [pkg for pkg in + self.installed[entry.get('name')] + if pkg.get('arch', None) == + inst.get('arch', None)] if len(arch_match) > 1: - self.logger.error("Multiple instances of package %s installed with the same achitecture." % \ - (entry.get('name'))) + self.logger.error("Multiple instances of package " + "%s installed with the same " + "achitecture." % + (entry.get('name'))) elif len(arch_match) == 1: # There is only one installed like there should be. # Check that it is the right version. for pkg in arch_match: - if inst.get('version') == 'any' or self.pkg_vr_equal(inst, pkg) or \ - self.inst_evra_equal(inst, pkg): - self.logger.debug(" %s" % self.str_evra(inst)) - self.instance_status[inst]['installed'] = True - - if (self.pkg_verify and - inst.get('pkg_verify', 'true').lower() == 'true'): - flags = inst.get('verify_flags', '').split(',') + self.verify_flags - if pkg.get('gpgkeyid', '')[-8:] not in self.gpg_keyids and \ - 'nosignature' not in flags: - flags += ['nosignature', 'nodigest'] - self.logger.info('WARNING: Package %s %s requires GPG Public key with ID %s'\ - % (pkg.get('name'), self.str_evra(pkg), \ - pkg.get('gpgkeyid', ''))) - self.logger.info(' Disabling signature check.') - - if self.setup.get('quick', False): - if rpmtools.prelink_exists: + if inst.get('version') == 'any' or \ + self.pkg_vr_equal(inst, pkg) or \ + self.inst_evra_equal(inst, pkg): + self.logger.debug(" %s" % + self.str_evra(inst)) + self.instance_status[inst]['installed'] = \ + True + + if (Bcfg2.Options.setup.rpm_pkg_verify and + inst.get( + 'pkg_verify', + 'true').lower() == 'true'): + flags = inst.get('verify_flags', + '').split(',') + \ + self.verify_flags + if pkg.get('gpgkeyid', '')[-8:] not in\ + self.gpg_keyids and 'nosignature'\ + not in flags: + flags += ['nosignature', + 'nodigest'] + self.logger.info( + 'WARNING: Package %s %s ' + 'requires GPG Public key with ' + 'ID %s' % (pkg.get('name'), + self.str_evra(pkg), + pkg.get('gpgkeyid', + ''))) + self.logger.info( + ' Disabling signature ' + 'check.') + + if Bcfg2.Options.setup.quick: + if prelink_exists: flags += ['nomd5', 'nosize'] else: flags += ['nomd5'] - self.logger.debug(" verify_flags = %s" % flags) + self.logger.debug( + " verify_flags = %s" % + flags) - if inst.get('verify', 'true') == 'false': + if inst.get('verify', 'true') == \ + 'false': self.instance_status[inst]['verify'] = None else: - vp_ts = rpmtools.rpmtransactionset() - self.instance_status[inst]['verify'] = \ - rpmtools.rpm_verify( vp_ts, pkg, flags ) + vp_ts = rpmtransactionset() + self.instance_status[inst]['verify'] = rpm_verify(vp_ts, pkg, flags) vp_ts.closeDB() del vp_ts else: # Wrong version installed. - self.instance_status[inst]['version_fail'] = True - self.logger.info(" Wrong version installed. Want %s, but have %s"\ - % (self.str_evra(inst), self.str_evra(pkg))) - - qtext_versions = qtext_versions + 'U(%s -> %s) ' % \ - (self.str_evra(pkg), self.str_evra(inst)) + self.instance_status[inst]['version_fail']\ + = True + self.logger.info(" Wrong version " + "installed. Want %s, but " + "have %s" % + (self.str_evra(inst), + self.str_evra(pkg))) + + qtext_versions = qtext_versions + \ + 'U(%s -> %s) ' % (self.str_evra(pkg), + self.str_evra(inst)) elif len(arch_match) == 0: # This instance is not installed. self.instance_status[inst]['installed'] = False - self.logger.info(" %s is not installed." % self.str_evra(inst)) - qtext_versions = qtext_versions + 'I(%s) ' % self.str_evra(inst) + self.logger.info(" %s is not installed." % + self.str_evra(inst)) + qtext_versions = qtext_versions + \ + 'I(%s) ' % self.str_evra(inst) # Check the rpm verify results. for inst in instances: instance_fail = False # Dump the rpm verify results. #****Write something to format this nicely.***** - if self.setup['debug'] and self.instance_status[inst].get('verify', None): + if (Bcfg2.Options.setup.debug and + self.instance_status[inst].get('verify', None)): self.logger.debug(self.instance_status[inst]['verify']) self.instance_status[inst]['verify_fail'] = False if self.instance_status[inst].get('verify', None): if len(self.instance_status[inst].get('verify')) > 1: - self.logger.info("WARNING: Verification of more than one package instance.") + self.logger.info("WARNING: Verification of more " + "than one package instance.") for result in self.instance_status[inst]['verify']: # Check header results if result.get('hdr', None): instance_fail = True - self.instance_status[inst]['verify_fail'] = True + self.instance_status[inst]['verify_fail'] = \ + True # Check dependency results if result.get('deps', None): instance_fail = True - self.instance_status[inst]['verify_fail'] = True - - # Check the rpm verify file results against the modlist - # and entry and per Instance Ignores. - ignores = [ig.get('name') for ig in entry.findall('Ignore')] + \ - [ig.get('name') for ig in inst.findall('Ignore')] + \ - self.ignores + self.instance_status[inst]['verify_fail'] = \ + True + + # check the rpm verify file results against + # the modlist and entry and per Instance Ignores. + ignores = [ig.get('name') + for ig in entry.findall('Ignore')] + \ + [ig.get('name') + for ig in inst.findall('Ignore')] + \ + self.ignores for file_result in result.get('files', []): if file_result[-1] not in modlist + ignores: instance_fail = True - self.instance_status[inst]['verify_fail'] = True + self.instance_status[inst]['verify_fail'] \ + = True else: - self.logger.debug(" Modlist/Ignore match: %s" % \ - (file_result[-1])) + self.logger.debug(" Modlist/Ignore " + "match: %s" % + (file_result[-1])) - if instance_fail == True: - self.logger.debug("*** Instance %s failed RPM verification ***" % \ + if instance_fail: + self.logger.debug("*** Instance %s failed RPM " + "verification ***" % self.str_evra(inst)) - qtext_versions = qtext_versions + 'R(%s) ' % self.str_evra(inst) + qtext_versions = qtext_versions + \ + 'R(%s) ' % self.str_evra(inst) self.modlists[entry] = modlist - # Attach status structure for return to server for reporting. - inst.set('verify_status', str(self.instance_status[inst])) + # Attach status structure for reporting. + inst.set('verify_status', + str(self.instance_status[inst])) - if self.instance_status[inst]['installed'] == False or \ - self.instance_status[inst].get('version_fail', False)== True or \ - self.instance_status[inst].get('verify_fail', False) == True: + version_fail = self.instance_status[inst].get( + 'version_fail', False) + verify_fail = self.instance_status[inst].get( + 'verify_fail', False) + if not self.instance_status[inst]['installed'] or \ + version_fail or verify_fail: package_fail = True self.instance_status[inst]['pkg'] = entry self.modlists[entry] = modlist # Find Installed Instances that are not in the Config. - extra_installed = self.FindExtraInstances(entry, self.installed[entry.get('name')]) - if extra_installed != None: + extra_installed = self.FindExtraInstances( + entry, self.installed[entry.get('name')]) + if extra_installed is not None: package_fail = True self.extra_instances.append(extra_installed) for inst in extra_installed.findall('Instance'): - qtext_versions = qtext_versions + 'D(%s) ' % self.str_evra(inst) - self.logger.debug("Found Extra Instances %s" % qtext_versions) - - if package_fail == True: - self.logger.info(" Package %s failed verification." % \ - (entry.get('name'))) - qtext = 'Install/Upgrade/delete Package %s instance(s) - %s (y/N) ' % \ - (entry.get('name'), qtext_versions) + qtext_versions = qtext_versions + \ + 'D(%s) ' % self.str_evra(inst) + self.logger.debug("Found Extra Instances %s" % + qtext_versions) + + if package_fail: + self.logger.info(" Package %s failed verification." + % (entry.get('name'))) + qtext = 'Install/Upgrade/delete Package %s instance(s) - '\ + '%s (y/N) ' % (entry.get('name'), qtext_versions) entry.set('qtext', qtext) bcfg2_versions = '' - for bcfg2_inst in [inst for inst in instances if inst.tag == 'Instance']: - bcfg2_versions = bcfg2_versions + '(%s) ' % self.str_evra(bcfg2_inst) + for bcfg2_inst in [inst for inst in instances + if inst.tag == 'Instance']: + bcfg2_versions = bcfg2_versions + \ + '(%s) ' % self.str_evra(bcfg2_inst) if bcfg2_versions != '': entry.set('version', bcfg2_versions) installed_versions = '' for installed_inst in self.installed[entry.get('name')]: - installed_versions = installed_versions + '(%s) ' % \ - self.str_evra(installed_inst) + installed_versions = installed_versions + \ + '(%s) ' % self.str_evra(installed_inst) entry.set('current_version', installed_versions) return False else: # There are no Instances of this package installed. - self.logger.debug("Package %s has no instances installed" % (entry.get('name'))) + self.logger.debug("Package %s has no instances installed" % + (entry.get('name'))) entry.set('current_exists', 'false') bcfg2_versions = '' for inst in instances: - qtext_versions = qtext_versions + 'I(%s) ' % self.str_evra(inst) + qtext_versions = qtext_versions + \ + 'I(%s) ' % self.str_evra(inst) self.instance_status.setdefault(inst, {})['installed'] = False self.modlists[entry] = modlist self.instance_status[inst]['pkg'] = entry if inst.tag == 'Instance': - bcfg2_versions = bcfg2_versions + '(%s) ' % self.str_evra(inst) + bcfg2_versions = bcfg2_versions + \ + '(%s) ' % self.str_evra(inst) if bcfg2_versions != '': entry.set('version', bcfg2_versions) - entry.set('qtext', "Install Package %s Instance(s) %s? (y/N) " % \ + entry.set('qtext', "Install Package %s Instance(s) %s? (y/N) " % (entry.get('name'), qtext_versions)) return False @@ -421,26 +1614,31 @@ class RPM(Bcfg2.Client.Tools.PkgTool): for pkg in packages: for inst in pkg: if pkg.get('name') != 'gpg-pubkey': - pkgspec = { 'name':pkg.get('name'), - 'epoch':inst.get('epoch', None), - 'version':inst.get('version'), - 'release':inst.get('release'), - 'arch':inst.get('arch') } + pkgspec = {'name': pkg.get('name'), + 'epoch': inst.get('epoch', None), + 'version': inst.get('version'), + 'release': inst.get('release'), + 'arch': inst.get('arch')} pkgspec_list.append(pkgspec) else: - pkgspec = { 'name':pkg.get('name'), - 'version':inst.get('version'), - 'release':inst.get('release')} - self.logger.info("WARNING: gpg-pubkey package not in configuration %s %s"\ - % (pkgspec.get('name'), self.str_evra(pkgspec))) - self.logger.info(" This package will be deleted in a future version of the RPM driver.") + pkgspec = {'name': pkg.get('name'), + 'version': inst.get('version'), + 'release': inst.get('release')} + self.logger.info("WARNING: gpg-pubkey package not in " + "configuration %s %s" % + (pkgspec.get('name'), + self.str_evra(pkgspec))) + self.logger.info(" This package will be deleted " + "in a future version of the RPM driver.") #pkgspec_list.append(pkg_spec) - erase_results = rpmtools.rpm_erase(pkgspec_list, self.erase_flags) + erase_results = rpm_erase(pkgspec_list, + Bcfg2.Options.setup.rpm_erase_flags) if erase_results == []: self.modified += packages for pkg in pkgspec_list: - self.logger.info("Deleted %s %s" % (pkg.get('name'), self.str_evra(pkg))) + self.logger.info("Deleted %s %s" % (pkg.get('name'), + self.str_evra(pkg))) else: self.logger.info("Bulk erase failed with errors:") self.logger.debug("Erase results = %s" % erase_results) @@ -450,30 +1648,38 @@ class RPM(Bcfg2.Client.Tools.PkgTool): pkg_modified = False for inst in pkg: if pkg.get('name') != 'gpg-pubkey': - pkgspec = { 'name':pkg.get('name'), - 'epoch':inst.get('epoch', None), - 'version':inst.get('version'), - 'release':inst.get('release'), - 'arch':inst.get('arch') } + pkgspec = {'name': pkg.get('name'), + 'epoch': inst.get('epoch', None), + 'version': inst.get('version'), + 'release': inst.get('release'), + 'arch': inst.get('arch')} pkgspec_list.append(pkgspec) else: - pkgspec = { 'name':pkg.get('name'), - 'version':inst.get('version'), - 'release':inst.get('release')} - self.logger.info("WARNING: gpg-pubkey package not in configuration %s %s"\ - % (pkgspec.get('name'), self.str_evra(pkgspec))) - self.logger.info(" This package will be deleted in a future version of the RPM driver.") - continue # Don't delete the gpg-pubkey packages for now. - erase_results = rpmtools.rpm_erase([pkgspec], self.erase_flags) + pkgspec = {'name': pkg.get('name'), + 'version': inst.get('version'), + 'release': inst.get('release')} + self.logger.info("WARNING: gpg-pubkey package not in " + "configuration %s %s" % + (pkgspec.get('name'), + self.str_evra(pkgspec))) + self.logger.info(" This package will be " + "deleted in a future version of the " + "RPM driver.") + continue # don't delete the gpg-pubkey packages + erase_results = rpm_erase( + [pkgspec], + Bcfg2.Options.setup.rpm_erase_flags) if erase_results == []: pkg_modified = True - self.logger.info("Deleted %s %s" % \ - (pkgspec.get('name'), self.str_evra(pkgspec))) + self.logger.info("Deleted %s %s" % + (pkgspec.get('name'), + self.str_evra(pkgspec))) else: - self.logger.error("unable to delete %s %s" % \ - (pkgspec.get('name'), self.str_evra(pkgspec))) + self.logger.error("unable to delete %s %s" % + (pkgspec.get('name'), + self.str_evra(pkgspec))) self.logger.debug("Failure = %s" % erase_results) - if pkg_modified == True: + if pkg_modified: self.modified.append(pkg) self.RefreshPackages() @@ -489,33 +1695,35 @@ class RPM(Bcfg2.Client.Tools.PkgTool): """ fix = False - if inst_status.get('installed', False) == False: - if instance.get('installed_action', 'install') == "install" and \ - self.installed_action == "install": + if not inst_status.get('installed', False): + if (instance.get('install_missing', 'true').lower() == "true" and + Bcfg2.Options.setup.rpm_install_missing): fix = True else: - self.logger.debug('Installed Action for %s %s is to not install' % \ + self.logger.debug('Installed Action for %s %s is to not ' + 'install' % (inst_status.get('pkg').get('name'), self.str_evra(instance))) - elif inst_status.get('version_fail', False) == True: - if instance.get('version_fail_action', 'upgrade') == "upgrade" and \ - self.version_fail_action == "upgrade": + elif inst_status.get('version_fail', False): + if (instance.get('fix_version', 'true').lower() == "true" and + Bcfg2.Options.setup.rpm_fix_version): fix = True else: - self.logger.debug('Version Fail Action for %s %s is to not upgrade' % \ + self.logger.debug('Version Fail Action for %s %s is to ' + 'not upgrade' % (inst_status.get('pkg').get('name'), self.str_evra(instance))) - elif inst_status.get('verify_fail', False) == True and self.name == "RPM": - # yum can't reinstall packages so only do this for rpm. - if instance.get('verify_fail_action', 'reinstall') == "reinstall" and \ - self.verify_fail_action == "reinstall": + elif inst_status.get('verify_fail', False): + if (instance.get('reinstall_broken', 'true').lower() == "true" and + Bcfg2.Options.setup.rpm_reinstall_broken): for inst in inst_status.get('verify'): # This needs to be a for loop rather than a straight get() # because the underlying routines handle multiple packages # and return a list of results. - self.logger.debug('reinstall_check: %s %s:%s-%s.%s' % inst.get('nevra')) + self.logger.debug('reinstall_check: %s %s:%s-%s.%s' % + inst.get('nevra')) if inst.get("hdr", False): fix = True @@ -523,7 +1731,8 @@ class RPM(Bcfg2.Client.Tools.PkgTool): elif inst.get('files', False): # Parse rpm verify file results for file_result in inst.get('files', []): - self.logger.debug('reinstall_check: file: %s' % file_result) + self.logger.debug('reinstall_check: file: %s' % + file_result) if file_result[-2] != 'c': fix = True break @@ -532,13 +1741,14 @@ class RPM(Bcfg2.Client.Tools.PkgTool): elif inst.get("deps", False): fix = False else: - self.logger.debug('Verify Fail Action for %s %s is to not reinstall' % \ - (inst_status.get('pkg').get('name'), - self.str_evra(instance))) + self.logger.debug('Verify Fail Action for %s %s is to not ' + 'reinstall' % + (inst_status.get('pkg').get('name'), + self.str_evra(instance))) return fix - def Install(self, packages, states): + def Install(self, packages): """ Try and fix everything that RPM.VerifyPackages() found wrong for each Package Entry. This can result in individual RPMs being @@ -559,6 +1769,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): """ self.logger.info('Runing RPM.Install()') + states = dict() install_only_pkgs = [] gpg_keys = [] upgrade_pkgs = [] @@ -566,20 +1777,21 @@ class RPM(Bcfg2.Client.Tools.PkgTool): # Remove extra instances. # Can not reverify because we don't have a package entry. if len(self.extra_instances) > 0: - if (self.setup.get('remove') == 'all' or \ - self.setup.get('remove') == 'packages') and\ - not self.setup.get('dryrun'): + if (Bcfg2.Options.setup.remove in ['all', 'packages'] and + not Bcfg2.Options.setup.dry_run): self.Remove(self.extra_instances) else: - self.logger.info("The following extra package instances will be removed by the '-r' option:") + self.logger.info("The following extra package instances will " + "be removed by the '-r' option:") for pkg in self.extra_instances: for inst in pkg: - self.logger.info(" %s %s" % (pkg.get('name'), self.str_evra(inst))) + self.logger.info(" %s %s" % (pkg.get('name'), + self.str_evra(inst))) # Figure out which instances of the packages actually need something # doing to them and place in the appropriate work 'queue'. for pkg in packages: - for inst in [instn for instn in pkg if instn.tag \ + for inst in [instn for instn in pkg if instn.tag in ['Instance', 'Package']]: if self.FixInstance(inst, self.instance_status[inst]): if pkg.get('name') == 'gpg-pubkey': @@ -592,10 +1804,10 @@ class RPM(Bcfg2.Client.Tools.PkgTool): # Fix installOnlyPackages if len(install_only_pkgs) > 0: self.logger.info("Attempting to install 'install only packages'") - install_args = \ - " ".join(os.path.join(self.instance_status[inst].get('pkg').get('uri'), - inst.get('simplefile')) - for inst in install_only_pkgs) + install_args = " ".join(os.path.join( + self.instance_status[inst].get('pkg').get('uri'), + inst.get('simplefile')) + for inst in install_only_pkgs) if self.cmd.run("rpm --install --quiet --oldpackage --replacepkgs " "%s" % install_args): # The rpm command succeeded. All packages installed. @@ -607,35 +1819,34 @@ class RPM(Bcfg2.Client.Tools.PkgTool): self.logger.error("Single Pass for InstallOnlyPackages Failed") installed_instances = [] for inst in install_only_pkgs: - install_args = \ - os.path.join(self.instance_status[inst].get('pkg').get('uri'), - inst.get('simplefile')) + pkguri = self.instance_status[inst].get('pkg').get('uri') + pkgname = self.instance_status[inst].get('pkg').get('name') + install_args = os.path.join(pkguri, inst.get('simplefile')) if self.cmd.run("rpm --install --quiet --oldpackage " "--replacepkgs %s" % install_args): installed_instances.append(inst) else: - self.logger.debug("InstallOnlyPackage %s %s would not install." % \ - (self.instance_status[inst].get('pkg').get('name'), \ - self.str_evra(inst))) + self.logger.debug("InstallOnlyPackage %s %s would not " + "install." % (pkgname, + self.str_evra(inst))) - install_pkg_set = set([self.instance_status[inst].get('pkg') \ - for inst in install_only_pkgs]) + install_pkg_set = set([self.instance_status[inst].get('pkg') + for inst in install_only_pkgs]) self.RefreshPackages() # Install GPG keys. if len(gpg_keys) > 0: for inst in gpg_keys: self.logger.info("Installing GPG keys.") - key_arg = os.path.join(self.instance_status[inst].get('pkg').get('uri'), \ - inst.get('simplefile')) + pkguri = self.instance_status[inst].get('pkg').get('uri') + pkgname = self.instance_status[inst].get('pkg').get('name') + key_arg = os.path.join(pkguri, inst.get('simplefile')) if not self.cmd.run("rpm --import %s" % key_arg): self.logger.debug("Unable to install %s-%s" % - (self.instance_status[inst].get('pkg').get('name'), - self.str_evra(inst))) + (pkgname, self.str_evra(inst))) else: self.logger.debug("Installed %s-%s-%s" % - (self.instance_status[inst].get('pkg').get('name'), - inst.get('version'), + (pkgname, inst.get('version'), inst.get('release'))) self.RefreshPackages() self.gpg_keyids = self.getinstalledgpg() @@ -645,9 +1856,10 @@ class RPM(Bcfg2.Client.Tools.PkgTool): # Fix upgradeable packages. if len(upgrade_pkgs) > 0: self.logger.info("Attempting to upgrade packages") - upgrade_args = " ".join([os.path.join(self.instance_status[inst].get('pkg').get('uri'), \ - inst.get('simplefile')) \ - for inst in upgrade_pkgs]) + upgrade_args = " ".join([os.path.join( + self.instance_status[inst].get('pkg').get('uri'), + inst.get('simplefile')) + for inst in upgrade_pkgs]) if self.cmd.run("rpm --upgrade --quiet --oldpackage --replacepkgs " "%s" % upgrade_args): # The rpm command succeeded. All packages upgraded. @@ -661,30 +1873,38 @@ class RPM(Bcfg2.Client.Tools.PkgTool): self.logger.error("Single Pass for Upgrading Packages Failed") upgraded_instances = [] for inst in upgrade_pkgs: - upgrade_args = os.path.join(self.instance_status[inst].get('pkg').get('uri'), \ - inst.get('simplefile')) - #self.logger.debug("rpm --upgrade --quiet --oldpackage --replacepkgs %s" % \ - # upgrade_args) + upgrade_args = os.path.join( + self.instance_status[inst].get('pkg').get('uri'), + inst.get('simplefile')) + #self.logger.debug("rpm --upgrade --quiet --oldpackage " + # "--replacepkgs %s" % upgrade_args) if self.cmd.run("rpm --upgrade --quiet --oldpackage " "--replacepkgs %s" % upgrade_args): upgraded_instances.append(inst) else: - self.logger.debug("Package %s %s would not upgrade." % - (self.instance_status[inst].get('pkg').get('name'), - self.str_evra(inst))) + self.logger.debug( + "Package %s %s would not upgrade." % + (self.instance_status[inst].get('pkg').get('name'), + self.str_evra(inst))) - upgrade_pkg_set = set([self.instance_status[inst].get('pkg') \ - for inst in upgrade_pkgs]) + upgrade_pkg_set = set([self.instance_status[inst].get('pkg') + for inst in upgrade_pkgs]) self.RefreshPackages() - if not self.setup['kevlar']: + if not Bcfg2.Options.setup.kevlar: for pkg_entry in packages: - self.logger.debug("Reverifying Failed Package %s" % (pkg_entry.get('name'))) - states[pkg_entry] = self.VerifyPackage(pkg_entry, \ - self.modlists.get(pkg_entry, [])) + self.logger.debug("Reverifying Failed Package %s" % + (pkg_entry.get('name'))) + states[pkg_entry] = self.VerifyPackage( + pkg_entry, self.modlists.get(pkg_entry, [])) - for entry in [ent for ent in packages if states[ent]]: - self.modified.append(entry) + self.modified.extend(ent for ent in packages if states[ent]) + return states + + def _log_incomplete_entry_install(self, etag, ename): + self.logger.error("Incomplete information for entry %s:%s; " + "cannot install" % (etag, ename)) + return def canInstall(self, entry): """Test if entry has enough information to be installed.""" @@ -692,18 +1912,17 @@ class RPM(Bcfg2.Client.Tools.PkgTool): return False if 'failure' in entry.attrib: - self.logger.error("Cannot install entry %s:%s with bind failure" % \ + self.logger.error("Cannot install entry %s:%s with bind failure" % (entry.tag, entry.get('name'))) return False - instances = entry.findall('Instance') - # If the entry wasn't verifiable, then we really don't want to try and fix something - # that we don't know is broken. + # If the entry wasn't verifiable, then we really don't want to try + # and fix something that we don't know is broken. if not self.canVerify(entry): - self.logger.debug("WARNING: Package %s was not verifiable, not passing to Install()" \ - % entry.get('name')) + self.logger.debug("WARNING: Package %s was not verifiable, not " + "passing to Install()" % entry.get('name')) return False if not instances: @@ -711,53 +1930,70 @@ class RPM(Bcfg2.Client.Tools.PkgTool): if entry.get('name') == 'gpg-pubkey': # gpg-pubkey packages aren't really pacakges, so we have to do # something a little different. - # Check that the Package Level has what we need for verification. - if [attr for attr in self.__gpg_ireq__[entry.tag] if attr not in entry.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot install" \ - % (entry.tag, entry.get('name'))) + # check that the Package level has + # what we need for verification. + if [attr for attr in self.__gpg_ireq__[entry.tag] + if attr not in entry.attrib]: + self._log_incomplete_entry_install(entry.tag, + entry.get('name')) return False else: - if [attr for attr in self.__ireq__[entry.tag] if attr not in entry.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot install" \ - % (entry.tag, entry.get('name'))) + if [attr for attr in self.__ireq__[entry.tag] + if attr not in entry.attrib]: + self._log_incomplete_entry_install(entry.tag, + entry.get('name')) return False else: if entry.get('name') == 'gpg-pubkey': # gpg-pubkey packages aren't really pacakges, so we have to do # something a little different. - # Check that the Package Level has what we need for verification. - if [attr for attr in self.__new_gpg_ireq__[entry.tag] if attr not in entry.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot install" \ - % (entry.tag, entry.get('name'))) + # check that the Package level has + # what we need for verification. + if [attr for attr in self.__new_gpg_ireq__[entry.tag] + if attr not in entry.attrib]: + self._log_incomplete_entry_install(entry.tag, + entry.get('name')) return False - # Check that the Instance Level has what we need for verification. + # check that the Instance level has + # what we need for verification. for inst in instances: - if [attr for attr in self.__new_gpg_ireq__[inst.tag] \ - if attr not in inst.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot install"\ - % (inst.tag, entry.get('name'))) + if [attr for attr in self.__new_gpg_ireq__[inst.tag] + if attr not in inst.attrib]: + self._log_incomplete_entry_install(inst.tag, + entry.get('name')) return False else: # New format with Instances. - # Check that the Package Level has what we need for verification. - if [attr for attr in self.__new_ireq__[entry.tag] if attr not in entry.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot install" \ - % (entry.tag, entry.get('name'))) - self.logger.error(" Required attributes that may not be present are %s" \ - % (self.__new_ireq__[entry.tag])) + # check that the Package level has + # what we need for verification. + if [attr for attr in self.__new_ireq__[entry.tag] + if attr not in entry.attrib]: + self._log_incomplete_entry_install(entry.tag, + entry.get('name')) + self.logger.error(" Required attributes that " + "may not be present are %s" % + (self.__new_ireq__[entry.tag])) return False - # Check that the Instance Level has what we need for verification. + # check that the Instance level has + # what we need for verification. for inst in instances: if inst.tag == 'Instance': - if [attr for attr in self.__new_ireq__[inst.tag] \ - if attr not in inst.attrib]: - self.logger.error("Incomplete information for %s of package %s; cannot install" \ - % (inst.tag, entry.get('name'))) - self.logger.error(" Required attributes that may not be present are %s" \ + if [attr for attr in self.__new_ireq__[inst.tag] + if attr not in inst.attrib]: + self._log_incomplete_entry_install( + inst.tag, + entry.get('name')) + self.logger.error(" Required attributes " + "that may not be present are %s" % (self.__new_ireq__[inst.tag])) return False return True + def _log_incomplete_entry_verify(self, etag, ename): + self.logger.error("Incomplete information for entry %s:%s; " + "cannot verify" % (etag, ename)) + return + def canVerify(self, entry): """ Test if entry has enough information to be verified. @@ -775,13 +2011,15 @@ class RPM(Bcfg2.Client.Tools.PkgTool): return False if 'failure' in entry.attrib: - self.logger.error("Entry %s:%s reports bind failure: %s" % \ - (entry.tag, entry.get('name'), entry.get('failure'))) + self.logger.error("Entry %s:%s reports bind failure: %s" % + (entry.tag, entry.get('name'), + entry.get('failure'))) return False - # We don't want to do any checks so we don't care what the entry has in it. - if (not self.pkg_checks or - entry.get('pkg_checks', 'true').lower() == 'false'): + # we don't want to do any checks so + # we don't care what the entry has in it. + if (not Bcfg2.Options.setup.rpm_pkg_checks or + entry.get('pkg_checks', 'true').lower() == 'false'): return True instances = entry.findall('Instance') @@ -791,53 +2029,72 @@ class RPM(Bcfg2.Client.Tools.PkgTool): if entry.get('name') == 'gpg-pubkey': # gpg-pubkey packages aren't really pacakges, so we have to do # something a little different. - # Check that the Package Level has what we need for verification. - if [attr for attr in self.__gpg_req__[entry.tag] if attr not in entry.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot verify" \ - % (entry.tag, entry.get('name'))) + # check that the Package level has + # what we need for verification. + if [attr for attr in self.__gpg_req__[entry.tag] + if attr not in entry.attrib]: + self._log_incomplete_entry_verify(entry.tag, + entry.get('name')) return False elif entry.tag == 'Path' and entry.get('type') == 'ignore': # ignored Paths are only relevant during failed package # verification pass else: - if [attr for attr in self.__req__[entry.tag] if attr not in entry.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot verify" \ - % (entry.tag, entry.get('name'))) + if [attr for attr in self.__req__[entry.tag] + if attr not in entry.attrib]: + self._log_incomplete_entry_verify(entry.tag, + entry.get('name')) return False else: if entry.get('name') == 'gpg-pubkey': # gpg-pubkey packages aren't really pacakges, so we have to do # something a little different. - # Check that the Package Level has what we need for verification. - if [attr for attr in self.__new_gpg_req__[entry.tag] if attr not in entry.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot verify" \ - % (entry.tag, entry.get('name'))) + # check that the Package level has + # what we need for verification. + if [attr for attr in self.__new_gpg_req__[entry.tag] + if attr not in entry.attrib]: + self._log_incomplete_entry_verify(entry.tag, + entry.get('name')) return False - # Check that the Instance Level has what we need for verification. + # check that the Instance level has + # what we need for verification. for inst in instances: - if [attr for attr in self.__new_gpg_req__[inst.tag] \ - if attr not in inst.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot verify" \ - % (inst.tag, inst.get('name'))) + if [attr for attr in self.__new_gpg_req__[inst.tag] + if attr not in inst.attrib]: + self._log_incomplete_entry_verify(inst.tag, + inst.get('name')) return False else: - # New format with Instances, or old style modified. - # Check that the Package Level has what we need for verification. - if [attr for attr in self.__new_req__[entry.tag] if attr not in entry.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot verify" \ - % (entry.tag, entry.get('name'))) + # new format with Instances, or old style modified. + # check that the Package level has + # what we need for verification. + if [attr for attr in self.__new_req__[entry.tag] + if attr not in entry.attrib]: + self._log_incomplete_entry_verify(entry.tag, + entry.get('name')) return False - # Check that the Instance Level has what we need for verification. + # check that the Instance level has + # what we need for verification. for inst in instances: if inst.tag == 'Instance': - if [attr for attr in self.__new_req__[inst.tag] \ - if attr not in inst.attrib]: - self.logger.error("Incomplete information for entry %s:%s; cannot verify" \ - % (inst.tag, inst.get('name'))) + if [attr for attr in self.__new_req__[inst.tag] + if attr not in inst.attrib]: + self._log_incomplete_entry_verify(inst.tag, + inst.get('name')) return False return True + def _get_tmp_entry(self, extra_entry, inst): + tmp_entry = Bcfg2.Client.XML.SubElement(extra_entry, 'Instance', + version=inst.get('version'), + release=inst.get('release')) + if inst.get('epoch', None) is not None: + tmp_entry.set('epoch', str(inst.get('epoch'))) + if installed_inst.get('arch', None) is not None: + tmp_entry.set('arch', inst.get('arch')) + return + def FindExtra(self): """Find extra packages.""" packages = [entry.get('name') for entry in self.getSupportedEntries()] @@ -845,22 +2102,17 @@ class RPM(Bcfg2.Client.Tools.PkgTool): for (name, instances) in list(self.installed.items()): if name not in packages: - extra_entry = Bcfg2.Client.XML.Element('Package', name=name, type=self.pkgtype) + extra_entry = Bcfg2.Client.XML.Element('Package', + name=name, + type=self.pkgtype) for installed_inst in instances: - if self.setup['extra']: - self.logger.info("Extra Package %s %s." % \ + if Bcfg2.Options.setup.extra: + self.logger.info("Extra Package %s %s." % (name, self.str_evra(installed_inst))) - tmp_entry = Bcfg2.Client.XML.SubElement(extra_entry, 'Instance', \ - version = installed_inst.get('version'), \ - release = installed_inst.get('release')) - if installed_inst.get('epoch', None) != None: - tmp_entry.set('epoch', str(installed_inst.get('epoch'))) - if installed_inst.get('arch', None) != None: - tmp_entry.set('arch', installed_inst.get('arch')) + self._get_tmp_entry(extra_entry, installed_inst) extras.append(extra_entry) return extras - def FindExtraInstances(self, pkg_entry, installed_entry): """ Check for installed instances that are not in the config. @@ -869,8 +2121,11 @@ class RPM(Bcfg2.Client.Tools.PkgTool): """ name = pkg_entry.get('name') - extra_entry = Bcfg2.Client.XML.Element('Package', name=name, type=self.pkgtype) - instances = [inst for inst in pkg_entry if inst.tag == 'Instance' or inst.tag == 'Package'] + extra_entry = Bcfg2.Client.XML.Element('Package', + name=name, + type=self.pkgtype) + instances = [inst for inst in pkg_entry if + inst.tag == 'Instance' or inst.tag == 'Package'] if name in self.installOnlyPkgs: for installed_inst in installed_entry: not_found = True @@ -879,36 +2134,25 @@ class RPM(Bcfg2.Client.Tools.PkgTool): self.inst_evra_equal(inst, installed_inst): not_found = False break - if not_found == True: + if not_found: # Extra package. - self.logger.info("Extra InstallOnlyPackage %s %s." % \ + self.logger.info("Extra InstallOnlyPackage %s %s." % (name, self.str_evra(installed_inst))) - tmp_entry = Bcfg2.Client.XML.SubElement(extra_entry, 'Instance', \ - version = installed_inst.get('version'), \ - release = installed_inst.get('release')) - if installed_inst.get('epoch', None) != None: - tmp_entry.set('epoch', str(installed_inst.get('epoch'))) - if installed_inst.get('arch', None) != None: - tmp_entry.set('arch', installed_inst.get('arch')) + self._get_tmp_entry(extra_entry, installed_inst) else: # Normal package, only check arch. for installed_inst in installed_entry: not_found = True for inst in instances: - if installed_inst.get('arch', None) == inst.get('arch', None) or\ - inst.tag == 'Package': + if (installed_inst.get('arch', None) == + inst.get('arch', None) or + inst.tag == 'Package'): not_found = False break if not_found: - self.logger.info("Extra Normal Package Instance %s %s" % \ + self.logger.info("Extra Normal Package Instance %s %s" % (name, self.str_evra(installed_inst))) - tmp_entry = Bcfg2.Client.XML.SubElement(extra_entry, 'Instance', \ - version = installed_inst.get('version'), \ - release = installed_inst.get('release')) - if installed_inst.get('epoch', None) != None: - tmp_entry.set('epoch', str(installed_inst.get('epoch'))) - if installed_inst.get('arch', None) != None: - tmp_entry.set('arch', installed_inst.get('arch')) + self._get_tmp_entry(extra_entry, installed_inst) if len(extra_entry) == 0: extra_entry = None @@ -932,9 +2176,10 @@ class RPM(Bcfg2.Client.Tools.PkgTool): Compare old style entry to installed entry. Which means ignore the epoch and arch. ''' - if (config_entry.tag == 'Package' and \ - config_entry.get('version') == installed_entry.get('version') and \ - config_entry.get('release') == installed_entry.get('release')): + if (config_entry.tag == 'Package' and + config_entry.get('version') == installed_entry.get('version') + and + config_entry.get('release') == installed_entry.get('release')): return True else: return False @@ -942,18 +2187,19 @@ class RPM(Bcfg2.Client.Tools.PkgTool): def inst_evra_equal(self, config_entry, installed_entry): """Compare new style instance to installed entry.""" - if config_entry.get('epoch', None) != None: + if config_entry.get('epoch', None) is not None: epoch = int(config_entry.get('epoch')) else: epoch = None - if (config_entry.tag == 'Instance' and \ - (epoch == installed_entry.get('epoch', 0) or \ - (epoch == 0 and installed_entry.get('epoch', 0) == None) or \ - (epoch == None and installed_entry.get('epoch', 0) == 0)) and \ - config_entry.get('version') == installed_entry.get('version') and \ - config_entry.get('release') == installed_entry.get('release') and \ - config_entry.get('arch', None) == installed_entry.get('arch', None)): + if (config_entry.tag == 'Instance' and + (epoch == installed_entry.get('epoch', 0) or + (epoch == 0 and installed_entry.get('epoch', 0) is None) or + (epoch is None and installed_entry.get('epoch', 0) == 0)) and + config_entry.get('version') == installed_entry.get('version') and + config_entry.get('release') == installed_entry.get('release') and + config_entry.get('arch', None) == installed_entry.get('arch', + None)): return True else: return False @@ -966,10 +2212,10 @@ class RPM(Bcfg2.Client.Tools.PkgTool): (big-endian) of the key ID which is good enough for our purposes. """ - init_ts = rpmtools.rpmtransactionset() - init_ts.setVSFlags(rpm._RPMVSF_NODIGESTS|rpm._RPMVSF_NOSIGNATURES) - gpg_hdrs = rpmtools.getheadersbykeyword(init_ts, **{'name':'gpg-pubkey'}) - keyids = [ header[rpm.RPMTAG_VERSION] for header in gpg_hdrs] + init_ts = rpmtransactionset() + init_ts.setVSFlags(rpm._RPMVSF_NODIGESTS | rpm._RPMVSF_NOSIGNATURES) + gpg_hdrs = getheadersbykeyword(init_ts, **{'name': 'gpg-pubkey'}) + keyids = [header[rpm.RPMTAG_VERSION] for header in gpg_hdrs] keyids.append('None') init_ts.closeDB() del init_ts diff --git a/src/lib/Bcfg2/Client/Tools/RPMng.py b/src/lib/Bcfg2/Client/Tools/RPMng.py deleted file mode 100644 index 0f0e4c700..000000000 --- a/src/lib/Bcfg2/Client/Tools/RPMng.py +++ /dev/null @@ -1,9 +0,0 @@ -""" RPM driver called 'RPMng' for backwards compat """ - -from Bcfg2.Client.Tools.RPM import RPM - - -class RPMng(RPM): - """ RPM driver called 'RPMng' for backwards compat """ - deprecated = True - name = "RPM" diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index e0c913dcd..a482dbc00 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -98,10 +98,10 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): # make sure service is disabled on boot bootcmd = '/sbin/rc-update del %s default' bootcmdrv = self.cmd.run(bootcmd % entry.get('name')).success - if self.setup['servicemode'] == 'disabled': + if Bcfg2.Options.setup.service_mode == 'disabled': # 'disabled' means we don't attempt to modify running svcs return bootcmdrv - buildmode = self.setup['servicemode'] == 'build' + buildmode = Bcfg2.Options.setup.service_mode == 'build' if (entry.get('status') == 'on' and not buildmode) and \ entry.get('current_status') == 'off': svccmdrv = self.start_service(entry) diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 0b4aba60d..7b5ff7813 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -7,6 +7,7 @@ import copy import glob import struct import socket +import logging import selinux import seobject import Bcfg2.Client.XML @@ -77,14 +78,13 @@ class SELinux(Bcfg2.Client.Tools.Tool): SEPort=['name', 'selinuxtype'], SEUser=['name', 'roles', 'prefix']) - def __init__(self, logger, setup, config): - Bcfg2.Client.Tools.Tool.__init__(self, logger, setup, config) + def __init__(self, config): + Bcfg2.Client.Tools.Tool.__init__(self, config) self.handlers = {} for handler in self.__handles__: etype = handler[0] self.handlers[etype] = \ - globals()["SELinux%sHandler" % etype.title()](self, logger, - setup, config) + globals()["SELinux%sHandler" % etype.title()](self, config) self.txn = False self.post_txn_queue = [] @@ -100,10 +100,6 @@ class SELinux(Bcfg2.Client.Tools.Tool): # http://docs.python.org/2/reference/datamodel.html#object.__getattr__ # for details - def BundleUpdated(self, _, states): - for handler in self.handlers.values(): - handler.BundleUpdated(states) - def FindExtra(self): extra = [] for handler in self.handlers.values(): @@ -119,7 +115,7 @@ class SELinux(Bcfg2.Client.Tools.Tool): in the specification """ return self.handlers[entry.tag].primarykey(entry) - def Install(self, entries, states): + def Install(self, entries): # start a transaction semanage = seobject.semanageRecords("") if hasattr(semanage, "start"): @@ -129,13 +125,14 @@ class SELinux(Bcfg2.Client.Tools.Tool): else: self.logger.debug("SELinux transactions not supported; this may " "slow things down considerably") - Bcfg2.Client.Tools.Tool.Install(self, entries, states) + states = Bcfg2.Client.Tools.Tool.Install(self, entries) if hasattr(semanage, "finish"): self.logger.debug("Committing SELinux transaction") semanage.finish() self.txn = False for func, arg, kwargs in self.post_txn_queue: states[arg] = func(*arg, **kwargs) + return states def GenericSEInstall(self, entry): """Dispatch install to the proper method according to entry tag""" @@ -144,7 +141,7 @@ class SELinux(Bcfg2.Client.Tools.Tool): def GenericSEVerify(self, entry, _): """Dispatch verify to the proper method according to entry tag""" rv = self.handlers[entry.tag].Verify(entry) - if entry.get('qtext') and self.setup['interactive']: + if entry.get('qtext') and Bcfg2.Options.setup.interactive: entry.set('qtext', '%s\nInstall %s: (y/N) ' % (entry.get('qtext'), @@ -174,10 +171,9 @@ class SELinuxEntryHandler(object): custom_re = re.compile(r' (?P<name>\S+)$') custom_format = None - def __init__(self, tool, logger, setup, config): + def __init__(self, tool, config): self.tool = tool - self.logger = logger - self.setup = setup + self.logger = logging.getLogger(self.__class__.__name__) self.config = config self._records = None self._all = None @@ -229,7 +225,7 @@ class SELinuxEntryHandler(object): match = self.custom_re.search(cmd) if match: if (len(self.custom_format) == 1 and - self.custom_format[0] == "name"): + self.custom_format[0] == "name"): keys.append(match.group("name")) else: keys.append(tuple([match.group(k) @@ -379,11 +375,6 @@ class SELinuxEntryHandler(object): for key in records.keys() if key not in specified] - def BundleUpdated(self, states): - """ perform any additional magic tasks that need to be run - when a bundle is updated """ - pass - class SELinuxSebooleanHandler(SELinuxEntryHandler): """ handle SELinux boolean entries """ @@ -631,8 +622,8 @@ class SELinuxSeuserHandler(SELinuxEntryHandler): etype = "user" value_format = ("prefix", None, None, "roles") - def __init__(self, tool, logger, setup, config): - SELinuxEntryHandler.__init__(self, tool, logger, setup, config) + def __init__(self, tool, config): + SELinuxEntryHandler.__init__(self, tool, config) self.needs_prefix = False @property @@ -725,9 +716,9 @@ class SELinuxSemoduleHandler(SELinuxEntryHandler): etype = "module" value_format = (None, "disabled") - def __init__(self, tool, logger, setup, config): - SELinuxEntryHandler.__init__(self, tool, logger, setup, config) - self.filetool = POSIXFile(logger, setup, config) + def __init__(self, tool, config): + SELinuxEntryHandler.__init__(self, tool, config) + self.filetool = POSIXFile(config) try: self.setype = selinux.selinux_getpolicytype()[1] except IndexError: diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 27c3d3785..332638de4 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -34,8 +34,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): pkgtype = 'sysv' pkgtool = ("/usr/sbin/pkgadd %s -n -d %%s", (('%s %s', ['url', 'name']))) - def __init__(self, logger, setup, config): - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) + def __init__(self, config): + Bcfg2.Client.Tools.PkgTool.__init__(self, config) # noaskfile needs to live beyond __init__ otherwise file is removed self.noaskfile = tempfile.NamedTemporaryFile() self.noaskname = self.noaskfile.name @@ -81,9 +81,9 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Calling install command: %s" % pkgcmd) return pkgcmd - def Install(self, packages, states): + def Install(self, packages): self.pkgmogrify(packages) - super(SYSV, self).Install(packages, states) + super(SYSV, self).Install(packages) def RefreshPackages(self): """Refresh memory hashes of packages.""" @@ -119,7 +119,7 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Package %s not installed" % entry.get("name")) else: - if self.setup['quick'] or \ + if Bcfg2.Options.setup.quick or \ entry.attrib.get('verify', 'true') == 'false': return True rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 027d91c71..3b60c8285 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,15 +13,25 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} + def get_svc_name(self, service): + """Append .service to name if name doesn't specify a unit type.""" + svc = service.get('name') + if svc.endswith(('.service', '.socket', '.device', '.mount', + '.automount', '.swap', '.target', '.path', + '.timer', '.snapshot', '.slice', '.scope')): + return svc + else: + return '%s.service' % svc + def get_svc_command(self, service, action): - return "/bin/systemctl %s %s.service" % (action, service.get('name')) + return "/bin/systemctl %s %s" % (action, self.get_svc_name(service)) def VerifyService(self, entry, _): """Verify Service status for entry.""" if entry.get('status') == 'ignore': return True - cmd = "/bin/systemctl status %s.service " % (entry.get('name')) + cmd = "/bin/systemctl status %s" % (self.get_svc_name(entry)) rv = self.cmd.run(cmd) if 'Loaded: error' in rv.stdout: diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 4fa2fb5e2..449503b55 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -88,8 +88,10 @@ class VCS(Bcfg2.Client.Tools.Tool): return False try: - client, path = dulwich.client.get_transport_and_path(entry.get('sourceurl')) - remote_refs = client.fetch_pack(path, (lambda x: None), None, None, None) + client, path = dulwich.client.get_transport_and_path( + entry.get('sourceurl')) + remote_refs = client.fetch_pack(path, + (lambda x: None), None, None, None) if expected_rev in remote_refs: expected_rev = remote_refs[expected_rev] except: @@ -119,10 +121,12 @@ class VCS(Bcfg2.Client.Tools.Tool): dulwich.file.ensure_dir_exists(destname) destr = dulwich.repo.Repo.init(destname) - cl, host_path = dulwich.client.get_transport_and_path(entry.get('sourceurl')) + determine_wants = destr.object_store.determine_wants_all + cl, host_path = dulwich.client.get_transport_and_path( + entry.get('sourceurl')) remote_refs = cl.fetch(host_path, destr, - determine_wants=destr.object_store.determine_wants_all, + determine_wants=determine_wants, progress=sys.stdout.write) if entry.get('revision') in remote_refs: diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index a584fec86..a8a80974a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -3,6 +3,7 @@ import copy import os.path import sys +import logging import yum import yum.packages import yum.rpmtrans @@ -10,8 +11,10 @@ import yum.callbacks import yum.Errors import yum.misc import rpmUtils.arch +import rpmUtils.miscutils import Bcfg2.Client.XML import Bcfg2.Client.Tools +import Bcfg2.Options def build_yname(pkgname, inst): @@ -65,13 +68,13 @@ class RPMDisplay(yum.rpmtrans.RPMBaseCallback): """We subclass the default RPM transaction callback so that we can control Yum's verbosity and pipe it through the right logger.""" - def __init__(self, logger): + def __init__(self): yum.rpmtrans.RPMBaseCallback.__init__(self) # we want to log events to *both* the Bcfg2 logger (which goes # to stderr or syslog or wherever the user wants it to go) # *and* the yum file logger, which will go to yum.log (ticket # #1103) - self.bcfg2_logger = logger + self.bcfg2_logger = logging.getLogger(self.__class__.__name__) self.state = None self.package = None @@ -110,13 +113,50 @@ class YumDisplay(yum.callbacks.ProcessTransBaseCallback): """Class to handle display of what step we are in the Yum transaction such as downloading packages, etc.""" - def __init__(self, logger): + def __init__(self): yum.callbacks.ProcessTransBaseCallback.__init__(self) - self.logger = logger + self.logger = logging.getLogger(self.__class__.__name__) class YUM(Bcfg2.Client.Tools.PkgTool): """Support for Yum packages.""" + + options = Bcfg2.Client.Tools.PkgTool.options + [ + Bcfg2.Options.BooleanOption( + cf=('YUM', 'pkg_checks'), default=True, dest="yum_pkg_checks", + help="Perform YUM package checks"), + Bcfg2.Options.BooleanOption( + cf=('YUM', 'pkg_verify'), default=True, dest="yum_pkg_verify", + help="Perform YUM package verify"), + Bcfg2.Options.BooleanOption( + cf=('YUM', 'install_missing'), default=True, + dest="yum_install_missing", + help="Install missing packages"), + Bcfg2.Options.Option( + cf=('YUM', 'erase_flags'), default=["allmatches"], + dest="yum_erase_flags", type=Bcfg2.Options.Types.comma_list, + help="YUM erase flags"), + Bcfg2.Options.BooleanOption( + cf=('YUM', 'fix_version'), default=True, + dest="yum_fix_version", + help="Fix (upgrade or downgrade) packages with the wrong version"), + Bcfg2.Options.BooleanOption( + cf=('YUM', 'reinstall_broken'), default=True, + dest="yum_reinstall_broken", + help="Reinstall packages that fail to verify"), + Bcfg2.Options.Option( + cf=('YUM', 'verify_flags'), default=[], + dest="yum_verify_flags", type=Bcfg2.Options.Types.comma_list, + help="YUM verify flags"), + Bcfg2.Options.Option( + cf=('YUM', 'disabled_plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, dest="yum_disabled_plugins", + help="YUM disabled plugins"), + Bcfg2.Options.Option( + cf=('YUM', 'enabled_plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, dest="yum_enabled_plugins", + help="YUM enabled plugins")] + pkgtype = 'yum' __execs__ = [] __handles__ = [('Package', 'yum'), @@ -126,11 +166,11 @@ class YUM(Bcfg2.Client.Tools.PkgTool): __req__ = {'Package': ['type'], 'Path': ['type']} - conflicts = ['YUM24', 'RPM', 'RPMng', 'YUMng'] + conflicts = ['RPM'] - def __init__(self, logger, setup, config): - self.yumbase = self._loadYumBase(setup=setup, logger=logger) - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) + def __init__(self, config): + self.yumbase = self._loadYumBase() + Bcfg2.Client.Tools.PkgTool.__init__(self, config) self.ignores = [] for struct in config: self.ignores.extend([entry.get('name') @@ -171,60 +211,56 @@ class YUM(Bcfg2.Client.Tools.PkgTool): else: dest[pname] = dict(data) - # Process the Yum section from the config file. These are all - # boolean flags, either we do stuff or we don't - self.pkg_checks = self.setup["yum_pkg_checks"] - self.pkg_verify = self.setup["yum_pkg_verify"] - self.do_install = self.setup["yum_installed_action"] == "install" - self.do_upgrade = self.setup["yum_version_fail_action"] == "upgrade" - self.do_reinst = self.setup["yum_verify_fail_action"] == "reinstall" - self.verify_flags = self.setup["yum_verify_flags"] - self.installonlypkgs = self.yumbase.conf.installonlypkgs if 'gpg-pubkey' not in self.installonlypkgs: self.installonlypkgs.append('gpg-pubkey') - self.logger.debug("Yum: Install missing: %s" % self.do_install) - self.logger.debug("Yum: pkg_checks: %s" % self.pkg_checks) - self.logger.debug("Yum: pkg_verify: %s" % self.pkg_verify) - self.logger.debug("Yum: Upgrade on version fail: %s" % self.do_upgrade) - self.logger.debug("Yum: Reinstall on verify fail: %s" % self.do_reinst) + self.logger.debug("Yum: Install missing: %s" % + Bcfg2.Options.setup.yum_install_missing) + self.logger.debug("Yum: pkg_checks: %s" % + Bcfg2.Options.setup.yum_pkg_checks) + self.logger.debug("Yum: pkg_verify: %s" % + Bcfg2.Options.setup.yum_pkg_verify) + self.logger.debug("Yum: Upgrade on version fail: %s" % + Bcfg2.Options.setup.yum_fix_version) + self.logger.debug("Yum: Reinstall on verify fail: %s" % + Bcfg2.Options.setup.yum_reinstall_broken) self.logger.debug("Yum: installonlypkgs: %s" % self.installonlypkgs) - self.logger.debug("Yum: verify_flags: %s" % self.verify_flags) - self.logger.debug("Yum: disabled_plugins: %s" % - self.setup["yum_disabled_plugins"]) - self.logger.debug("Yum: enabled_plugins: %s" % - self.setup["yum_enabled_plugins"]) - - def _loadYumBase(self, setup=None, logger=None): + self.logger.debug("Yum: verify_flags: %s" % + Bcfg2.Options.setup.yum_verify_flags) + self.logger.debug("Yum: disabled_plugins: %s" % + Bcfg2.Options.setup.yum_disabled_plugins) + self.logger.debug("Yum: enabled_plugins: %s" % + Bcfg2.Options.setup.yum_enabled_plugins) + + def _loadYumBase(self): ''' this may be called before PkgTool.__init__() is called on this object (when the YUM object is first instantiated; PkgTool.__init__() calls RefreshPackages(), which requires a YumBase object already exist), or after __init__() has completed, when we reload the yum config before installing - packages. Consequently, we support both methods by allowing - setup and logger, the only object properties we use in this - function, to be passed as keyword arguments or to be omitted - and drawn from the object itself.''' + packages. ''' rv = yum.YumBase() # pylint: disable=C0103 - if setup is None: - setup = self.setup - if logger is None: + if hasattr(self, "logger"): logger = self.logger + else: + logger = logging.getLogger(self.name) - if setup['debug']: + if Bcfg2.Options.setup.debug: debuglevel = 3 - elif setup['verbose']: + elif Bcfg2.Options.setup.verbose: debuglevel = 2 else: debuglevel = 0 - if setup['yum_disabled_plugins']: - rv.preconf.disabled_plugins = setup['yum_disabled_plugins'] + if len(Bcfg2.Options.setup.yum_disabled_plugins) > 0: + rv.preconf.disabled_plugins = \ + Bcfg2.Options.setup.yum_disabled_plugins - if setup['yum_enabled_plugins']: - rv.preconf.enabled_plugins = setup['yum_enabled_plugins'] + if len(Bcfg2.Options.setup.yum_enabled_plugins) > 0: + rv.preconf.enabled_plugins = \ + Bcfg2.Options.setup.yum_enabled_plugins # pylint: disable=E1121,W0212 try: @@ -252,7 +288,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): to the newest available """ # old style entry; synthesize Instances from current installed if (entry.get('name') not in self.yum_installed and - entry.get('name') not in self.yum_avail): + entry.get('name') not in self.yum_avail): # new entry; fall back to default entry.set('version', 'any') else: @@ -306,7 +342,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): missing = Bcfg2.Client.Tools.PkgTool.missing_attrs(self, entry) if (entry.get('name', None) is None and - entry.get('group', None) is None): + entry.get('group', None) is None): missing += ['name', 'group'] return missing @@ -321,7 +357,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): using. Disabling file checksums is a new feature yum 3.2.17-ish """ try: - return pkg.verify(fast=self.setup.get('quick', False)) + return pkg.verify(fast=Bcfg2.Options.setup.quick) except TypeError: # Older Yum API return pkg.verify() @@ -446,9 +482,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool): package_fail = False qtext_versions = [] virt_pkg = False - pkg_checks = (self.pkg_checks and + pkg_checks = (Bcfg2.Options.setup.yum_pkg_checks and entry.get('pkg_checks', 'true').lower() == 'true') - pkg_verify = (self.pkg_verify and + pkg_verify = (Bcfg2.Options.setup.yum_pkg_verify and entry.get('pkg_verify', 'true').lower() == 'true') yum_group = False @@ -541,7 +577,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): inst.get('verify_flags').lower().replace(' ', ',').split(',') else: - verify_flags = self.verify_flags + verify_flags = Bcfg2.Options.setup.yum_verify_flags if 'arch' in nevra: # If arch is specified use it to select the package @@ -625,7 +661,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra.get('release', 'any')) entry.set('current_version', "%s:%s-%s" % current_evr) entry.set('version', "%s:%s-%s" % wanted_evr) - if yum.compareEVR(current_evr, wanted_evr) == 1: + if rpmUtils.miscutils.compareEVR(current_evr, wanted_evr) == 1: entry.set("package_fail_action", "downgrade") else: entry.set("package_fail_action", "update") @@ -633,7 +669,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): qtext_versions.append("U(%s)" % str(all_pkg_objs[0])) continue - if self.setup.get('quick', False): + if Bcfg2.Options.setup.quick: # Passed -q on the command line continue if not (pkg_verify and @@ -707,7 +743,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): install_only = False if virt_pkg or \ - (install_only and not self.setup['kevlar']) or \ + (install_only and not Bcfg2.Options.setup.kevlar) or \ yum_group: # virtual capability supplied, we are probably dealing # with multiple packages of different names. This check @@ -830,8 +866,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.yumbase.closeRpmDB() self.RefreshPackages() - rpm_display = RPMDisplay(self.logger) - yum_display = YumDisplay(self.logger) + rpm_display = RPMDisplay() + yum_display = YumDisplay() # Run the Yum Transaction try: rescode, restring = self.yumbase.buildTransaction() @@ -880,7 +916,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): cleanup() - def Install(self, packages, states): # pylint: disable=R0912,R0914,R0915 + def Install(self, packages): # pylint: disable=R0912,R0914,R0915 """ Try and fix everything that Yum.VerifyPackages() found wrong for each Package Entry. This can result in individual RPMs being installed (for the first time), deleted, downgraded @@ -898,6 +934,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): entry is set to True. """ self.logger.debug('Running Yum.Install()') + states = dict() install_pkgs = [] gpg_keys = [] upgrade_pkgs = [] @@ -915,8 +952,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Remove extra instances. # Can not reverify because we don't have a package entry. if self.extra_instances is not None and len(self.extra_instances) > 0: - if (self.setup.get('remove') == 'all' or - self.setup.get('remove') == 'packages'): + if Bcfg2.Options.setup.remove in ['all', 'packages']: self.Remove(self.extra_instances) else: self.logger.info("The following extra package instances will " @@ -941,14 +977,17 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra2string(build_yname(pkg.get('name'), inst))) continue status = self.instance_status[inst] - if not status.get('installed', False) and self.do_install: + if not status.get('installed', False) and \ + Bcfg2.Options.setup.yum_install_missing: queue_pkg(pkg, inst, install_pkgs) - elif status.get('version_fail', False) and self.do_upgrade: + elif (status.get('version_fail', False) and + Bcfg2.Options.setup.yum_fix_version): if pkg.get("package_fail_action") == "downgrade": queue_pkg(pkg, inst, downgrade_pkgs) else: queue_pkg(pkg, inst, upgrade_pkgs) - elif status.get('verify_fail', False) and self.do_reinst: + elif (status.get('verify_fail', False) and + Bcfg2.Options.setup.yum_reinstall_broken): queue_pkg(pkg, inst, reinstall_pkgs) else: # Either there was no Install/Version/Verify @@ -1037,7 +1076,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self._runYumTransaction() - if not self.setup['kevlar']: + if not Bcfg2.Options.setup.kevlar: for pkg_entry in [p for p in packages if self.canVerify(p)]: self.logger.debug("Reverifying Failed Package %s" % pkg_entry.get('name')) @@ -1045,8 +1084,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.VerifyPackage(pkg_entry, self.modlists.get(pkg_entry, [])) - for entry in [ent for ent in packages if states[ent]]: - self.modified.append(entry) + self.modified.extend(ent for ent in packages if states[ent]) + return states def Remove(self, packages): """ diff --git a/src/lib/Bcfg2/Client/Tools/YUM24.py b/src/lib/Bcfg2/Client/Tools/YUM24.py deleted file mode 100644 index d78127ddd..000000000 --- a/src/lib/Bcfg2/Client/Tools/YUM24.py +++ /dev/null @@ -1,399 +0,0 @@ -"""This provides bcfg2 support for yum.""" - -import copy -import os.path -import sys -import yum -import Bcfg2.Client.XML -from Bcfg2.Client.Tools.RPM import RPM - - -def build_yname(pkgname, inst): - """Build yum appropriate package name.""" - ypname = pkgname - if inst.get('version') != 'any': - ypname += '-' - if inst.get('epoch', False): - ypname += "%s:" % inst.get('epoch') - if inst.get('version', False) and inst.get('version') != 'any': - ypname += "%s" % (inst.get('version')) - if inst.get('release', False) and inst.get('release') != 'any': - ypname += "-%s" % (inst.get('release')) - if inst.get('arch', False) and inst.get('arch') != 'any': - ypname += ".%s" % (inst.get('arch')) - return ypname - - -class YUM24(RPM): - """Support for Yum packages.""" - pkgtype = 'yum' - deprecated = True - __execs__ = ['/usr/bin/yum', '/var/lib/rpm'] - __handles__ = [('Package', 'yum'), - ('Package', 'rpm'), - ('Path', 'ignore')] - - __req__ = {'Package': ['name', 'version']} - __ireq__ = {'Package': ['name']} - #__ireq__ = {'Package': ['name', 'version']} - - __new_req__ = {'Package': ['name'], - 'Instance': ['version', 'release', 'arch']} - __new_ireq__ = {'Package': ['name'], \ - 'Instance': []} - #__new_ireq__ = {'Package': ['name', 'uri'], \ - # 'Instance': ['simplefile', 'version', 'release', 'arch']} - - __gpg_req__ = {'Package': ['name', 'version']} - __gpg_ireq__ = {'Package': ['name', 'version']} - - __new_gpg_req__ = {'Package': ['name'], - 'Instance': ['version', 'release']} - __new_gpg_ireq__ = {'Package': ['name'], - 'Instance': ['version', 'release']} - - def __init__(self, logger, setup, config): - RPM.__init__(self, logger, setup, config) - self.__important__ = self.__important__ + \ - [entry.get('name') for struct in config \ - for entry in struct \ - if entry.tag in ['Path', 'ConfigFile'] and \ - (entry.get('name').startswith('/etc/yum.d') \ - or entry.get('name').startswith('/etc/yum.repos.d')) \ - or entry.get('name') == '/etc/yum.conf'] - self.autodep = setup.get("yum24_autodep") - self.yum_avail = dict() - self.yum_installed = dict() - self.yb = yum.YumBase() - self.yb.doConfigSetup() - self.yb.doTsSetup() - self.yb.doRpmDBSetup() - yup = self.yb.doPackageLists(pkgnarrow='updates') - if hasattr(self.yb.rpmdb, 'pkglist'): - yinst = self.yb.rpmdb.pkglist - else: - yinst = self.yb.rpmdb.getPkgList() - for dest, source in [(self.yum_avail, yup.updates), - (self.yum_installed, yinst)]: - for pkg in source: - if dest is self.yum_avail: - pname = pkg.name - data = {pkg.arch: (pkg.epoch, pkg.version, pkg.release)} - else: - pname = pkg[0] - if pkg[1] is None: - a = 'noarch' - else: - a = pkg[1] - if pkg[2] is None: - e = '0' - else: - e = pkg[2] - data = {a: (e, pkg[3], pkg[4])} - if pname in dest: - dest[pname].update(data) - else: - dest[pname] = dict(data) - - def VerifyPackage(self, entry, modlist): - pinned_version = None - if entry.get('version', False) == 'auto': - # old style entry; synthesize Instances from current installed - if entry.get('name') not in self.yum_installed and \ - entry.get('name') not in self.yum_avail: - # new entry; fall back to default - entry.set('version', 'any') - else: - data = copy.copy(self.yum_installed[entry.get('name')]) - if entry.get('name') in self.yum_avail: - # installed but out of date - data.update(self.yum_avail[entry.get('name')]) - for (arch, (epoch, vers, rel)) in list(data.items()): - x = Bcfg2.Client.XML.SubElement(entry, "Instance", - name=entry.get('name'), - version=vers, arch=arch, - release=rel, epoch=epoch) - if 'verify_flags' in entry.attrib: - x.set('verify_flags', entry.get('verify_flags')) - if 'verify' in entry.attrib: - x.set('verify', entry.get('verify')) - - if entry.get('type', False) == 'yum': - # Check for virtual provides or packages. If we don't have - # this package use Yum to resolve it to a real package name - knownPkgs = list(self.yum_installed.keys()) + list(self.yum_avail.keys()) - if entry.get('name') not in knownPkgs: - # If the package name matches something installed - # or available the that's the correct package. - try: - pkgDict = dict([(i.name, i) for i in \ - self.yb.returnPackagesByDep(entry.get('name'))]) - except yum.Errors.YumBaseError: - e = sys.exc_info()[1] - self.logger.error('Yum Error Depsolving for %s: %s' % \ - (entry.get('name'), str(e))) - pkgDict = {} - - if len(pkgDict) > 1: - # What do we do with multiple packages? - s = "YUM24: returnPackagesByDep(%s) returned many packages" - self.logger.info(s % entry.get('name')) - s = "YUM24: matching packages: %s" - self.logger.info(s % str(list(pkgDict.keys()))) - pkgs = set(pkgDict.keys()) & set(self.yum_installed.keys()) - if len(pkgs) > 0: - # Virtual packages matches an installed real package - pkg = pkgDict[pkgs.pop()] - s = "YUM24: chosing: %s" % pkg.name - self.logger.info(s) - else: - # What's the right package? This will fail verify - # and Yum should Do The Right Thing on package install - pkg = None - elif len(pkgDict) == 1: - pkg = list(pkgDict.values())[0] - else: # len(pkgDict) == 0 - s = "YUM24: returnPackagesByDep(%s) returned no results" - self.logger.info(s % entry.get('name')) - pkg = None - - if pkg is not None: - s = "YUM24: remapping virtual package %s to %s" - self.logger.info(s % (entry.get('name'), pkg.name)) - entry.set('name', pkg.name) - - return RPM.VerifyPackage(self, entry, modlist) - - def Install(self, packages, states): - """ - Try and fix everything that YUM24.VerifyPackages() found wrong for - each Package Entry. This can result in individual RPMs being - installed (for the first time), deleted, downgraded - or upgraded. - - NOTE: YUM can not reinstall a package that it thinks is already - installed. - - packages is a list of Package Elements that has - states[<Package Element>] == False - - The following effects occur: - - states{} is conditionally updated for each package. - - self.installed{} is rebuilt, possibly multiple times. - - self.instance_status{} is conditionally updated for each instance - of a package. - - Each package will be added to self.modified[] if its states{} - entry is set to True. - - """ - self.logger.info('Running YUM24.Install()') - - install_pkgs = [] - gpg_keys = [] - upgrade_pkgs = [] - - # Remove extra instances. - # Can not reverify because we don't have a package entry. - if len(self.extra_instances) > 0: - if (self.setup.get('remove') == 'all' or \ - self.setup.get('remove') == 'packages'): - self.Remove(self.extra_instances) - else: - self.logger.info("The following extra package instances will be removed by the '-r' option:") - for pkg in self.extra_instances: - for inst in pkg: - self.logger.info(" %s %s" % \ - ((pkg.get('name'), self.str_evra(inst)))) - - # Figure out which instances of the packages actually need something - # doing to them and place in the appropriate work 'queue'. - for pkg in packages: - insts = [pinst for pinst in pkg \ - if pinst.tag in ['Instance', 'Package']] - if insts: - for inst in insts: - if self.FixInstance(inst, self.instance_status[inst]): - if self.instance_status[inst].get('installed', False) \ - == False: - if pkg.get('name') == 'gpg-pubkey': - gpg_keys.append(inst) - else: - install_pkgs.append(inst) - elif self.instance_status[inst].get('version_fail', \ - False) == True: - upgrade_pkgs.append(inst) - else: - install_pkgs.append(pkg) - - # Install GPG keys. - # Alternatively specify the required keys using 'gpgkey' in the - # repository definition in yum.conf. YUM will install the keys - # automatically. - if len(gpg_keys) > 0: - for inst in gpg_keys: - self.logger.info("Installing GPG keys.") - if inst.get('simplefile') is None: - self.logger.error("GPG key has no simplefile attribute") - continue - key_arg = os.path.join(self.instance_status[inst].get('pkg').get('uri'), \ - inst.get('simplefile')) - if self.cmd.run("rpm --import %s" % key_arg).success: - self.logger.debug("Unable to install %s-%s" % \ - (self.instance_status[inst].get('pkg').get('name'), \ - self.str_evra(inst))) - else: - self.logger.debug("Installed %s-%s-%s" % \ - (self.instance_status[inst].get('pkg').get('name'), \ - inst.get('version'), inst.get('release'))) - self.RefreshPackages() - self.gpg_keyids = self.getinstalledgpg() - pkg = self.instance_status[gpg_keys[0]].get('pkg') - states[pkg] = self.VerifyPackage(pkg, []) - - # Install packages. - if len(install_pkgs) > 0: - self.logger.info("Attempting to install packages") - - if self.autodep: - pkgtool = "/usr/bin/yum -d0 -y install %s" - else: - pkgtool = "/usr/bin/yum -d0 install %s" - - install_args = [] - for inst in install_pkgs: - pkg_arg = self.instance_status[inst].get('pkg').get('name') - install_args.append(build_yname(pkg_arg, inst)) - - if self.cmd.run(pkgtool % " ".join(install_args)).success: - # The yum command succeeded. All packages installed. - self.logger.info("Single Pass for Install Succeeded") - self.RefreshPackages() - else: - # The yum command failed. No packages installed. - # Try installing instances individually. - self.logger.error("Single Pass Install of Packages Failed") - installed_instances = [] - for inst in install_pkgs: - pkg_arg = build_yname(self.instance_status[inst].get('pkg').get('name'), inst) - - if self.cmd.run(pkgtool % pkg_arg).success: - installed_instances.append(inst) - else: - self.logger.debug("%s %s would not install." % - (self.instance_status[inst].get('pkg').get('name'), - self.str_evra(inst))) - self.RefreshPackages() - - # Fix upgradeable packages. - if len(upgrade_pkgs) > 0: - self.logger.info("Attempting to upgrade packages") - - if self.autodep: - pkgtool = "/usr/bin/yum -d0 -y update %s" - else: - pkgtool = "/usr/bin/yum -d0 update %s" - - upgrade_args = [] - for inst in upgrade_pkgs: - pkg_arg = build_yname(self.instance_status[inst].get('pkg').get('name'), inst) - upgrade_args.append(pkg_arg) - - if self.cmd.run(pkgtool % " ".join(upgrade_args)).success: - # The yum command succeeded. All packages installed. - self.logger.info("Single Pass for Install Succeeded") - self.RefreshPackages() - else: - # The yum command failed. No packages installed. - # Try installing instances individually. - self.logger.error("Single Pass Install of Packages Failed") - installed_instances = [] - for inst in upgrade_pkgs: - pkg_arg = build_yname(self.instance_status[inst].get('pkg').get('name'), inst) - if self.cmd.run(pkgtool % pkg_arg).success: - installed_instances.append(inst) - else: - self.logger.debug("%s %s would not install." % \ - (self.instance_status[inst].get('pkg').get('name'), \ - self.str_evra(inst))) - - self.RefreshPackages() - - if not self.setup['kevlar']: - for pkg_entry in [p for p in packages if self.canVerify(p)]: - self.logger.debug("Reverifying Failed Package %s" % (pkg_entry.get('name'))) - states[pkg_entry] = self.VerifyPackage(pkg_entry, \ - self.modlists.get(pkg_entry, [])) - - for entry in [ent for ent in packages if states[ent]]: - self.modified.append(entry) - - def Remove(self, packages): - """ - Remove specified entries. - - packages is a list of Package Entries with Instances generated - by FindExtra(). - """ - self.logger.debug('Running YUM24.Remove()') - - if self.autodep: - pkgtool = "/usr/bin/yum -d0 -y erase %s" - else: - pkgtool = "/usr/bin/yum -d0 erase %s" - - erase_args = [] - for pkg in packages: - for inst in pkg: - if pkg.get('name') != 'gpg-pubkey': - pkg_arg = pkg.get('name') + '-' - if inst.get('epoch', False): - pkg_arg = pkg_arg + inst.get('epoch') + ':' - pkg_arg = pkg_arg + inst.get('version') + '-' + inst.get('release') - if inst.get('arch', False): - pkg_arg = pkg_arg + '.' + inst.get('arch') - erase_args.append(pkg_arg) - else: - pkgspec = {'name': pkg.get('name'), - 'version': inst.get('version'), - 'release': inst.get('release')} - self.logger.info("WARNING: gpg-pubkey package not in configuration %s %s"\ - % (pkgspec.get('name'), self.str_evra(pkgspec))) - self.logger.info(" This package will be deleted in a future version of the YUM24 driver.") - - rv = self.cmd.run(pkgtool % " ".join(erase_args)) - if rv.success: - self.modified += packages - for pkg in erase_args: - self.logger.info("Deleted %s" % (pkg)) - else: - self.logger.info("Bulk erase failed with errors:") - self.logger.debug("Erase results: %s" % rv.error) - self.logger.info("Attempting individual erase for each package.") - for pkg in packages: - pkg_modified = False - for inst in pkg: - if pkg.get('name') != 'gpg-pubkey': - pkg_arg = pkg.get('name') + '-' - if 'epoch' in inst.attrib: - pkg_arg = pkg_arg + inst.get('epoch') + ':' - pkg_arg = pkg_arg + inst.get('version') + '-' + inst.get('release') - if 'arch' in inst.attrib: - pkg_arg = pkg_arg + '.' + inst.get('arch') - else: - self.logger.info("WARNING: gpg-pubkey package not in configuration %s %s"\ - % (pkg.get('name'), self.str_evra(pkg))) - self.logger.info(" This package will be deleted in a future version of the YUM24 driver.") - continue - - rv = self.cmd.run(self.pkgtool % pkg_arg) - if rv.success: - pkg_modified = True - self.logger.info("Deleted %s" % pkg_arg) - else: - self.logger.error("Unable to delete %s" % pkg_arg) - self.logger.debug("Failure: %s" % rv.error) - if pkg_modified == True: - self.modified.append(pkg) - - self.RefreshPackages() - self.extra = self.FindExtra() diff --git a/src/lib/Bcfg2/Client/Tools/YUMng.py b/src/lib/Bcfg2/Client/Tools/YUMng.py deleted file mode 100644 index 22fbba537..000000000 --- a/src/lib/Bcfg2/Client/Tools/YUMng.py +++ /dev/null @@ -1,9 +0,0 @@ -""" YUM driver called 'YUMng' for backwards compat """ - -from Bcfg2.Client.Tools.YUM import YUM - - -class YUMng(YUM): - """ YUM driver called 'YUMng' for backwards compat """ - deprecated = True - conflicts = ['YUM24', 'RPM', 'RPMng'] diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index 0bec71e20..ae7fa3aed 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -1,25 +1,13 @@ """This contains all Bcfg2 Tool modules""" import os -import stat import sys - +import stat +import logging +import Bcfg2.Options import Bcfg2.Client import Bcfg2.Client.XML -from Bcfg2.Client.Frame import matches_white_list, passes_black_list from Bcfg2.Utils import Executor, ClassName -from Bcfg2.Compat import walk_packages # pylint: disable=W0622 - -__all__ = [m[1] for m in walk_packages(path=__path__)] - -# pylint: disable=C0103 -#: All available tools -drivers = [item for item in __all__ if item not in ['rpmtools']] - -#: The default set of tools that will be used if "drivers" is not set -#: in bcfg2.conf -default = drivers[:] -# pylint: enable=C0103 class ToolInstantiationError(Exception): @@ -37,6 +25,12 @@ class Tool(object): .. autoattribute:: Bcfg2.Client.Tools.Tool.__important__ """ + options = [ + Bcfg2.Options.Option( + cf=('client', 'command_timeout'), + help="Timeout when running external commands other than probes", + type=Bcfg2.Options.Types.timeout)] + #: The name of the tool. By default this uses #: :class:`Bcfg2.Client.Tools.ClassName` to ensure that it is the #: same as the name of the class. @@ -80,30 +74,22 @@ class Tool(object): #: runtime with a warning. conflicts = [] - def __init__(self, logger, setup, config): + def __init__(self, config): """ - :param logger: Logger that will be used for logging by this tool - :type logger: logging.Logger - :param setup: The option set Bcfg2 was invoked with - :type setup: Bcfg2.Options.OptionParser :param config: The XML configuration for this client :type config: lxml.etree._Element :raises: :exc:`Bcfg2.Client.Tools.ToolInstantiationError` """ - #: A :class:`Bcfg2.Options.OptionParser` object describing the - #: option set Bcfg2 was invoked with - self.setup = setup - #: A :class:`logging.Logger` object that will be used by this #: tool for logging - self.logger = logger + self.logger = logging.getLogger(self.name) #: The XML configuration for this client self.config = config #: An :class:`Bcfg2.Utils.Executor` object for #: running external commands. - self.cmd = Executor(timeout=self.setup['command_timeout']) + self.cmd = Executor(timeout=Bcfg2.Options.setup.command_timeout) #: A list of entries that have been modified by this tool self.modified = [] @@ -124,7 +110,7 @@ class Tool(object): for struct in self.config: for entry in struct: if (entry.tag == 'Path' and - entry.get('important', 'false').lower() == 'true'): + entry.get('important', 'false').lower() == 'true'): self.__important__.append(entry.get('name')) self.handled = self.getSupportedEntries() @@ -146,39 +132,41 @@ class Tool(object): def _install_allowed(self, entry): """ Return true if the given entry is allowed to be installed by the whitelist or blacklist """ - if self.setup['decision'] == 'whitelist' and \ - not matches_white_list(entry, self.setup['decision_list']): - self.logger.info("In whitelist mode: suppressing %s: %s" % - (entry.tag, entry.get('name'))) + if (Bcfg2.Options.setup.decision == 'whitelist' and + not Bcfg2.Client.matches_white_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In whitelist mode: suppressing Action: %s" % + entry.get('name')) return False - if self.setup['decision'] == 'blacklist' and \ - not passes_black_list(entry, self.setup['decision_list']): - self.logger.info("In blacklist mode: suppressing %s: %s" % - (entry.tag, entry.get('name'))) + if (Bcfg2.Options.setup.decision == 'blacklist' and + not Bcfg2.Client.passes_black_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In blacklist mode: suppressing Action: %s" % + entry.get('name')) return False return True - def BundleUpdated(self, bundle, states): # pylint: disable=W0613 + def BundleUpdated(self, bundle): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. :param bundle: The bundle that has been updated :type bundle: lxml.etree._Element - :param states: The :attr:`Bcfg2.Client.Frame.Frame.states` dict - :type states: dict - :returns: None """ - return + :returns: dict - A dict of the state of entries suitable for + updating :attr:`Bcfg2.Client.Client.states` + """ + return dict() - def BundleNotUpdated(self, bundle, states): # pylint: disable=W0613 + def BundleNotUpdated(self, bundle): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. :param bundle: The bundle that has been updated :type bundle: lxml.etree._Element - :param states: The :attr:`Bcfg2.Client.Frame.Frame.states` dict - :type states: dict - :returns: None """ - return + :returns: dict - A dict of the state of entries suitable for + updating :attr:`Bcfg2.Client.Client.states` + """ + return dict() - def Inventory(self, states, structures=None): + def Inventory(self, structures=None): """ Take an inventory of the system as it exists. This involves two steps: @@ -193,18 +181,19 @@ class Tool(object): is the entry tag. E.g., a Path entry would be verified by calling :func:`VerifyPath`. - :param states: The :attr:`Bcfg2.Client.Frame.Frame.states` dict - :type states: dict :param structures: The list of structures (i.e., bundles) to get entries from. If this is not given, all children of :attr:`Bcfg2.Client.Tools.Tool.config` will be used. :type structures: list of lxml.etree._Element - :returns: None """ + :returns: dict - A dict of the state of entries suitable for + updating :attr:`Bcfg2.Client.Client.states` + """ if not structures: structures = self.config.getchildren() mods = self.buildModlist() + states = dict() for struct in structures: for entry in struct.getchildren(): if self.canVerify(entry): @@ -222,8 +211,9 @@ class Tool(object): self.primarykey(entry)), exc_info=1) self.extra = self.FindExtra() + return states - def Install(self, entries, states): + def Install(self, entries): """ Install entries. 'Install' in this sense means either initially install, or update as necessary to match the specification. @@ -235,9 +225,10 @@ class Tool(object): :param entries: The entries to install :type entries: list of lxml.etree._Element - :param states: The :attr:`Bcfg2.Client.Frame.Frame.states` dict - :type states: dict - :returns: None """ + :returns: dict - A dict of the state of entries suitable for + updating :attr:`Bcfg2.Client.Client.states` + """ + states = dict() for entry in entries: try: func = getattr(self, "Install%s" % entry.tag) @@ -253,6 +244,7 @@ class Tool(object): self.logger.error("%s: Unexpected failure installing %s" % (self.name, self.primarykey(entry)), exc_info=1) + return states def Remove(self, entries): """ Remove specified extra entries. @@ -413,8 +405,8 @@ class PkgTool(Tool): #: The ``type`` attribute of Packages handled by this tool. pkgtype = 'echo' - def __init__(self, logger, setup, config): - Tool.__init__(self, logger, setup, config) + def __init__(self, config): + Tool.__init__(self, config) #: A dict of installed packages; the keys should be package #: names and the values should be simple strings giving the @@ -451,32 +443,27 @@ class PkgTool(Tool): for pkg in packages) return self.pkgtool[0] % pkgargs - def Install(self, packages, states): + def Install(self, packages): """ Run a one-pass install where all required packages are installed with a single command, followed by single package installs in case of failure. :param entries: The entries to install :type entries: list of lxml.etree._Element - :param states: The :attr:`Bcfg2.Client.Frame.Frame.states` dict - :type states: dict - :returns: None """ + :returns: dict - A dict of the state of entries suitable for + updating :attr:`Bcfg2.Client.Client.states` + """ self.logger.info("Trying single pass package install for pkgtype %s" % self.pkgtype) - pkgcmd = self._get_package_command(packages) - self.logger.debug("Running command: %s" % pkgcmd) - if self.cmd.run(pkgcmd): + states = dict() + if self.cmd.run(self._get_package_command(packages)): self.logger.info("Single Pass Succeded") # set all package states to true and flush workqueues - pkgnames = [pkg.get('name') for pkg in packages] - for entry in list(states.keys()): - if (entry.tag == 'Package' - and entry.get('type') == self.pkgtype - and entry.get('name') in pkgnames): - self.logger.debug('Setting state to true for pkg %s' % - entry.get('name')) - states[entry] = True + for entry in packages: + self.logger.debug('Setting state to true for %s' % + self.primarykey(entry)) + states[entry] = True self.RefreshPackages() else: self.logger.error("Single Pass Failed") @@ -494,10 +481,13 @@ class PkgTool(Tool): if self.cmd.run(self._get_package_command([pkg])): states[pkg] = True else: + states[pkg] = False self.logger.error("Failed to install package %s" % pkg.get('name')) self.RefreshPackages() - self.modified.extend(entry for entry in packages if states[entry]) + self.modified.extend(entry for entry in packages + if entry in states and states[entry]) + return states def RefreshPackages(self): """ Refresh the internal representation of the package @@ -519,8 +509,14 @@ class PkgTool(Tool): class SvcTool(Tool): """ Base class for tools that handle Service entries """ - def __init__(self, logger, setup, config): - Tool.__init__(self, logger, setup, config) + options = Tool.options + [ + Bcfg2.Options.Option( + '-s', '--service-mode', default='default', + choices=['default', 'disabled', 'build'], + help='Set client service mode')] + + def __init__(self, config): + Tool.__init__(self, config) #: List of services that have been restarted self.restarted = [] __init__.__doc__ = Tool.__init__.__doc__ @@ -597,14 +593,14 @@ class SvcTool(Tool): return bool(self.cmd.run(self.get_svc_command(service, 'status'))) def Remove(self, services): - if self.setup['servicemode'] != 'disabled': + if Bcfg2.Options.setup.service_mode != 'disabled': for entry in services: entry.set("status", "off") self.InstallService(entry) Remove.__doc__ = Tool.Remove.__doc__ - def BundleUpdated(self, bundle, states): - if self.setup['servicemode'] == 'disabled': + def BundleUpdated(self, bundle): + if Bcfg2.Options.setup.service_mode == 'disabled': return for entry in bundle: @@ -615,15 +611,16 @@ class SvcTool(Tool): estatus = entry.get('status') restart = entry.get("restart", "true").lower() if (restart == "false" or estatus == 'ignore' or - (restart == "interactive" and not self.setup['interactive'])): + (restart == "interactive" and + not Bcfg2.Options.setup.interactive)): continue success = False if estatus == 'on': - if self.setup['servicemode'] == 'build': + if Bcfg2.Options.setup.service_mode == 'build': success = self.stop_service(entry) elif entry.get('name') not in self.restarted: - if self.setup['interactive']: + if Bcfg2.Options.setup.interactive: if not Bcfg2.Client.prompt('Restart service %s? (y/N) ' % entry.get('name')): continue @@ -635,9 +632,10 @@ class SvcTool(Tool): if not success: self.logger.error("Failed to manipulate service %s" % (entry.get('name'))) + return dict() BundleUpdated.__doc__ = Tool.BundleUpdated.__doc__ - def Install(self, entries, states): + def Install(self, entries): install_entries = [] for entry in entries: if entry.get('install', 'true').lower() == 'false': @@ -645,7 +643,7 @@ class SvcTool(Tool): (entry.tag, entry.get('name'))) else: install_entries.append(entry) - return Tool.Install(self, install_entries, states) + return Tool.Install(self, install_entries) Install.__doc__ = Tool.Install.__doc__ def InstallService(self, entry): diff --git a/src/lib/Bcfg2/Client/Tools/launchd.py b/src/lib/Bcfg2/Client/Tools/launchd.py index b0661b26b..a4aeab6c7 100644 --- a/src/lib/Bcfg2/Client/Tools/launchd.py +++ b/src/lib/Bcfg2/Client/Tools/launchd.py @@ -12,8 +12,8 @@ class launchd(Bcfg2.Client.Tools.Tool): # pylint: disable=C0103 __execs__ = ['/bin/launchctl', '/usr/bin/defaults'] __req__ = {'Service': ['name', 'status']} - def __init__(self, logger, setup, config): - Bcfg2.Client.Tools.Tool.__init__(self, logger, setup, config) + def __init__(self, config): + Bcfg2.Client.Tools.Tool.__init__(self, config) # Locate plist file that provides given reverse-fqdn name: # @@ -117,9 +117,11 @@ class launchd(Bcfg2.Client.Tools.Tool): # pylint: disable=C0103 status='on') for name in allsrv] - def BundleUpdated(self, bundle, states): + def BundleUpdated(self, bundle): """Reload launchd plist.""" - for entry in [entry for entry in bundle if self.handlesEntry(entry)]: + for entry in bundle: + if not self.handlesEntry(entry): + continue if not self.canInstall(entry): self.logger.error("Insufficient information to restart " "service %s" % entry.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/rpmtools.py b/src/lib/Bcfg2/Client/Tools/rpmtools.py deleted file mode 100755 index 32a04262d..000000000 --- a/src/lib/Bcfg2/Client/Tools/rpmtools.py +++ /dev/null @@ -1,1091 +0,0 @@ -#!/usr/bin/env python -""" - Module that uses rpm-python to implement the following rpm - functionality for the bcfg2 RPM and YUM client drivers: - - rpm -qa - rpm --verify - rpm --erase - - The code closely follows the rpm C code. - - The code was written to be used in the bcfg2 RPM/YUM drivers. - - Some command line options have been provided to assist with - testing and development, but the output isn't pretty and looks - nothing like rpm output. - - Run 'rpmtools' -h for the options. - -""" - -import grp -import optparse -import os -import pwd -import rpm -import stat -import sys -if sys.version_info >= (2, 5): - import hashlib - py24compat = False -else: - # FIXME: Remove when client python dep is 2.5 or greater - py24compat = True - import md5 - -# Determine what prelink tools we have available. -# The isprelink module is a python extension that examines the ELF headers -# to see if the file has been prelinked. If it is not present a lot of files -# are unnecessarily run through the prelink command. -try: - from isprelink import * - isprelink_imported = True -except ImportError: - isprelink_imported = False - -# If the prelink command is installed on the system then we need to do -# prelink -y on files. -if os.access('/usr/sbin/prelink', os.X_OK): - prelink_exists = True -else: - prelink_exists = False - -# If we don't have isprelink then we will use the prelink configuration file to -# filter what we have to put through prelink -y. -import re -blacklist = [] -whitelist = [] -try: - f = open('/etc/prelink.conf', mode='r') - for line in f: - if line.startswith('#'): - continue - option, pattern = line.split() - if pattern.startswith('*.'): - pattern = pattern.replace('*.', '\.') - pattern += '$' - elif pattern.startswith('/'): - pattern = '^' + pattern - if option == '-b': - blacklist.append(pattern) - elif option == '-l': - whitelist.append(pattern) - f.close() -except IOError: - pass - -blacklist_re = re.compile('|'.join(blacklist)) -whitelist_re = re.compile('|'.join(whitelist)) - -# Flags that are not defined in rpm-python. -# They are defined in lib/rpmcli.h -# Bit(s) for verifyFile() attributes. -# -RPMVERIFY_NONE = 0 # /*!< */ -RPMVERIFY_MD5 = 1 # 1 << 0 # /*!< from %verify(md5) */ -RPMVERIFY_FILESIZE = 2 # 1 << 1 # /*!< from %verify(size) */ -RPMVERIFY_LINKTO = 4 # 1 << 2 # /*!< from %verify(link) */ -RPMVERIFY_USER = 8 # 1 << 3 # /*!< from %verify(user) */ -RPMVERIFY_GROUP = 16 # 1 << 4 # /*!< from %verify(group) */ -RPMVERIFY_MTIME = 32 # 1 << 5 # /*!< from %verify(mtime) */ -RPMVERIFY_MODE = 64 # 1 << 6 # /*!< from %verify(mode) */ -RPMVERIFY_RDEV = 128 # 1 << 7 # /*!< from %verify(rdev) */ -RPMVERIFY_CONTEXTS = 32768 # (1 << 15) # /*!< from --nocontexts */ -RPMVERIFY_READLINKFAIL = 268435456 # (1 << 28) # /*!< readlink failed */ -RPMVERIFY_READFAIL = 536870912 # (1 << 29) # /*!< file read failed */ -RPMVERIFY_LSTATFAIL = 1073741824 # (1 << 30) # /*!< lstat failed */ -RPMVERIFY_LGETFILECONFAIL = 2147483648 # (1 << 31) # /*!< lgetfilecon failed */ - -RPMVERIFY_FAILURES = \ - (RPMVERIFY_LSTATFAIL|RPMVERIFY_READFAIL|RPMVERIFY_READLINKFAIL| \ - RPMVERIFY_LGETFILECONFAIL) - -# Bit(s) to control rpm_verify() operation. -# -VERIFY_DEFAULT = 0, # /*!< */ -VERIFY_MD5 = 1 << 0 # /*!< from --nomd5 */ -VERIFY_SIZE = 1 << 1 # /*!< from --nosize */ -VERIFY_LINKTO = 1 << 2 # /*!< from --nolinkto */ -VERIFY_USER = 1 << 3 # /*!< from --nouser */ -VERIFY_GROUP = 1 << 4 # /*!< from --nogroup */ -VERIFY_MTIME = 1 << 5 # /*!< from --nomtime */ -VERIFY_MODE = 1 << 6 # /*!< from --nomode */ -VERIFY_RDEV = 1 << 7 # /*!< from --nodev */ -# /* bits 8-14 unused, reserved for rpmVerifyAttrs */ -VERIFY_CONTEXTS = 1 << 15 # /*!< verify: from --nocontexts */ -VERIFY_FILES = 1 << 16 # /*!< verify: from --nofiles */ -VERIFY_DEPS = 1 << 17 # /*!< verify: from --nodeps */ -VERIFY_SCRIPT = 1 << 18 # /*!< verify: from --noscripts */ -VERIFY_DIGEST = 1 << 19 # /*!< verify: from --nodigest */ -VERIFY_SIGNATURE = 1 << 20 # /*!< verify: from --nosignature */ -VERIFY_PATCHES = 1 << 21 # /*!< verify: from --nopatches */ -VERIFY_HDRCHK = 1 << 22 # /*!< verify: from --nohdrchk */ -VERIFY_FOR_LIST = 1 << 23 # /*!< query: from --list */ -VERIFY_FOR_STATE = 1 << 24 # /*!< query: from --state */ -VERIFY_FOR_DOCS = 1 << 25 # /*!< query: from --docfiles */ -VERIFY_FOR_CONFIG = 1 << 26 # /*!< query: from --configfiles */ -VERIFY_FOR_DUMPFILES = 1 << 27 # /*!< query: from --dump */ -# /* bits 28-31 used in rpmVerifyAttrs */ - -# Comes from C cource. lib/rpmcli.h -VERIFY_ATTRS = \ - (VERIFY_MD5 | VERIFY_SIZE | VERIFY_LINKTO | VERIFY_USER | VERIFY_GROUP | \ - VERIFY_MTIME | VERIFY_MODE | VERIFY_RDEV | VERIFY_CONTEXTS) - -VERIFY_ALL = \ - (VERIFY_ATTRS | VERIFY_FILES | VERIFY_DEPS | VERIFY_SCRIPT | VERIFY_DIGEST |\ - VERIFY_SIGNATURE | VERIFY_HDRCHK) - - -# Some masks for what checks to NOT do on these file types. -# The C code actiually resets these up for every file. -DIR_FLAGS = ~(RPMVERIFY_MD5 | RPMVERIFY_FILESIZE | RPMVERIFY_MTIME | \ - RPMVERIFY_LINKTO) - -# These file types all have the same mask, but hopefully this will make the -# code more readable. -FIFO_FLAGS = CHR_FLAGS = BLK_FLAGS = GHOST_FLAGS = DIR_FLAGS - -LINK_FLAGS = ~(RPMVERIFY_MD5 | RPMVERIFY_FILESIZE | RPMVERIFY_MTIME | \ - RPMVERIFY_MODE | RPMVERIFY_USER | RPMVERIFY_GROUP) - -REG_FLAGS = ~(RPMVERIFY_LINKTO) - - -def s_isdev(mode): - """ - Check to see if a file is a device. - - """ - return stat.S_ISBLK(mode) | stat.S_ISCHR(mode) - -def rpmpackagelist(rts): - """ - Equivalent of rpm -qa. Intended for RefreshPackages() in the RPM Driver. - Requires rpmtransactionset() to be run first to get a ts. - Returns a list of pkgspec dicts. - - e.g. [ {'name':'foo', 'epoch':'20', 'version':'1.2', 'release':'5', 'arch':'x86_64' }, - {'name':'bar', 'epoch':'10', 'version':'5.2', 'release':'2', 'arch':'x86_64' } ] - - """ - return [{'name':header[rpm.RPMTAG_NAME], - 'epoch':header[rpm.RPMTAG_EPOCH], - 'version':header[rpm.RPMTAG_VERSION], - 'release':header[rpm.RPMTAG_RELEASE], - 'arch':header[rpm.RPMTAG_ARCH], - 'gpgkeyid':header.sprintf("%|SIGGPG?{%{SIGGPG:pgpsig}}:{None}|").split()[-1]} - for header in rts.dbMatch()] - -def getindexbykeyword(index_ts, **kwargs): - """ - Return list of indexs from the rpmdb matching keywords - ex: getHeadersByKeyword(name='foo', version='1', release='1') - - Can be passed any structure that can be indexed by the pkgspec - keyswords as other keys are filtered out. - - """ - lst = [] - name = kwargs.get('name') - if name: - index_mi = index_ts.dbMatch(rpm.RPMTAG_NAME, name) - else: - index_mi = index_ts.dbMatch() - - if 'epoch' in kwargs: - if kwargs['epoch'] != None and kwargs['epoch'] != 'None': - kwargs['epoch'] = int(kwargs['epoch']) - else: - del(kwargs['epoch']) - - keywords = [key for key in list(kwargs.keys()) \ - if key in ('name', 'epoch', 'version', 'release', 'arch')] - keywords_len = len(keywords) - for hdr in index_mi: - match = 0 - for keyword in keywords: - if hdr[keyword] == kwargs[keyword]: - match += 1 - if match == keywords_len: - lst.append(index_mi.instance()) - del index_mi - return lst - -def getheadersbykeyword(header_ts, **kwargs): - """ - Borrowed parts of this from from Yum. Need to fix it though. - Epoch is not handled right. - - Return list of headers from the rpmdb matching keywords - ex: getHeadersByKeyword(name='foo', version='1', release='1') - - Can be passed any structure that can be indexed by the pkgspec - keyswords as other keys are filtered out. - - """ - lst = [] - name = kwargs.get('name') - if name: - header_mi = header_ts.dbMatch(rpm.RPMTAG_NAME, name) - else: - header_mi = header_ts.dbMatch() - - if 'epoch' in kwargs: - if kwargs['epoch'] != None and kwargs['epoch'] != 'None': - kwargs['epoch'] = int(kwargs['epoch']) - else: - del(kwargs['epoch']) - - keywords = [key for key in list(kwargs.keys()) \ - if key in ('name', 'epoch', 'version', 'release', 'arch')] - keywords_len = len(keywords) - for hdr in header_mi: - match = 0 - for keyword in keywords: - if hdr[keyword] == kwargs[keyword]: - match += 1 - if match == keywords_len: - lst.append(hdr) - del header_mi - return lst - -def prelink_md5_check(filename): - """ - Checks if a file is prelinked. If it is run it through prelink -y - to get the unprelinked md5 and file size. - - Return 0 if the file was not prelinked, otherwise return the file size. - Always return the md5. - - """ - prelink = False - try: - plf = open(filename, "rb") - except IOError: - return False, 0 - - if prelink_exists: - if isprelink_imported: - plfd = plf.fileno() - if isprelink(plfd): - plf.close() - cmd = '/usr/sbin/prelink -y %s 2> /dev/null' \ - % (re.escape(filename)) - plf = os.popen(cmd, 'rb') - prelink = True - elif whitelist_re.search(filename) and not blacklist_re.search(filename): - plf.close() - cmd = '/usr/sbin/prelink -y %s 2> /dev/null' \ - % (re.escape(filename)) - plf = os.popen(cmd, 'rb') - prelink = True - - fsize = 0 - if py24compat: - chksum = md5.new() - else: - chksum = hashlib.md5() - while 1: - data = plf.read() - if not data: - break - fsize += len(data) - chksum.update(data) - plf.close() - file_md5 = chksum.hexdigest() - if prelink: - return file_md5, fsize - else: - return file_md5, 0 - -def prelink_size_check(filename): - """ - This check is only done if the prelink_md5_check() is not done first. - - Checks if a file is prelinked. If it is run it through prelink -y - to get the unprelinked file size. - - Return 0 if the file was not prelinked, otherwise return the file size. - - """ - fsize = 0 - try: - plf = open(filename, "rb") - except IOError: - return False - - if prelink_exists: - if isprelink_imported: - plfd = plf.fileno() - if isprelink(plfd): - plf.close() - cmd = '/usr/sbin/prelink -y %s 2> /dev/null' \ - % (re.escape(filename)) - plf = os.popen(cmd, 'rb') - - while 1: - data = plf.read() - if not data: - break - fsize += len(data) - - elif whitelist_re.search(filename) and not blacklist_re.search(filename): - plf.close() - cmd = '/usr/sbin/prelink -y %s 2> /dev/null' \ - % (re.escape(filename)) - plf = os.popen(cmd, 'rb') - - while 1: - data = plf.read() - if not data: - break - fsize += len(data) - - plf.close() - - return fsize - -def debug_verify_flags(vflags): - """ - Decodes the verify flags bits. - """ - if vflags & RPMVERIFY_MD5: - print('RPMVERIFY_MD5') - if vflags & RPMVERIFY_FILESIZE: - print('RPMVERIFY_FILESIZE') - if vflags & RPMVERIFY_LINKTO: - print('RPMVERIFY_LINKTO') - if vflags & RPMVERIFY_USER: - print('RPMVERIFY_USER') - if vflags & RPMVERIFY_GROUP: - print('RPMVERIFY_GROUP') - if vflags & RPMVERIFY_MTIME: - print('RPMVERIFY_MTIME') - if vflags & RPMVERIFY_MODE: - print('RPMVERIFY_MODE') - if vflags & RPMVERIFY_RDEV: - print('RPMVERIFY_RDEV') - if vflags & RPMVERIFY_CONTEXTS: - print('RPMVERIFY_CONTEXTS') - if vflags & RPMVERIFY_READLINKFAIL: - print('RPMVERIFY_READLINKFAIL') - if vflags & RPMVERIFY_READFAIL: - print('RPMVERIFY_READFAIL') - if vflags & RPMVERIFY_LSTATFAIL: - print('RPMVERIFY_LSTATFAIL') - if vflags & RPMVERIFY_LGETFILECONFAIL: - print('RPMVERIFY_LGETFILECONFAIL') - -def debug_file_flags(fflags): - """ - Decodes the file flags bits. - """ - if fflags & rpm.RPMFILE_CONFIG: - print('rpm.RPMFILE_CONFIG') - - if fflags & rpm.RPMFILE_DOC: - print('rpm.RPMFILE_DOC') - - if fflags & rpm.RPMFILE_ICON: - print('rpm.RPMFILE_ICON') - - if fflags & rpm.RPMFILE_MISSINGOK: - print('rpm.RPMFILE_MISSINGOK') - - if fflags & rpm.RPMFILE_NOREPLACE: - print('rpm.RPMFILE_NOREPLACE') - - if fflags & rpm.RPMFILE_GHOST: - print('rpm.RPMFILE_GHOST') - - if fflags & rpm.RPMFILE_LICENSE: - print('rpm.RPMFILE_LICENSE') - - if fflags & rpm.RPMFILE_README: - print('rpm.RPMFILE_README') - - if fflags & rpm.RPMFILE_EXCLUDE: - print('rpm.RPMFILE_EXLUDE') - - if fflags & rpm.RPMFILE_UNPATCHED: - print('rpm.RPMFILE_UNPATCHED') - - if fflags & rpm.RPMFILE_PUBKEY: - print('rpm.RPMFILE_PUBKEY') - -def rpm_verify_file(fileinfo, rpmlinktos, omitmask): - """ - Verify all the files in a package. - - Returns a list of error flags, the file type and file name. The list - entries are strings that are the same as the labels for the bitwise - flags used in the C code. - - """ - (fname, fsize, fmode, fmtime, fflags, frdev, finode, fnlink, fstate, \ - vflags, fuser, fgroup, fmd5) = fileinfo - - # 1. rpmtsRootDir stuff. What does it do and where to I get it from? - - file_results = [] - flags = vflags - - # Check to see if the file was installed - if not pretend all is ok. - # This is what the rpm C code does! - if fstate != rpm.RPMFILE_STATE_NORMAL: - return file_results - - # Get the installed files stats - try: - lstat = os.lstat(fname) - except OSError: - if not (fflags & (rpm.RPMFILE_MISSINGOK|rpm.RPMFILE_GHOST)): - file_results.append('RPMVERIFY_LSTATFAIL') - #file_results.append(fname) - return file_results - - # 5. Contexts? SELinux stuff? - - # Setup what checks to do. This is straight out of the C code. - if stat.S_ISDIR(lstat.st_mode): - flags &= DIR_FLAGS - elif stat.S_ISLNK(lstat.st_mode): - flags &= LINK_FLAGS - elif stat.S_ISFIFO(lstat.st_mode): - flags &= FIFO_FLAGS - elif stat.S_ISCHR(lstat.st_mode): - flags &= CHR_FLAGS - elif stat.S_ISBLK(lstat.st_mode): - flags &= BLK_FLAGS - else: - flags &= REG_FLAGS - - if (fflags & rpm.RPMFILE_GHOST): - flags &= GHOST_FLAGS - - flags &= ~(omitmask | RPMVERIFY_FAILURES) - - # 8. SELinux stuff. - - prelink_size = 0 - if flags & RPMVERIFY_MD5: - prelink_md5, prelink_size = prelink_md5_check(fname) - if prelink_md5 == False: - file_results.append('RPMVERIFY_MD5') - file_results.append('RPMVERIFY_READFAIL') - elif prelink_md5 != fmd5: - file_results.append('RPMVERIFY_MD5') - - if flags & RPMVERIFY_LINKTO: - linkto = os.readlink(fname) - if not linkto: - file_results.append('RPMVERIFY_READLINKFAIL') - file_results.append('RPMVERIFY_LINKTO') - else: - if len(rpmlinktos) == 0 or linkto != rpmlinktos: - file_results.append('RPMVERIFY_LINKTO') - - if flags & RPMVERIFY_FILESIZE: - if not (flags & RPMVERIFY_MD5): # prelink check hasn't been done. - prelink_size = prelink_size_check(fname) - if (prelink_size != 0): # This is a prelinked file. - if (prelink_size != fsize): - file_results.append('RPMVERIFY_FILESIZE') - elif lstat.st_size != fsize: # It wasn't a prelinked file. - file_results.append('RPMVERIFY_FILESIZE') - - if flags & RPMVERIFY_MODE: - metamode = fmode - filemode = lstat.st_mode - - # Comparing the type of %ghost files is meaningless, but perms are ok. - if fflags & rpm.RPMFILE_GHOST: - metamode &= ~0xf000 - filemode &= ~0xf000 - - if (stat.S_IFMT(metamode) != stat.S_IFMT(filemode)) or \ - (stat.S_IMODE(metamode) != stat.S_IMODE(filemode)): - file_results.append('RPMVERIFY_MODE') - - if flags & RPMVERIFY_RDEV: - if (stat.S_ISCHR(fmode) != stat.S_ISCHR(lstat.st_mode) or - stat.S_ISBLK(fmode) != stat.S_ISBLK(lstat.st_mode)): - file_results.append('RPMVERIFY_RDEV') - elif (s_isdev(fmode) & s_isdev(lstat.st_mode)): - st_rdev = lstat.st_rdev - if frdev != st_rdev: - file_results.append('RPMVERIFY_RDEV') - - if flags & RPMVERIFY_MTIME: - if lstat.st_mtime != fmtime: - file_results.append('RPMVERIFY_MTIME') - - if flags & RPMVERIFY_USER: - try: - user = pwd.getpwuid(lstat.st_uid)[0] - except KeyError: - user = None - if not user or not fuser or (user != fuser): - file_results.append('RPMVERIFY_USER') - - if flags & RPMVERIFY_GROUP: - try: - group = grp.getgrgid(lstat.st_gid)[0] - except KeyError: - group = None - if not group or not fgroup or (group != fgroup): - file_results.append('RPMVERIFY_GROUP') - - return file_results - -def rpm_verify_dependencies(header): - """ - Check package dependencies. Header is an rpm.hdr. - - Don't like opening another ts to do this, but - it was the only way I could find of clearing the ts - out. - - Have asked on the rpm-maint list on how to do - this the right way (28 Feb 2007). - - ts.check() returns: - - ((name, version, release), (reqname, reqversion), \ - flags, suggest, sense) - - """ - _ts1 = rpmtransactionset() - _ts1.addInstall(header, 'Dep Check', 'i') - dep_errors = _ts1.check() - _ts1.closeDB() - return dep_errors - -def rpm_verify_package(vp_ts, header, verify_options): - """ - Verify a single package specified by header. Header is an rpm.hdr. - - If errors are found it returns a dictionary of errors. - - """ - # Set some transaction level flags. - vsflags = 0 - if 'nodigest' in verify_options: - vsflags |= rpm._RPMVSF_NODIGESTS - if 'nosignature' in verify_options: - vsflags |= rpm._RPMVSF_NOSIGNATURES - ovsflags = vp_ts.setVSFlags(vsflags) - - # Map from the Python options to the rpm bitwise flags. - omitmask = 0 - - if 'nolinkto' in verify_options: - omitmask |= VERIFY_LINKTO - if 'nomd5' in verify_options: - omitmask |= VERIFY_MD5 - if 'nosize' in verify_options: - omitmask |= VERIFY_SIZE - if 'nouser' in verify_options: - omitmask |= VERIFY_USER - if 'nogroup' in verify_options: - omitmask |= VERIFY_GROUP - if 'nomtime' in verify_options: - omitmask |= VERIFY_MTIME - if 'nomode' in verify_options: - omitmask |= VERIFY_MODE - if 'nordev' in verify_options: - omitmask |= VERIFY_RDEV - - omitmask = ((~omitmask & VERIFY_ATTRS) ^ VERIFY_ATTRS) - - package_results = {} - - # Check Signatures and Digests. - # No idea what this might return. Need to break something to see. - # Setting the vsflags above determines what gets checked in the header. - hdr_stat = vp_ts.hdrCheck(header.unload()) - if hdr_stat: - package_results['hdr'] = hdr_stat - - # Check Package Depencies. - if 'nodeps' not in verify_options: - dep_stat = rpm_verify_dependencies(header) - if dep_stat: - package_results['deps'] = dep_stat - - # Check all the package files. - if 'nofiles' not in verify_options: - vp_fi = header.fiFromHeader() - for fileinfo in vp_fi: - # Do not bother doing anything with ghost files. - # This is what RPM does. - if fileinfo[4] & rpm.RPMFILE_GHOST: - continue - - # This is only needed because of an inconsistency in the - # rpm.fi interface. - linktos = vp_fi.FLink() - - file_stat = rpm_verify_file(fileinfo, linktos, omitmask) - - #if len(file_stat) > 0 or options.verbose: - if len(file_stat) > 0: - fflags = fileinfo[4] - if fflags & rpm.RPMFILE_CONFIG: - file_stat.append('c') - elif fflags & rpm.RPMFILE_DOC: - file_stat.append('d') - elif fflags & rpm.RPMFILE_GHOST: - file_stat.append('g') - elif fflags & rpm.RPMFILE_LICENSE: - file_stat.append('l') - elif fflags & rpm.RPMFILE_PUBKEY: - file_stat.append('P') - elif fflags & rpm.RPMFILE_README: - file_stat.append('r') - else: - file_stat.append(' ') - - file_stat.append(fileinfo[0]) # The filename. - package_results.setdefault('files', []).append(file_stat) - - # Run the verify script if there is one. - # Do we want this? - #if 'noscripts' not in verify_options: - # script_stat = rpmVerifyscript() - # if script_stat: - # package_results['script'] = script_stat - - # If there have been any errors, add the package nevra to the result. - if len(package_results) > 0: - package_results.setdefault('nevra', (header[rpm.RPMTAG_NAME], \ - header[rpm.RPMTAG_EPOCH], \ - header[rpm.RPMTAG_VERSION], \ - header[rpm.RPMTAG_RELEASE], \ - header[rpm.RPMTAG_ARCH])) - else: - package_results = None - - # Put things back the way we found them. - vsflags = vp_ts.setVSFlags(ovsflags) - - return package_results - -def rpm_verify(verify_ts, verify_pkgspec, verify_options=[]): - """ - Requires rpmtransactionset() to be run first to get a ts. - - pkgspec is a dict specifying the package - e.g.: - For a single package - { name='foo', epoch='20', version='1', release='1', arch='x86_64'} - - For all packages - {} - - Or any combination of keywords to select one or more packages to verify. - - options is a list of 'rpm --verify' options. Default is to check everything. - e.g.: - [ 'nodeps', 'nodigest', 'nofiles', 'noscripts', 'nosignature', - 'nolinkto' 'nomd5', 'nosize', 'nouser', 'nogroup', 'nomtime', - 'nomode', 'nordev' ] - - Returns a list. One list entry per package. Each list entry is a - dictionary. Dict keys are 'files', 'deps', 'nevra' and 'hdr'. - Entries only get added for the failures. If nothing failed, None is - returned. - - Its all a bit messy and probably needs reviewing. - - [ { 'hdr': [???], - 'deps: [((name, version, release), (reqname, reqversion), - flags, suggest, sense), .... ] - 'files': [ ['filename1', 'RPMVERIFY_GROUP', 'RPMVERIFY_USER' ], - ['filename2', 'RPMVERFIY_LSTATFAIL']] - 'nevra': ['name1', 'epoch1', 'version1', 'release1', 'arch1'] } - { 'hdr': [???], - 'deps: [((name, version, release), (reqname, reqversion), - flags, suggest, sense), .... ] - 'files': [ ['filename', 'RPMVERIFY_GROUP', 'RPMVERIFY_USER" ], - ['filename2', 'RPMVERFIY_LSTATFAIL']] - 'nevra': ['name2', 'epoch2', 'version2', 'release2', 'arch2'] } ] - - """ - verify_results = [] - headers = getheadersbykeyword(verify_ts, **verify_pkgspec) - for header in headers: - result = rpm_verify_package(verify_ts, header, verify_options) - if result: - verify_results.append(result) - - return verify_results - -def rpmtransactionset(): - """ - A simple wrapper for rpm.TransactionSet() to keep everthiing together. - Might use it to set some ts level flags later. - - """ - ts = rpm.TransactionSet() - return ts - -class Rpmtscallback(object): - """ - Callback for ts.run(). Used for adding, upgrading and removing packages. - Starting with all possible reasons codes, but bcfg2 will probably only - make use of a few of them. - - Mostly just printing stuff at the moment to understand how the callback - is used. - - """ - def __init__(self): - self.fdnos = {} - - def callback(self, reason, amount, total, key, client_data): - """ - Generic rpmts call back. - """ - if reason == rpm.RPMCALLBACK_INST_OPEN_FILE: - pass - elif reason == rpm.RPMCALLBACK_INST_CLOSE_FILE: - pass - elif reason == rpm.RPMCALLBACK_INST_START: - pass - elif reason == rpm.RPMCALLBACK_TRANS_PROGRESS or \ - reason == rpm.RPMCALLBACK_INST_PROGRESS: - pass - # rpm.RPMCALLBACK_INST_PROGRESS' - elif reason == rpm.RPMCALLBACK_TRANS_START: - pass - elif reason == rpm.RPMCALLBACK_TRANS_STOP: - pass - elif reason == rpm.RPMCALLBACK_REPACKAGE_START: - pass - elif reason == rpm.RPMCALLBACK_REPACKAGE_PROGRESS: - pass - elif reason == rpm.RPMCALLBACK_REPACKAGE_STOP: - pass - elif reason == rpm.RPMCALLBACK_UNINST_PROGRESS: - pass - elif reason == rpm.RPMCALLBACK_UNINST_START: - pass - elif reason == rpm.RPMCALLBACK_UNINST_STOP: - pass - # How do we get at this? - # RPM.modified += key - elif reason == rpm.RPMCALLBACK_UNPACK_ERROR: - pass - elif reason == rpm.RPMCALLBACK_CPIO_ERROR: - pass - elif reason == rpm.RPMCALLBACK_UNKNOWN: - pass - else: - print('ERROR - Fell through callBack') - - -def rpm_erase(erase_pkgspecs, erase_flags): - """ - pkgspecs is a list of pkgspec dicts specifying packages - e.g.: - For a single package - { name='foo', epoch='20', version='1', release='1', arch='x86_64'} - - """ - erase_ts_flags = 0 - if 'noscripts' in erase_flags: - erase_ts_flags |= rpm.RPMTRANS_FLAG_NOSCRIPTS - if 'notriggers' in erase_flags: - erase_ts_flags |= rpm.RPMTRANS_FLAG_NOTRIGGERS - if 'repackage' in erase_flags: - erase_ts_flags |= rpm.RPMTRANS_FLAG_REPACKAGE - - erase_ts = rpmtransactionset() - erase_ts.setFlags(erase_ts_flags) - - for pkgspec in erase_pkgspecs: - idx_list = getindexbykeyword(erase_ts, **pkgspec) - if len(idx_list) > 1 and not 'allmatches' in erase_flags: - #pass - print('ERROR - Multiple package match for erase', pkgspec) - else: - for idx in idx_list: - erase_ts.addErase(idx) - - #for te in erase_ts: - - erase_problems = [] - if 'nodeps' not in erase_flags: - erase_problems = erase_ts.check() - - if erase_problems == []: - erase_ts.order() - erase_callback = Rpmtscallback() - erase_ts.run(erase_callback.callback, 'Erase') - #else: - - erase_ts.closeDB() - del erase_ts - return erase_problems - -def display_verify_file(file_results): - ''' - Display file results similar to rpm --verify. - ''' - filename = file_results[-1] - filetype = file_results[-2] - - result_string = '' - - if 'RPMVERIFY_LSTATFAIL' in file_results: - result_string = 'missing ' - else: - if 'RPMVERIFY_FILESIZE' in file_results: - result_string = result_string + 'S' - else: - result_string = result_string + '.' - - if 'RPMVERIFY_MODE' in file_results: - result_string = result_string + 'M' - else: - result_string = result_string + '.' - - if 'RPMVERIFY_MD5' in file_results: - if 'RPMVERIFY_READFAIL' in file_results: - result_string = result_string + '?' - else: - result_string = result_string + '5' - else: - result_string = result_string + '.' - - if 'RPMVERIFY_RDEV' in file_results: - result_string = result_string + 'D' - else: - result_string = result_string + '.' - - if 'RPMVERIFY_LINKTO' in file_results: - if 'RPMVERIFY_READLINKFAIL' in file_results: - result_string = result_string + '?' - else: - result_string = result_string + 'L' - else: - result_string = result_string + '.' - - if 'RPMVERIFY_USER' in file_results: - result_string = result_string + 'U' - else: - result_string = result_string + '.' - - if 'RPMVERIFY_GROUP' in file_results: - result_string = result_string + 'G' - else: - result_string = result_string + '.' - - if 'RPMVERIFY_MTIME' in file_results: - result_string = result_string + 'T' - else: - result_string = result_string + '.' - - print(result_string + ' ' + filetype + ' ' + filename) - sys.stdout.flush() - -#=============================================================================== -# Some options and output to assist with development and testing. -# These are not intended for normal use. -if __name__ == "__main__": - - p = optparse.OptionParser() - - p.add_option('--name', action='store', \ - default=None, \ - help='''Package name to verify. - - ****************************************** - NOT SPECIFYING A NAME MEANS 'ALL' PACKAGES. - ****************************************** - - The specified operation will be carried out on all - instances of packages that match the package specification - (name, epoch, version, release, arch).''') - - p.add_option('--epoch', action='store', \ - default=None, \ - help='''Package epoch.''') - - p.add_option('--version', action='store', \ - default=None, \ - help='''Package version.''') - - p.add_option('--release', action='store', \ - default=None, \ - help='''Package release.''') - - p.add_option('--arch', action='store', \ - default=None, \ - help='''Package arch.''') - - p.add_option('--erase', '-e', action='store_true', \ - default=None, \ - help='''**************************************************** - REMOVE PACKAGES. THERE ARE NO WARNINGS. MULTIPLE - PACKAGES WILL BE REMOVED IF A FULL PACKAGE SPEC IS NOT - GIVEN. E.G. IF JUST A NAME IS GIVEN ALL INSTALLED - INSTANCES OF THAT PACKAGE WILL BE REMOVED PROVIDED - DEPENDENCY CHECKS PASS. IF JUST AN EPOCH IS GIVEN - ALL PACKAGE INSTANCES WITH THAT EPOCH WILL BE REMOVED. - ****************************************************''') - - p.add_option('--list', '-l', action='store_true', \ - help='''List package identity info. rpm -qa ish equivalent - intended for use in RefreshPackages().''') - - p.add_option('--verify', action='store_true', \ - help='''Verify Package(s). Output is only produced after all - packages has been verified. Be patient.''') - - p.add_option('--verbose', '-v', action='store_true', \ - help='''Verbose output for --verify option. Output is the - same as rpm -v --verify.''') - - p.add_option('--nodeps', action='store_true', \ - default=False, \ - help='Do not do dependency testing.') - - p.add_option('--nodigest', action='store_true', \ - help='Do not check package digests.') - - p.add_option('--nofiles', action='store_true', \ - help='Do not do file checks.') - - p.add_option('--noscripts', action='store_true', \ - help='Do not run verification scripts.') - - p.add_option('--nosignature', action='store_true', \ - help='Do not do package signature verification.') - - p.add_option('--nolinkto', action='store_true', \ - help='Do not do symlink tests.') - - p.add_option('--nomd5', action='store_true', \ - help='''Do not do MD5 checksums on files. Note that this does - not work for prelink files yet.''') - - p.add_option('--nosize', action='store_true', \ - help='''Do not do file size tests. Note that this does not work - for prelink files yet.''') - - p.add_option('--nouser', action='store_true', \ - help='Do not check file user ownership.') - - p.add_option('--nogroup', action='store_true', \ - help='Do not check file group ownership.') - - p.add_option('--nomtime', action='store_true', \ - help='Do not check file modification times.') - - p.add_option('--nomode', action='store_true', \ - help='Do not check file modes (permissions).') - - p.add_option('--nordev', action='store_true', \ - help='Do not check device node.') - - p.add_option('--notriggers', action='store_true', \ - help='Do not do not generate triggers on erase.') - - p.add_option('--repackage', action='store_true', \ - help='''Do repackage on erase.i Packages are put - in /var/spool/repackage.''') - - p.add_option('--allmatches', action='store_true', \ - help='''Remove all package instances that match the - pkgspec. - - *************************************************** - NO WARNINGS ARE GIVEN. IF THERE IS NO PACKAGE SPEC - THAT MEANS ALL PACKAGES!!!! - ***************************************************''') - - options, arguments = p.parse_args() - - pkgspec = {} - rpm_options = [] - - if options.nodeps: - rpm_options.append('nodeps') - - if options.nodigest: - rpm_options.append('nodigest') - - if options.nofiles: - rpm_options.append('nofiles') - - if options.noscripts: - rpm_options.append('noscripts') - - if options.nosignature: - rpm_options.append('nosignature') - - if options.nolinkto: - rpm_options.append('nolinkto') - - if options.nomd5: - rpm_options.append('nomd5') - - if options.nosize: - rpm_options.append('nosize') - - if options.nouser: - rpm_options.append('nouser') - - if options.nogroup: - rpm_options.append('nogroup') - - if options.nomtime: - rpm_options.append('nomtime') - - if options.nomode: - rpm_options.append('nomode') - - if options.nordev: - rpm_options.append('nordev') - - if options.repackage: - rpm_options.append('repackage') - - if options.allmatches: - rpm_options.append('allmatches') - - main_ts = rpmtransactionset() - - cmdline_pkgspec = {} - if options.name != 'all': - if options.name: - cmdline_pkgspec['name'] = str(options.name) - if options.epoch: - cmdline_pkgspec['epoch'] = str(options.epoch) - if options.version: - cmdline_pkgspec['version'] = str(options.version) - if options.release: - cmdline_pkgspec['release'] = str(options.release) - if options.arch: - cmdline_pkgspec['arch'] = str(options.arch) - - if options.verify: - results = rpm_verify(main_ts, cmdline_pkgspec, rpm_options) - for r in results: - files = r.get('files', '') - for f in files: - display_verify_file(f) - - elif options.list: - for p in rpmpackagelist(main_ts): - print(p) - - elif options.erase: - if options.name: - rpm_erase([cmdline_pkgspec], rpm_options) - else: - print('You must specify the "--name" option') diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 6d1cb9d40..5f4f15dcc 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -2,8 +2,55 @@ import os import sys -import select -from Bcfg2.Compat import input # pylint: disable=W0622 +import stat +import time +import fcntl +import socket +import fnmatch +import logging +import argparse +import tempfile +import Bcfg2.Logger +import Bcfg2.Options +from Bcfg2.Client import XML +from Bcfg2.Client import Proxy +from Bcfg2.Client import Tools +from Bcfg2.Utils import locked, Executor, safe_input +from Bcfg2.version import __version__ +# pylint: disable=W0622 +from Bcfg2.Compat import xmlrpclib, walk_packages, any, all, cmp +# pylint: enable=W0622 + + +def cmpent(ent1, ent2): + """Sort entries.""" + if ent1.tag != ent2.tag: + return cmp(ent1.tag, ent2.tag) + else: + return cmp(ent1.get('name'), ent2.get('name')) + + +def matches_entry(entryspec, entry): + """ Determine if the Decisions-style entry specification matches + the entry. Both are tuples of (tag, name). The entryspec can + handle the wildcard * in either position. """ + if entryspec == entry: + return True + return all(fnmatch.fnmatch(entry[i], entryspec[i]) for i in [0, 1]) + + +def matches_white_list(entry, whitelist): + """ Return True if (<entry tag>, <entry name>) is in the given + whitelist. """ + return any(matches_entry(we, (entry.tag, entry.get('name'))) + for we in whitelist) + + +def passes_black_list(entry, blacklist): + """ Return True if (<entry tag>, <entry name>) is not in the given + blacklist. """ + return not any(matches_entry(be, (entry.tag, entry.get('name'))) + for be in blacklist) def prompt(msg): @@ -16,14 +63,850 @@ def prompt(msg): contain "[y/N]" if desired, etc. :type msg: string :returns: bool - True if yes, False if no """ - while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) try: - ans = input(msg) + ans = safe_input(msg) return ans in ['y', 'Y'] except UnicodeEncodeError: ans = input(msg.encode('utf-8')) return ans in ['y', 'Y'] - except EOFError: - # handle ^C on rhel-based platforms + except (EOFError, KeyboardInterrupt): + # handle ^C raise SystemExit(1) + except: + print("Error while reading input: %s" % sys.exc_info()[1]) + return False + + +class ClientDriverAction(Bcfg2.Options.ComponentAction): + """ Action to load client drivers """ + bases = ['Bcfg2.Client.Tools'] + fail_silently = True + + +class Client(object): + """ The main Bcfg2 client class """ + + options = Proxy.ComponentProxy.options + [ + Bcfg2.Options.Common.syslog, + Bcfg2.Options.Common.interactive, + Bcfg2.Options.BooleanOption( + "-q", "--quick", help="Disable some checksum verification"), + Bcfg2.Options.Option( + cf=('client', 'probe_timeout'), + type=Bcfg2.Options.Types.timeout, + help="Timeout when running client probes"), + Bcfg2.Options.Option( + "-b", "--only-bundles", default=[], + type=Bcfg2.Options.Types.colon_list, + help='Only configure the given bundle(s)'), + Bcfg2.Options.Option( + "-B", "--except-bundles", default=[], + type=Bcfg2.Options.Types.colon_list, + help='Configure everything except the given bundle(s)'), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption( + "-Q", "--bundle-quick", + help='Only verify the given bundle(s)'), + Bcfg2.Options.Option( + '-r', '--remove', + choices=['all', 'services', 'packages', 'users'], + help='Force removal of additional configuration items')), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.PathOption( + '-f', '--file', type=argparse.FileType('rb'), + help='Configure from a file rather than querying the server'), + Bcfg2.Options.PathOption( + '-c', '--cache', type=argparse.FileType('wb'), + help='Store the configuration in a file')), + Bcfg2.Options.BooleanOption( + '--exit-on-probe-failure', default=True, + cf=('client', 'exit_on_probe_failure'), + help="The client should exit if a probe fails"), + Bcfg2.Options.Option( + '-p', '--profile', cf=('client', 'profile'), + help='Assert the given profile for the host'), + Bcfg2.Options.Option( + '-l', '--decision', cf=('client', 'decision'), + choices=['whitelist', 'blacklist', 'none'], + help='Run client in server decision list mode'), + Bcfg2.Options.BooleanOption( + "-O", "--no-lock", help='Omit lock check'), + Bcfg2.Options.PathOption( + cf=('components', 'lockfile'), default='/var/lock/bcfg2.run', + help='Client lock file'), + Bcfg2.Options.BooleanOption( + "-n", "--dry-run", help='Do not actually change the system'), + Bcfg2.Options.Option( + "-D", "--drivers", cf=('client', 'drivers'), + type=Bcfg2.Options.Types.comma_list, + default=[m[1] for m in walk_packages(path=Tools.__path__)], + action=ClientDriverAction, help='Client drivers'), + Bcfg2.Options.BooleanOption( + "-e", "--show-extra", help='Enable extra entry output'), + Bcfg2.Options.BooleanOption( + "-k", "--kevlar", help='Run in bulletproof mode'), + Bcfg2.Options.BooleanOption( + "-i", "--only-important", + help='Only configure the important entries')] + + def __init__(self): + self.config = None + self._proxy = None + self.logger = logging.getLogger('bcfg2') + self.cmd = Executor(Bcfg2.Options.setup.probe_timeout) + self.tools = [] + self.times = dict() + self.times['initialization'] = time.time() + + if Bcfg2.Options.setup.bundle_quick: + if (not Bcfg2.Options.setup.only_bundles and + not Bcfg2.Options.setup.except_bundles): + self.logger.error("-Q option requires -b or -B") + raise SystemExit(1) + if Bcfg2.Options.setup.remove == 'services': + self.logger.error("Service removal is nonsensical; " + "removed services will only be disabled") + if not Bcfg2.Options.setup.server.startswith('https://'): + Bcfg2.Options.setup.server = \ + 'https://' + Bcfg2.Options.setup.server + + #: A dict of the state of each entry. Keys are the entries. + #: Values are boolean: True means that the entry is good, + #: False means that the entry is bad. + self.states = {} + self.whitelist = [] + self.blacklist = [] + self.removal = [] + self.unhandled = [] + self.logger = logging.getLogger(__name__) + + def _probe_failure(self, probename, msg): + """ handle failure of a probe in the way the user wants us to + (exit or continue) """ + message = "Failed to execute probe %s: %s" % (probename, msg) + if Bcfg2.Options.setup.exit_on_probe_failure: + self.fatal_error(message) + else: + self.logger.error(message) + + def run_probe(self, probe): + """Execute probe.""" + name = probe.get('name') + self.logger.info("Running probe %s" % name) + ret = XML.Element("probe-data", name=name, source=probe.get('source')) + try: + scripthandle, scriptname = tempfile.mkstemp() + if sys.hexversion >= 0x03000000: + script = os.fdopen(scripthandle, 'w', + encoding=Bcfg2.Options.setup.encoding) + else: + script = os.fdopen(scripthandle, 'w') + try: + script.write("#!%s\n" % + (probe.attrib.get('interpreter', '/bin/sh'))) + if sys.hexversion >= 0x03000000: + script.write(probe.text) + else: + script.write(probe.text.encode('utf-8')) + script.close() + os.chmod(scriptname, + stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | + stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | + stat.S_IWUSR) # 0755 + rv = self.cmd.run(scriptname) + if rv.stderr: + self.logger.warning("Probe %s has error output: %s" % + (name, rv.stderr)) + if not rv.success: + self._probe_failure(name, "Return value %s" % rv.retval) + self.logger.info("Probe %s has result:" % name) + self.logger.info(rv.stdout) + if sys.hexversion >= 0x03000000: + ret.text = rv.stdout + else: + ret.text = rv.stdout.decode('utf-8') + finally: + os.unlink(scriptname) + except SystemExit: + raise + except: + self._probe_failure(name, sys.exc_info()[1]) + return ret + + def fatal_error(self, message): + """Signal a fatal error.""" + self.logger.error("Fatal error: %s" % (message)) + raise SystemExit(1) + + @property + def proxy(self): + """ get an XML-RPC proxy to the server """ + if self._proxy is None: + self._proxy = Proxy.ComponentProxy() + return self._proxy + + def run_probes(self): + """ run probes and upload probe data """ + try: + probes = XML.XML(str(self.proxy.GetProbes())) + except (Proxy.ProxyError, + Proxy.CertificateError, + socket.gaierror, + socket.error): + err = sys.exc_info()[1] + self.fatal_error("Failed to download probes from bcfg2: %s" % err) + except XML.ParseError: + err = sys.exc_info()[1] + self.fatal_error("Server returned invalid probe requests: %s" % + err) + + self.times['probe_download'] = time.time() + + # execute probes + probedata = XML.Element("ProbeData") + for probe in probes.findall(".//probe"): + probedata.append(self.run_probe(probe)) + + if len(probes.findall(".//probe")) > 0: + try: + # upload probe responses + self.proxy.RecvProbeData( + XML.tostring(probedata, + xml_declaration=False).decode('utf-8')) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to upload probe data: %s" % err) + + self.times['probe_upload'] = time.time() + + def get_config(self): + """ load the configuration, either from the cached + configuration file (-f), or from the server """ + if Bcfg2.Options.setup.file: + # read config from file + try: + self.logger.debug("Reading cached configuration from %s" % + Bcfg2.Options.setup.file.name) + return Bcfg2.Options.setup.file.read() + except IOError: + self.fatal_error("Failed to read cached configuration from: %s" + % Bcfg2.Options.setup.file.name) + else: + # retrieve config from server + if Bcfg2.Options.setup.profile: + try: + self.proxy.AssertProfile(Bcfg2.Options.setup.profile) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to set client profile: %s" % err) + + try: + self.proxy.DeclareVersion(__version__) + except (xmlrpclib.Fault, + Proxy.ProxyError, + Proxy.CertificateError, + socket.gaierror, + socket.error): + err = sys.exc_info()[1] + self.fatal_error("Failed to declare version: %s" % err) + + self.run_probes() + + if Bcfg2.Options.setup.decision in ['whitelist', 'blacklist']: + try: + # TODO: read decision list from --decision-list + Bcfg2.Options.setup.decision_list = \ + self.proxy.GetDecisionList( + Bcfg2.Options.setup.decision) + self.logger.info("Got decision list from server:") + self.logger.info(Bcfg2.Options.setup.decision_list) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to get decision list: %s" % err) + + try: + rawconfig = self.proxy.GetConfig().encode('utf-8') + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to download configuration from " + "Bcfg2: %s" % err) + + self.times['config_download'] = time.time() + + if Bcfg2.Options.setup.cache: + try: + Bcfg2.Options.setup.cache.write(rawconfig) + os.chmod(Bcfg2.Options.setup.cache.name, 384) # 0600 + except IOError: + self.logger.warning("Failed to write config cache file %s" % + (Bcfg2.Options.setup.cache)) + self.times['caching'] = time.time() + + return rawconfig + + def parse_config(self, rawconfig): + """ Parse the XML configuration received from the Bcfg2 server """ + try: + self.config = XML.XML(rawconfig) + except XML.ParseError: + syntax_error = sys.exc_info()[1] + self.fatal_error("The configuration could not be parsed: %s" % + syntax_error) + + self.load_tools() + + # find entries not handled by any tools + self.unhandled = [entry for struct in self.config + for entry in struct + if entry not in self.handled] + + if self.unhandled: + self.logger.error("The following entries are not handled by any " + "tool:") + for entry in self.unhandled: + self.logger.error("%s:%s:%s" % (entry.tag, entry.get('type'), + entry.get('name'))) + + # find duplicates + self.find_dups(self.config) + + pkgs = [(entry.get('name'), entry.get('origin')) + for struct in self.config + for entry in struct + if entry.tag == 'Package'] + if pkgs: + self.logger.debug("The following packages are specified in bcfg2:") + self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] is None]) + self.logger.debug("The following packages are prereqs added by " + "Packages:") + self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages']) + + self.times['config_parse'] = time.time() + + def run(self): + """Perform client execution phase.""" + # begin configuration + self.times['start'] = time.time() + + self.logger.info("Starting Bcfg2 client run at %s" % + self.times['start']) + + self.parse_config(self.get_config().decode('utf-8')) + + if self.config.tag == 'error': + self.fatal_error("Server error: %s" % (self.config.text)) + + if Bcfg2.Options.setup.bundle_quick: + newconfig = XML.XML('<Configuration/>') + for bundle in self.config.getchildren(): + name = bundle.get("name") + if (name and (name in Bcfg2.Options.setup.only_bundles or + name not in Bcfg2.Options.setup.except_bundles)): + newconfig.append(bundle) + self.config = newconfig + + if not Bcfg2.Options.setup.no_lock: + # check lock here + try: + lockfile = open(Bcfg2.Options.setup.lockfile, 'w') + if locked(lockfile.fileno()): + self.fatal_error("Another instance of Bcfg2 is running. " + "If you want to bypass the check, run " + "with the -O/--no-lock option") + except SystemExit: + raise + except: + lockfile = None + self.logger.error("Failed to open lockfile %s: %s" % + (Bcfg2.Options.setup.lockfile, + sys.exc_info()[1])) + + # execute the configuration + self.Execute() + + if not Bcfg2.Options.setup.no_lock: + # unlock here + if lockfile: + try: + fcntl.lockf(lockfile.fileno(), fcntl.LOCK_UN) + os.remove(Bcfg2.Options.setup.lockfile) + except OSError: + self.logger.error("Failed to unlock lockfile %s" % + lockfile.name) + + if (not Bcfg2.Options.setup.file and + not Bcfg2.Options.setup.bundle_quick): + # upload statistics + feedback = self.GenerateStats() + + try: + self.proxy.RecvStats( + XML.tostring(feedback, + xml_declaration=False).decode('utf-8')) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.logger.error("Failed to upload configuration statistics: " + "%s" % err) + raise SystemExit(2) + + self.logger.info("Finished Bcfg2 client run at %s" % time.time()) + + def load_tools(self): + """ Load all applicable client tools """ + for tool in Bcfg2.Options.setup.drivers: + try: + self.tools.append(tool(self.config)) + except Tools.ToolInstantiationError: + continue + except: + self.logger.error("Failed to instantiate tool %s" % tool, + exc_info=1) + + for tool in self.tools[:]: + for conflict in getattr(tool, 'conflicts', []): + for item in self.tools: + if item.name == conflict: + self.tools.remove(item) + + self.logger.info("Loaded tool drivers:") + self.logger.info([tool.name for tool in self.tools]) + + deprecated = [tool.name for tool in self.tools if tool.deprecated] + if deprecated: + self.logger.warning("Loaded deprecated tool drivers:") + self.logger.warning(deprecated) + experimental = [tool.name for tool in self.tools if tool.experimental] + if experimental: + self.logger.warning("Loaded experimental tool drivers:") + self.logger.warning(experimental) + + def find_dups(self, config): + """ Find duplicate entries and warn about them """ + entries = dict() + for struct in config: + for entry in struct: + for tool in self.tools: + if tool.handlesEntry(entry): + pkey = tool.primarykey(entry) + if pkey in entries: + entries[pkey] += 1 + else: + entries[pkey] = 1 + multi = [e for e, c in entries.items() if c > 1] + if multi: + self.logger.debug("The following entries are included multiple " + "times:") + for entry in multi: + self.logger.debug(entry) + + def promptFilter(self, msg, entries): + """Filter a supplied list based on user input.""" + ret = [] + entries.sort(key=lambda e: e.tag + ":" + e.get('name')) + for entry in entries[:]: + if entry in self.unhandled: + # don't prompt for entries that can't be installed + continue + if 'qtext' in entry.attrib: + iprompt = entry.get('qtext') + else: + iprompt = msg % (entry.tag, entry.get('name')) + if prompt(iprompt): + ret.append(entry) + return ret + + def __getattr__(self, name): + if name in ['extra', 'handled', 'modified', '__important__']: + ret = [] + for tool in self.tools: + ret += getattr(tool, name) + return ret + elif name in self.__dict__: + return self.__dict__[name] + raise AttributeError(name) + + def InstallImportant(self): + """Install important entries + + We also process the decision mode stuff here because we want to prevent + non-whitelisted/blacklisted 'important' entries from being installed + prior to determining the decision mode on the client. + """ + # Need to process decision stuff early so that dryrun mode + # works with it + self.whitelist = [entry for entry in self.states + if not self.states[entry]] + if not Bcfg2.Options.setup.file: + if Bcfg2.Options.setup.decision == 'whitelist': + dwl = Bcfg2.Options.setup.decision_list + w_to_rem = [e for e in self.whitelist + if not matches_white_list(e, dwl)] + if w_to_rem: + self.logger.info("In whitelist mode: " + "suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in w_to_rem]) + self.whitelist = [x for x in self.whitelist + if x not in w_to_rem] + elif Bcfg2.Options.setup.decision == 'blacklist': + b_to_rem = \ + [e for e in self.whitelist + if not + passes_black_list(e, Bcfg2.Options.setup.decision_list)] + if b_to_rem: + self.logger.info("In blacklist mode: " + "suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in b_to_rem]) + self.whitelist = [x for x in self.whitelist + if x not in b_to_rem] + + # take care of important entries first + if (not Bcfg2.Options.setup.dry_run or + Bcfg2.Options.setup.only_important): + important_installs = set() + for parent in self.config.findall(".//Path/.."): + name = parent.get("name") + if not name or (name in Bcfg2.Options.setup.except_bundles and + name not in Bcfg2.Options.setup.only_bundles): + continue + for cfile in parent.findall("./Path"): + if (cfile.get('name') not in self.__important__ or + cfile.get('type') != 'file' or + cfile not in self.whitelist): + continue + tools = [t for t in self.tools + if t.handlesEntry(cfile) and t.canVerify(cfile)] + if not tools: + continue + if Bcfg2.Options.setup.dry_run: + important_installs.add(cfile) + continue + if (Bcfg2.Options.setup.interactive and not + self.promptFilter("Install %s: %s? (y/N):", + [cfile])): + self.whitelist.remove(cfile) + continue + try: + self.states[cfile] = tools[0].InstallPath(cfile) + if self.states[cfile]: + tools[0].modified.append(cfile) + except: # pylint: disable=W0702 + self.logger.error("Unexpected tool failure", + exc_info=1) + cfile.set('qtext', '') + if tools[0].VerifyPath(cfile, []): + self.whitelist.remove(cfile) + if Bcfg2.Options.setup.dry_run and len(important_installs) > 0: + self.logger.info("In dryrun mode: " + "suppressing entry installation for:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in important_installs]) + + def Inventory(self): + """ + Verify all entries, + find extra entries, + and build up workqueues + + """ + # initialize all states + for struct in self.config.getchildren(): + for entry in struct.getchildren(): + self.states[entry] = False + for tool in self.tools: + try: + self.states.update(tool.Inventory()) + except: # pylint: disable=W0702 + self.logger.error("%s.Inventory() call failed:" % tool.name, + exc_info=1) + + def Decide(self): # pylint: disable=R0912 + """Set self.whitelist based on user interaction.""" + iprompt = "Install %s: %s? (y/N): " + rprompt = "Remove %s: %s? (y/N): " + if Bcfg2.Options.setup.remove: + if Bcfg2.Options.setup.remove == 'all': + self.removal = self.extra + elif Bcfg2.Options.setup.remove == 'services': + self.removal = [entry for entry in self.extra + if entry.tag == 'Service'] + elif Bcfg2.Options.setup.remove == 'packages': + self.removal = [entry for entry in self.extra + if entry.tag == 'Package'] + elif Bcfg2.Options.setup.remove == 'users': + self.removal = [entry for entry in self.extra + if entry.tag in ['POSIXUser', 'POSIXGroup']] + + candidates = [entry for entry in self.states + if not self.states[entry]] + + if Bcfg2.Options.setup.dry_run: + if self.whitelist: + self.logger.info("In dryrun mode: " + "suppressing entry installation for:") + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) + for entry in self.whitelist]) + self.whitelist = [] + if self.removal: + self.logger.info("In dryrun mode: " + "suppressing entry removal for:") + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) + for entry in self.removal]) + self.removal = [] + + # Here is where most of the work goes + # first perform bundle filtering + all_bundle_names = [b.get('name') + for b in self.config.findall('./Bundle')] + bundles = self.config.getchildren() + if Bcfg2.Options.setup.only_bundles: + # warn if non-existent bundle given + for bundle in Bcfg2.Options.setup.only_bundles: + if bundle not in all_bundle_names: + self.logger.info("Warning: Bundle %s not found" % bundle) + bundles = [b for b in bundles + if b.get('name') in Bcfg2.Options.setup.only_bundles] + if Bcfg2.Options.setup.except_bundles: + # warn if non-existent bundle given + if not Bcfg2.Options.setup.bundle_quick: + for bundle in Bcfg2.Options.setup.except_bundles: + if bundle not in all_bundle_names: + self.logger.info("Warning: Bundle %s not found" % + bundle) + bundles = [ + b for b in bundles + if b.get('name') not in Bcfg2.Options.setup.except_bundles] + self.whitelist = [e for e in self.whitelist + if any(e in b for b in bundles)] + + # first process prereq actions + for bundle in bundles[:]: + if bundle.tag == 'Bundle': + bmodified = any((item in self.whitelist or + item in self.modified) for item in bundle) + else: + bmodified = False + actions = [a for a in bundle.findall('./Action') + if (a.get('timing') in ['pre', 'both'] and + (bmodified or a.get('when') == 'always'))] + # now we process all "pre" and "both" actions that are either + # always or the bundle has been modified + if Bcfg2.Options.setup.interactive: + self.promptFilter(iprompt, actions) + self.DispatchInstallCalls(actions) + + if bundle.tag != 'Bundle': + continue + + # need to test to fail entries in whitelist + if not all(self.states[a] for a in actions): + # then display bundles forced off with entries + self.logger.info("%s %s failed prerequisite action" % + (bundle.tag, bundle.get('name'))) + bundles.remove(bundle) + b_to_remv = [ent for ent in self.whitelist if ent in bundle] + if b_to_remv: + self.logger.info("Not installing entries from %s %s" % + (bundle.tag, bundle.get('name'))) + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in b_to_remv]) + for ent in b_to_remv: + self.whitelist.remove(ent) + + self.logger.debug("Installing entries in the following bundle(s):") + self.logger.debug(" %s" % ", ".join(b.get("name") for b in bundles + if b.get("name"))) + + if Bcfg2.Options.setup.interactive: + self.whitelist = self.promptFilter(iprompt, self.whitelist) + self.removal = self.promptFilter(rprompt, self.removal) + + for entry in candidates: + if entry not in self.whitelist: + self.blacklist.append(entry) + + def DispatchInstallCalls(self, entries): + """Dispatch install calls to underlying tools.""" + for tool in self.tools: + handled = [entry for entry in entries if tool.canInstall(entry)] + if not handled: + continue + try: + self.states.update(tool.Install(handled)) + except: # pylint: disable=W0702 + self.logger.error("%s.Install() call failed:" % tool.name, + exc_info=1) + + def Install(self): + """Install all entries.""" + self.DispatchInstallCalls(self.whitelist) + mods = self.modified + mbundles = [struct for struct in self.config.findall('Bundle') + if any(True for mod in mods if mod in struct)] + + if self.modified: + # Handle Bundle interdeps + if mbundles: + self.logger.info("The Following Bundles have been modified:") + self.logger.info([mbun.get('name') for mbun in mbundles]) + tbm = [(t, b) for t in self.tools for b in mbundles] + for tool, bundle in tbm: + try: + self.states.update(tool.Inventory(structures=[bundle])) + except: # pylint: disable=W0702 + self.logger.error("%s.Inventory() call failed:" % + tool.name, + exc_info=1) + clobbered = [entry for bundle in mbundles for entry in bundle + if (not self.states[entry] and + entry not in self.blacklist)] + if clobbered: + self.logger.debug("Found clobbered entries:") + self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) + for entry in clobbered]) + if not Bcfg2.Options.setup.interactive: + self.DispatchInstallCalls(clobbered) + + for bundle in self.config.findall('.//Bundle'): + if (Bcfg2.Options.setup.only_bundles and + bundle.get('name') not in + Bcfg2.Options.setup.only_bundles): + # prune out unspecified bundles when running with -b + continue + if bundle in mbundles: + self.logger.debug("Bundle %s was modified" % + bundle.get('name')) + func = "BundleUpdated" + else: + self.logger.debug("Bundle %s was not modified" % + bundle.get('name')) + func = "BundleNotUpdated" + for tool in self.tools: + try: + self.states.update(getattr(tool, func)(bundle)) + except: # pylint: disable=W0702 + self.logger.error("%s.%s(%s:%s) call failed:" % + (tool.name, func, bundle.tag, + bundle.get("name")), exc_info=1) + + for indep in self.config.findall('.//Independent'): + for tool in self.tools: + try: + self.states.update(tool.BundleNotUpdated(indep)) + except: # pylint: disable=W0702 + self.logger.error("%s.BundleNotUpdated(%s:%s) call failed:" + % (tool.name, indep.tag, + indep.get("name")), exc_info=1) + + def Remove(self): + """Remove extra entries.""" + for tool in self.tools: + extras = [entry for entry in self.removal + if tool.handlesEntry(entry)] + if extras: + try: + tool.Remove(extras) + except: # pylint: disable=W0702 + self.logger.error("%s.Remove() failed" % tool.name, + exc_info=1) + + def CondDisplayState(self, phase): + """Conditionally print tracing information.""" + self.logger.info('Phase: %s' % phase) + self.logger.info('Correct entries: %d' % + list(self.states.values()).count(True)) + self.logger.info('Incorrect entries: %d' % + list(self.states.values()).count(False)) + if phase == 'final' and list(self.states.values()).count(False): + for entry in sorted(self.states.keys(), key=lambda e: e.tag + ":" + + e.get('name')): + if not self.states[entry]: + etype = entry.get('type') + if etype: + self.logger.info("%s:%s:%s" % (entry.tag, etype, + entry.get('name'))) + else: + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) + self.logger.info('Total managed entries: %d' % + len(list(self.states.values()))) + self.logger.info('Unmanaged entries: %d' % len(self.extra)) + if phase == 'final' and Bcfg2.Options.setup.show_extra: + for entry in sorted(self.extra, + key=lambda e: e.tag + ":" + e.get('name')): + etype = entry.get('type') + if etype: + self.logger.info("%s:%s:%s" % (entry.tag, etype, + entry.get('name'))) + else: + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) + + if ((list(self.states.values()).count(False) == 0) and not self.extra): + self.logger.info('All entries correct.') + + def ReInventory(self): + """Recheck everything.""" + if not Bcfg2.Options.setup.dry_run and Bcfg2.Options.setup.kevlar: + self.logger.info("Rechecking system inventory") + self.Inventory() + + def Execute(self): + """Run all methods.""" + self.Inventory() + self.times['inventory'] = time.time() + self.CondDisplayState('initial') + self.InstallImportant() + if not Bcfg2.Options.setup.only_important: + self.Decide() + self.Install() + self.times['install'] = time.time() + self.Remove() + self.times['remove'] = time.time() + + if self.modified: + self.ReInventory() + self.times['reinventory'] = time.time() + self.times['finished'] = time.time() + self.CondDisplayState('final') + + def GenerateStats(self): + """Generate XML summary of execution statistics.""" + states = {} + for (item, val) in list(self.states.items()): + if not Bcfg2.Options.setup.only_important or \ + item.get('important', 'false').lower() == 'true': + states[item] = val + + feedback = XML.Element("upload-statistics") + stats = XML.SubElement(feedback, + 'Statistics', total=str(len(states)), + version='2.0', + revision=self.config.get('revision', '-1')) + good_entries = [key for key, val in list(states.items()) if val] + good = len(good_entries) + stats.set('good', str(good)) + if any(not val for val in list(states.values())): + stats.set('state', 'dirty') + else: + stats.set('state', 'clean') + + # List bad elements of the configuration + for (data, ename) in [(self.modified, 'Modified'), + (self.extra, "Extra"), + (good_entries, "Good"), + ([entry for entry in states + if not states[entry]], "Bad")]: + container = XML.SubElement(stats, ename) + for item in data: + item.set('qtext', '') + container.append(item) + item.text = None + + timeinfo = XML.Element("OpStamps") + feedback.append(stats) + for (event, timestamp) in list(self.times.items()): + timeinfo.set(event, str(timestamp)) + stats.append(timeinfo) + return feedback diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py new file mode 100644 index 000000000..b817ecb94 --- /dev/null +++ b/src/lib/Bcfg2/DBSettings.py @@ -0,0 +1,285 @@ +""" Django settings for the Bcfg2 server """ + +import os +import sys +import logging +import Bcfg2.Logger +import Bcfg2.Options + +try: + import django + import django.core.management + import django.conf + HAS_DJANGO = True +except ImportError: + HAS_DJANGO = False + +# required for reporting +try: + import south # pylint: disable=W0611 + HAS_SOUTH = True +except ImportError: + HAS_SOUTH = False + +settings = dict( # pylint: disable=C0103 + TIME_ZONE=None, + TEMPLATE_DEBUG=False, + DEBUG=False, + ALLOWED_HOSTS=['*'], + MEDIA_URL='/site_media/', + MANAGERS=(('Root', 'root'),), + ADMINS=(('Root', 'root'),), + # Language code for this installation. All choices can be found + # here: + # http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes + # http://blogs.law.harvard.edu/tech/stories/storyReader$15 + LANGUAGE_CODE='en-us', + SITE_ID=1, + INSTALLED_APPS=('django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.admin', + 'Bcfg2.Server'), + MEDIA_ROOT='', + STATIC_URL='/media/', + # TODO - make this unique + SECRET_KEY='eb5+y%oy-qx*2+62vv=gtnnxg1yig_odu0se5$h0hh#pc*lmo7', + TEMPLATE_LOADERS=('django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader'), + MIDDLEWARE_CLASSES=( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.middleware.doc.XViewMiddleware'), + ROOT_URLCONF='Bcfg2.Reporting.urls', + AUTHENTICATION_BACKENDS=('django.contrib.auth.backends.ModelBackend'), + LOGIN_URL='/login', + SESSION_EXPIRE_AT_BROWSER_CLOSE=True, + TEMPLATE_DIRS=( + '/usr/share/python-support/python-django/django/contrib/admin/' + 'templates/'), + TEMPLATE_CONTEXT_PROCESSORS=( + 'django.contrib.auth.context_processors.auth', + 'django.core.context_processors.debug', + 'django.core.context_processors.i18n', + 'django.core.context_processors.media', + 'django.core.context_processors.request'), + DATABASE_ROUTERS=['Bcfg2.DBSettings.PerApplicationRouter']) + +if HAS_SOUTH: + settings['INSTALLED_APPS'] += ('south', 'Bcfg2.Reporting') +if 'BCFG2_LEGACY_MODELS' in os.environ: + settings['INSTALLED_APPS'] += ('Bcfg2.Server.Reports.reports',) + +if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 3: + settings['CACHE_BACKEND'] = 'locmem:///' +else: + settings['CACHES'] = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + } + } + + +def finalize_django_config(opts=None, silent=False): + """ Perform final Django configuration """ + if opts is None: + opts = Bcfg2.Options.setup + settings['DATABASES'] = dict( + default=dict( + ENGINE="django.db.backends.%s" % opts.db_engine, + NAME=opts.db_name, + USER=opts.db_user, + PASSWORD=opts.db_password, + HOST=opts.db_host, + PORT=opts.db_port, + OPTIONS=opts.db_opts, + SCHEMA=opts.db_schema)) + + if hasattr(opts, "reporting_db_engine") and \ + opts.reporting_db_engine is not None: + settings['DATABASES']['Reporting'] = dict( + ENGINE="django.db.backends.%s" % opts.reporting_db_engine, + NAME=opts.reporting_db_name, + USER=opts.reporting_db_user, + PASSWORD=opts.reporting_db_password, + HOST=opts.reporting_db_host, + PORT=opts.reporting_db_port, + OPTIONS=opts.reporting_db_opts, + SCHEMA=opts.reporting_db_schema) + + settings['TIME_ZONE'] = opts.timezone + + settings['TEMPLATE_DEBUG'] = settings['DEBUG'] = \ + opts.web_debug + if opts.web_debug: + print("Warning: Setting web_debug to True causes extraordinary " + "memory leaks. Only use this setting if you know what " + "you're doing.") + + if opts.web_prefix: + settings['MEDIA_URL'] = \ + opts.web_prefix.rstrip('/') + \ + settings['MEDIA_URL'] + + logger = logging.getLogger() + + logger.debug("Finalizing Django settings: %s" % settings) + module = sys.modules[__name__] + for name, value in settings.items(): + setattr(module, name, value) + try: + django.conf.settings.configure(**settings) + except RuntimeError: + if not silent: + logger.warning("Failed to finalize Django settings: %s" % + sys.exc_info()[1]) + + +def sync_databases(**kwargs): + """ Synchronize all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Syncing database %s" % (database)) + django.core.management.call_command("syncdb", database=database, + **kwargs) + + +def migrate_databases(**kwargs): + """ Do South migrations on all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Migrating database %s" % (database)) + django.core.management.call_command("migrate", database=database, + **kwargs) + + +def get_db_label(application): + """ Get the name of the database for a given Django "application". The + rule is that if a database with the same name as the application exists, + use it. Otherwise use the default. Returns a string suitible for use as a + key in the Django database settings dict """ + if application in settings['DATABASES']: + return application + + return 'default' + + +class PerApplicationRouter(object): + """ Django database router for redirecting different applications to their + own database """ + + def _db_per_app(self, model, **_): + """ If a database with the same name as the application exists, use it. + Otherwise use the default """ + return get_db_label(model._meta.app_label) # pylint: disable=W0212 + + def db_for_read(self, model, **hints): + """ Called when Django wants to find out what database to read from """ + return self._db_per_app(model, **hints) + + def db_for_write(self, model, **hints): + """ Called when Django wants to find out what database to write to """ + return self._db_per_app(model, **hints) + + def allow_relation(self, obj1, obj2, **_): + """ Called when Django wants to determine what relations to allow. Only + allow relations within an app """ + # pylint: disable=W0212 + return obj1._meta.app_label == obj2._meta.app_label + # pylint: enable=W0212 + + def allow_syncdb(self, *_): + """ Called when Django wants to determine which models to sync to a + given database. Take the cowards way out and sync all models to all + databases to allow for easy migrations. """ + return True + + +class _OptionContainer(object): + """ Container for options loaded at import-time to configure + databases """ + parse_first = True + options = [ + Bcfg2.Options.Common.repository, + Bcfg2.Options.PathOption( + '-W', '--web-config', cf=('reporting', 'config'), + default="/etc/bcfg2-web.conf", + action=Bcfg2.Options.ConfigFileAction, + help='Web interface configuration file'), + # default database options + Bcfg2.Options.Option( + cf=('database', 'engine'), default='sqlite3', + help='Database engine', dest='db_engine'), + Bcfg2.Options.Option( + cf=('database', 'name'), default='<repository>/etc/bcfg2.sqlite', + help="Database name", dest="db_name"), + Bcfg2.Options.Option( + cf=('database', 'user'), help='Database username', dest='db_user'), + Bcfg2.Options.Option( + cf=('database', 'password'), help='Database password', + dest='db_password'), + Bcfg2.Options.Option( + cf=('database', 'host'), help='Database host', dest='db_host'), + Bcfg2.Options.Option( + cf=('database', 'port'), help='Database port', dest='db_port'), + Bcfg2.Options.Option( + cf=('database', 'schema'), help='Database schema', + dest='db_schema', default='public'), + Bcfg2.Options.Option( + cf=('database', 'options'), help='Database options', + dest='db_opts', type=Bcfg2.Options.Types.comma_dict, + default=dict()), + # reporting database options + Bcfg2.Options.Option( + cf=('database', 'reporting_engine'), + help='Reporting database engine', dest='reporting_db_engine'), + Bcfg2.Options.Option( + cf=('database', 'reporting_name'), + default='<repository>/etc/reporting.sqlite', + help="Reporting database name", dest="reporting_db_name"), + Bcfg2.Options.Option( + cf=('database', 'reporting_user'), + help='Reporting database username', dest='reporting_db_user'), + Bcfg2.Options.Option( + cf=('database', 'reporting_password'), + help='Reporting database password', dest='reporting_db_password'), + Bcfg2.Options.Option( + cf=('database', 'reporting_host'), + help='Reporting database host', dest='reporting_db_host'), + Bcfg2.Options.Option( + cf=('database', 'reporting_port'), + help='Reporting database port', dest='reporting_db_port'), + Bcfg2.Options.Option( + cf=('database', 'reporting_schema'), + help='Reporting database schema', dest='reporting_db_schema', + default='public'), + Bcfg2.Options.Option( + cf=('database', 'reporting_options'), + help='Reporting database options', dest='reporting_db_opts', + type=Bcfg2.Options.Types.comma_dict, default=dict()), + # Django options + Bcfg2.Options.Option( + cf=('reporting', 'timezone'), help='Django timezone'), + Bcfg2.Options.BooleanOption( + cf=('reporting', 'web_debug'), help='Django debug'), + Bcfg2.Options.Option( + cf=('reporting', 'web_prefix'), help='Web prefix')] + + @staticmethod + def component_parsed_hook(opts): + """ Finalize the Django config after this component's options + are parsed. """ + finalize_django_config(opts=opts) + + @staticmethod + def options_parsed_hook(): + """ Finalize the Django config after all options are parsed. + This is added in case the DBSettings component isn't added + early enough in option parsing to be parsed in the 'early' + phase. Chances are good that things will break if that + happens, but we do our best to be a good citizen. """ + finalize_django_config(silent=True) + +Bcfg2.Options.get_parser().add_component(_OptionContainer) diff --git a/src/lib/Bcfg2/Encryption.py b/src/lib/Bcfg2/Encryption.py deleted file mode 100755 index 69d40ea37..000000000 --- a/src/lib/Bcfg2/Encryption.py +++ /dev/null @@ -1,228 +0,0 @@ -""" Bcfg2.Encryption provides a number of convenience methods for -handling encryption in Bcfg2. See :ref:`server-encryption` for more -details. """ - -import os -import sys -from M2Crypto import Rand -from M2Crypto.EVP import Cipher, EVPError -from Bcfg2.Compat import StringIO, md5, b64encode, b64decode - -#: Constant representing the encryption operation for -#: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This -#: makes our code more readable. -ENCRYPT = 1 - -#: Constant representing the decryption operation for -#: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This -#: makes our code more readable. -DECRYPT = 0 - -#: Default cipher algorithm. To get a full list of valid algorithms, -#: you can run:: -#: -#: openssl list-cipher-algorithms | grep -v ' => ' | \ -#: tr 'A-Z-' 'a-z_' | sort -u -ALGORITHM = "aes_256_cbc" - -#: Default initialization vector. For best security, you should use a -#: unique IV for each message. :func:`ssl_encrypt` does this in an -#: automated fashion. -IV = r'\0' * 16 - -#: The config file section encryption options and passphrases are -#: stored in -CFG_SECTION = "encryption" - -#: The config option used to store the algorithm -CFG_ALGORITHM = "algorithm" - -#: The config option used to store the decryption strictness -CFG_DECRYPT = "decrypt" - -Rand.rand_seed(os.urandom(1024)) - - -def _cipher_filter(cipher, instr): - """ M2Crypto reads and writes file-like objects, so this uses - StringIO to pass data through it """ - inbuf = StringIO(instr) - outbuf = StringIO() - while 1: - buf = inbuf.read() - if not buf: - break - outbuf.write(cipher.update(buf)) - outbuf.write(cipher.final()) - rv = outbuf.getvalue() - inbuf.close() - outbuf.close() - return rv - - -def str_encrypt(plaintext, key, iv=IV, algorithm=ALGORITHM, salt=None): - """ Encrypt a string with a key. For a higher-level encryption - interface, see :func:`ssl_encrypt`. - - :param plaintext: The plaintext data to encrypt - :type plaintext: string - :param key: The key to encrypt the data with - :type key: string - :param iv: The initialization vector - :type iv: string - :param algorithm: The cipher algorithm to use - :type algorithm: string - :param salt: The salt to use - :type salt: string - :returns: string - The decrypted data - """ - cipher = Cipher(alg=algorithm, key=key, iv=iv, op=ENCRYPT, salt=salt) - return _cipher_filter(cipher, plaintext) - - -def str_decrypt(crypted, key, iv=IV, algorithm=ALGORITHM): - """ Decrypt a string with a key. For a higher-level decryption - interface, see :func:`ssl_decrypt`. - - :param crypted: The raw binary encrypted data - :type crypted: string - :param key: The encryption key to decrypt with - :type key: string - :param iv: The initialization vector - :type iv: string - :param algorithm: The cipher algorithm to use - :type algorithm: string - :returns: string - The decrypted data - """ - cipher = Cipher(alg=algorithm, key=key, iv=iv, op=DECRYPT) - return _cipher_filter(cipher, crypted) - - -def ssl_decrypt(data, passwd, algorithm=ALGORITHM): - """ Decrypt openssl-encrypted data. This can decrypt data - encrypted by :func:`ssl_encrypt`, or ``openssl enc``. It performs - a base64 decode first if the data is base64 encoded, and - automatically determines the salt and initialization vector (both - of which are embedded in the encrypted data). - - :param data: The encrypted data (either base64-encoded or raw - binary) to decrypt - :type data: string - :param passwd: The password to use to decrypt the data - :type passwd: string - :param algorithm: The cipher algorithm to use - :type algorithm: string - :returns: string - The decrypted data - """ - # base64-decode the data - try: - data = b64decode(data) - except TypeError: - # we do not include the data in the error message, because one - # of the common causes of this is data that claims to be - # encrypted but is not. we don't want to include a plaintext - # secret in the error logs. - raise TypeError("Could not decode base64 data: %s" % - sys.exc_info()[1]) - salt = data[8:16] - # pylint: disable=E1101,E1121 - hashes = [md5(passwd + salt).digest()] - for i in range(1, 3): - hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) - # pylint: enable=E1101,E1121 - key = hashes[0] + hashes[1] - iv = hashes[2] - - return str_decrypt(data[16:], key=key, iv=iv, algorithm=algorithm) - - -def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): - """ Encrypt data in a format that is openssl compatible. - - :param plaintext: The plaintext data to encrypt - :type plaintext: string - :param passwd: The password to use to encrypt the data - :type passwd: string - :param algorithm: The cipher algorithm to use - :type algorithm: string - :param salt: The salt to use. If none is provided, one will be - randomly generated. - :type salt: bytes - :returns: string - The base64-encoded, salted, encrypted string. - The string includes a trailing newline to make it fully - compatible with openssl command-line tools. - """ - if salt is None: - salt = Rand.rand_bytes(8) - - # pylint: disable=E1101,E1121 - hashes = [md5(passwd + salt).digest()] - for i in range(1, 3): - hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) - # pylint: enable=E1101,E1121 - key = hashes[0] + hashes[1] - iv = hashes[2] - - crypted = str_encrypt(plaintext, key=key, salt=salt, iv=iv, - algorithm=algorithm) - return b64encode("Salted__" + salt + crypted) + "\n" - - -def get_algorithm(setup): - """ Get the cipher algorithm from the config file. This is used - in case someone uses the OpenSSL algorithm name (e.g., - "AES-256-CBC") instead of the M2Crypto name (e.g., "aes_256_cbc"), - and to handle errors in a sensible way and deduplicate this code. - - :param setup: The Bcfg2 option set to extract passphrases from - :type setup: Bcfg2.Options.OptionParser - :returns: dict - a dict of ``<passphrase name>``: ``<passphrase>`` - """ - return setup.cfp.get(CFG_SECTION, CFG_ALGORITHM, - default=ALGORITHM).lower().replace("-", "_") - - -def get_passphrases(setup): - """ Get all candidate encryption passphrases from the config file. - - :param setup: The Bcfg2 option set to extract passphrases from - :type setup: Bcfg2.Options.OptionParser - :returns: dict - a dict of ``<passphrase name>``: ``<passphrase>`` - """ - section = CFG_SECTION - if setup.cfp.has_section(section): - return dict([(o, setup.cfp.get(section, o)) - for o in setup.cfp.options(section) - if o not in [CFG_ALGORITHM, CFG_DECRYPT]]) - else: - return dict() - - -def bruteforce_decrypt(crypted, passphrases=None, setup=None, - algorithm=ALGORITHM): - """ Convenience method to decrypt the given encrypted string by - trying the given passphrases or all passphrases (as returned by - :func:`get_passphrases`) sequentially until one is found that - works. - - Either ``passphrases`` or ``setup`` must be provided. - - :param crypted: The data to decrypt - :type crypted: string - :param passphrases: The passphrases to try. - :type passphrases: list - :param setup: A Bcfg2 option set to extract passphrases from - :type setup: Bcfg2.Options.OptionParser - :param algorithm: The cipher algorithm to use - :type algorithm: string - :returns: string - The decrypted data - :raises: :class:`M2Crypto.EVP.EVPError`, if the data cannot be decrypted - """ - if passphrases is None: - passphrases = get_passphrases(setup).values() - for passwd in passphrases: - try: - return ssl_decrypt(crypted, passwd, algorithm=algorithm) - except EVPError: - pass - raise EVPError("Failed to decrypt") diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index e537b6148..11eaeebd1 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -9,6 +9,7 @@ import socket import struct import sys import termios +import Bcfg2.Options logging.raiseExceptions = 0 @@ -20,7 +21,7 @@ class TermiosFormatter(logging.Formatter): def __init__(self, fmt=None, datefmt=None): logging.Formatter.__init__(self, fmt, datefmt) - if sys.stdout.isatty(): + if hasattr(sys.stdout, 'isatty') and sys.stdout.isatty(): # now get termios info try: self.width = struct.unpack('hhhh', @@ -150,8 +151,11 @@ def add_console_handler(level=logging.DEBUG): logging.root.addHandler(console) -def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG): +def add_syslog_handler(procname=None, syslog_facility='daemon', + level=logging.DEBUG): """Add a logging handler that logs as procname to syslog_facility.""" + if procname is None: + procname = Bcfg2.Options.get_parser().prog try: try: syslog = FragmentingSysLogHandler(procname, @@ -175,9 +179,9 @@ def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG): print("Failed to activate syslogging") -def add_file_handler(to_file, level=logging.DEBUG): - """Add a logging handler that logs to to_file.""" - filelog = logging.FileHandler(to_file) +def add_file_handler(level=logging.DEBUG): + """Add a logging handler that logs to a file.""" + filelog = logging.FileHandler(Bcfg2.Options.setup.logfile) try: filelog.set_name("file") # pylint: disable=E1101 except AttributeError: @@ -188,34 +192,127 @@ def add_file_handler(to_file, level=logging.DEBUG): logging.root.addHandler(filelog) -def setup_logging(procname, to_console=True, to_syslog=True, - syslog_facility='daemon', level=0, to_file=None): +def default_log_level(): + """ Get the default log level, according to the configuration """ + if Bcfg2.Options.setup.debug: + return logging.DEBUG + elif Bcfg2.Options.setup.verbose: + return logging.INFO + else: + return logging.WARNING + + +def setup_logging(): """Setup logging for Bcfg2 software.""" if hasattr(logging, 'already_setup'): return + level = default_log_level() params = [] + to_console = True + if hasattr(Bcfg2.Options.setup, "daemon"): + if Bcfg2.Options.setup.daemon: + to_console = False + # if a command can be daemonized, but hasn't been, then we + # assume that they're running it in the foreground and thus + # want some more output. + clvl = min(level, logging.INFO) + else: + clvl = level if to_console: - if to_console is True: - to_console = logging.WARNING - if level == 0: - clvl = to_console - else: - clvl = min(to_console, level) params.append("%s to console" % logging.getLevelName(clvl)) - add_console_handler(clvl) - if to_syslog: - if level == 0: - slvl = logging.INFO - else: - slvl = min(level, logging.INFO) + add_console_handler(level=clvl) + + if hasattr(Bcfg2.Options.setup, "syslog") and Bcfg2.Options.setup.syslog: + slvl = min(level, logging.INFO) params.append("%s to syslog" % logging.getLevelName(slvl)) - add_syslog_handler(procname, syslog_facility, level=slvl) - if to_file is not None: - params.append("%s to %s" % (logging.getLevelName(level), to_file)) - add_file_handler(to_file, level=level) + add_syslog_handler(level=slvl) + + if Bcfg2.Options.setup.logfile: + params.append("%s to %s" % (logging.getLevelName(level), + Bcfg2.Options.setup.logfile)) + add_file_handler(level=level) logging.root.setLevel(logging.DEBUG) logging.root.debug("Configured logging: %s" % "; ".join(params)) logging.already_setup = True + + +class Debuggable(object): + """ Mixin to add a debugging interface to an object """ + + options = [] + + #: List of names of methods to be exposed as XML-RPC functions, if + #: applicable to the child class + __rmi__ = ['toggle_debug', 'set_debug'] + + #: How exposed XML-RPC functions should be dispatched to child + #: processes. + __child_rmi__ = __rmi__[:] + + def __init__(self, name=None): + """ + :param name: The name of the logger object to get. If none is + supplied, the full name of the class (including + module) will be used. + :type name: string + """ + if name is None: + name = "%s.%s" % (self.__class__.__module__, + self.__class__.__name__) + self.debug_flag = Bcfg2.Options.setup.debug + self.logger = logging.getLogger(name) + + def set_debug(self, debug): + """ Explicitly enable or disable debugging. + + :returns: bool - The new value of the debug flag + """ + self.debug_flag = debug + return debug + + def toggle_debug(self): + """ Turn debugging output on or off. + + :returns: bool - The new value of the debug flag + """ + return self.set_debug(not self.debug_flag) + + def debug_log(self, message, flag=None): + """ Log a message at the debug level. + + :param message: The message to log + :type message: string + :param flag: Override the current debug flag with this value + :type flag: bool + :returns: None + """ + if (flag is None and self.debug_flag) or flag: + self.logger.error(message) + + +class _OptionContainer(object): + """ Container for options loaded at import-time to configure + logging """ + options = [ + Bcfg2.Options.BooleanOption( + '-d', '--debug', help='Enable debugging output', + cf=('logging', 'debug')), + Bcfg2.Options.BooleanOption( + '-v', '--verbose', help='Enable verbose output', + cf=('logging', 'verbose')), + Bcfg2.Options.PathOption( + '-o', '--logfile', help='Set path of file log', + cf=('logging', 'path'))] + + @staticmethod + def options_parsed_hook(): + """ initialize settings from /etc/bcfg2-web.conf or + /etc/bcfg2.conf, or set up basic defaults. this lets + manage.py work in all cases """ + setup_logging() + + +Bcfg2.Options.get_parser().add_component(_OptionContainer) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py deleted file mode 100644 index 652e216a5..000000000 --- a/src/lib/Bcfg2/Options.py +++ /dev/null @@ -1,1396 +0,0 @@ -"""Option parsing library for utilities.""" - -import ast -import copy -import getopt -import grp -import inspect -import os -import pwd -import re -import shlex -import sys - -import Bcfg2.Client.Tools -from Bcfg2.Compat import ConfigParser -from Bcfg2.version import __version__ - - -class OptionFailure(Exception): - """ raised when malformed Option objects are instantiated """ - pass - -DEFAULT_CONFIG_LOCATION = '/etc/bcfg2.conf' -DEFAULT_INSTALL_PREFIX = '/usr' - - -class DefaultConfigParser(ConfigParser.ConfigParser): - """ A config parser that can be used to query options with default - values in the event that the option is not found """ - - def __init__(self, *args, **kwargs): - """Make configuration options case sensitive""" - ConfigParser.ConfigParser.__init__(self, *args, **kwargs) - self.optionxform = str - - def get(self, section, option, **kwargs): - """ convenience method for getting config items """ - default = None - if 'default' in kwargs: - default = kwargs['default'] - del kwargs['default'] - try: - return ConfigParser.ConfigParser.get(self, section, option, - **kwargs) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - if default is not None: - return default - else: - raise - - def getboolean(self, section, option, **kwargs): - """ convenience method for getting boolean config items """ - default = None - if 'default' in kwargs: - default = kwargs['default'] - del kwargs['default'] - try: - return ConfigParser.ConfigParser.getboolean(self, section, - option, **kwargs) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, - ValueError): - if default is not None: - return default - else: - raise - - -class Option(object): - """ a single option, which might be read from the command line, - environment, or config file """ - - # pylint: disable=C0103,R0913 - def __init__(self, desc, default, cmd=None, odesc=False, - env=False, cf=False, cook=False, long_arg=False, - deprecated_cf=None): - self.desc = desc - self.default = default - self.cmd = cmd - self.long = long_arg - if not self.long: - if cmd and (cmd[0] != '-' or len(cmd) != 2): - raise OptionFailure("Poorly formed command %s" % cmd) - elif cmd and not cmd.startswith('--'): - raise OptionFailure("Poorly formed command %s" % cmd) - self.odesc = odesc - self.env = env - self.cf = cf - self.deprecated_cf = deprecated_cf - self.boolean = False - if not odesc and not cook and isinstance(self.default, bool): - self.boolean = True - self.cook = cook - self.value = None - # pylint: enable=C0103,R0913 - - def get_cooked_value(self, value): - """ get the value of this option after performing any option - munging specified in the 'cook' keyword argument to the - constructor """ - if self.boolean: - return True - if self.cook: - return self.cook(value) - else: - return value - - def __str__(self): - rv = ["%s: " % self.__class__.__name__, self.desc] - if self.cmd or self.cf: - rv.append(" (") - if self.cmd: - if self.odesc: - if self.long: - rv.append("%s=%s" % (self.cmd, self.odesc)) - else: - rv.append("%s %s" % (self.cmd, self.odesc)) - else: - rv.append("%s" % self.cmd) - - if self.cf: - if self.cmd: - rv.append("; ") - rv.append("[%s].%s" % self.cf) - if self.cmd or self.cf: - rv.append(")") - if hasattr(self, "value"): - rv.append(": %s" % self.value) - return "".join(rv) - - def buildHelpMessage(self): - """ build the help message for this option """ - vals = [] - if not self.cmd: - return '' - if self.odesc: - if self.long: - vals.append("%s=%s" % (self.cmd, self.odesc)) - else: - vals.append("%s %s" % (self.cmd, self.odesc)) - else: - vals.append(self.cmd) - vals.append(self.desc) - return " %-28s %s\n" % tuple(vals) - - def buildGetopt(self): - """ build a string suitable for describing this short option - to getopt """ - gstr = '' - if self.long: - return gstr - if self.cmd: - gstr = self.cmd[1] - if self.odesc: - gstr += ':' - return gstr - - def buildLongGetopt(self): - """ build a string suitable for describing this long option to - getopt """ - if self.odesc: - return self.cmd[2:] + '=' - else: - return self.cmd[2:] - - def parse(self, opts, rawopts, configparser=None): - """ parse a single option. try parsing the data out of opts - (the results of getopt), rawopts (the raw option string), the - environment, and finally the config parser. either opts or - rawopts should be provided, but not both """ - if self.cmd and opts: - # Processing getopted data - optinfo = [opt[1] for opt in opts if opt[0] == self.cmd] - if optinfo: - if optinfo[0]: - self.value = self.get_cooked_value(optinfo[0]) - else: - self.value = True - return - if self.cmd and self.cmd in rawopts: - if self.odesc: - data = rawopts[rawopts.index(self.cmd) + 1] - else: - data = True - self.value = self.get_cooked_value(data) - return - # No command line option found - if self.env and self.env in os.environ: - self.value = self.get_cooked_value(os.environ[self.env]) - return - if self.cf and configparser: - try: - self.value = self.get_cooked_value(configparser.get(*self.cf)) - return - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - pass - if self.deprecated_cf: - try: - self.value = self.get_cooked_value( - configparser.get(*self.deprecated_cf)) - print("Warning: [%s] %s is deprecated, use [%s] %s instead" - % (self.deprecated_cf[0], self.deprecated_cf[1], - self.cf[0], self.cf[1])) - return - except (ConfigParser.NoSectionError, - ConfigParser.NoOptionError): - pass - - # Default value not cooked - self.value = self.default - - -class OptionSet(dict): - """ a set of Option objects that interfaces with getopt and - DefaultConfigParser to populate a dict of <option name>:<value> - """ - - def __init__(self, *args, **kwargs): - dict.__init__(self, *args) - self.hm = self.buildHelpMessage() # pylint: disable=C0103 - if 'configfile' in kwargs: - self.cfile = kwargs['configfile'] - else: - self.cfile = DEFAULT_CONFIG_LOCATION - if 'quiet' in kwargs: - self.quiet = kwargs['quiet'] - else: - self.quiet = False - self.cfp = DefaultConfigParser() - if len(self.cfp.read(self.cfile)) == 0 and not self.quiet: - # suppress warnings if called from bcfg2-admin init - caller = inspect.stack()[-1][1].split('/')[-1] - if caller == 'bcfg2-admin' and len(sys.argv) > 1: - if sys.argv[1] == 'init': - return - else: - print("Warning! Unable to read specified configuration file: " - "%s" % self.cfile) - - def buildGetopt(self): - """ build a short option description string suitable for use - by getopt.getopt """ - return ''.join([opt.buildGetopt() for opt in list(self.values())]) - - def buildLongGetopt(self): - """ build a list of long options suitable for use by - getopt.getopt """ - return [opt.buildLongGetopt() for opt in list(self.values()) - if opt.long] - - def buildHelpMessage(self): - """ Build the help mesage for this option set, or use self.hm - if it is set """ - if hasattr(self, 'hm'): - return self.hm - hlist = [] # list of _non-empty_ help messages - for opt in list(self.values()): - helpmsg = opt.buildHelpMessage() - if helpmsg: - hlist.append(helpmsg) - return ''.join(hlist) - - def helpExit(self, msg='', code=1): - """ print help and exit """ - if msg: - print(msg) - print("Usage:") - print(self.buildHelpMessage()) - raise SystemExit(code) - - def versionExit(self, code=0): - """ print the version of bcfg2 and exit """ - print("%s %s on Python %s" % - (os.path.basename(sys.argv[0]), - __version__, - ".".join(str(v) for v in sys.version_info[0:3]))) - raise SystemExit(code) - - def parse(self, argv, do_getopt=True): - '''Parse options from command line.''' - if VERSION not in self.values(): - self['__version__'] = VERSION - if do_getopt: - try: - opts, args = getopt.getopt(argv, self.buildGetopt(), - self.buildLongGetopt()) - except getopt.GetoptError: - err = sys.exc_info()[1] - self.helpExit(err) - if '-h' in argv: - self.helpExit('', 0) - if '--version' in argv: - self.versionExit() - self['args'] = args - for key in list(self.keys()): - if key == 'args': - continue - option = self[key] - if do_getopt: - option.parse(opts, [], configparser=self.cfp) - else: - option.parse([], argv, configparser=self.cfp) - if hasattr(option, 'value'): - val = option.value - self[key] = val - if "__version__" in self: - del self['__version__'] - - -def list_split(c_string): - """ split an option string on commas, optionally surrounded by - whitespace, returning a list """ - if c_string: - return re.split(r'\s*,\s*', c_string) - return [] - - -def list_split_anchored_regex(c_string): - """ like list_split but split on whitespace and compile each element as - anchored regex """ - try: - return [re.compile('^' + x + '$') for x in re.split(r'\s+', c_string)] - except re.error: - raise ValueError("Not a list of regexes", c_string) - - -def colon_split(c_string): - """ split an option string on colons, returning a list """ - if c_string: - return c_string.split(r':') - return [] - - -def dict_split(c_string): - """ literally evaluate the option in order to allow for arbitrarily nested - dictionaries """ - return ast.literal_eval(c_string) - - -def get_bool(val): - """ given a string value of a boolean configuration option, return - an actual bool (True or False) """ - # these values copied from ConfigParser.RawConfigParser.getboolean - # with the addition of True and False - truelist = ["1", "yes", "True", "true", "on"] - falselist = ["0", "no", "False", "false", "off"] - if val in truelist: - return True - elif val in falselist: - return False - else: - raise ValueError("Not a boolean value", val) - - -def get_int(val): - """ given a string value of an integer configuration option, - return an actual int """ - return int(val) - - -def get_timeout(val): - """ convert the timeout value into a float or None """ - if val is None: - return val - timeout = float(val) # pass ValueError up the stack - if timeout <= 0: - return None - return timeout - - -def get_size(value): - """ Given a number of bytes in a human-readable format (e.g., - '512m', '2g'), get the absolute number of bytes as an integer """ - if value == -1: - return value - mat = re.match(r'(\d+)([KkMmGg])?', value) - if not mat: - raise ValueError("Not a valid size", value) - rvalue = int(mat.group(1)) - mult = mat.group(2).lower() - if mult == 'k': - return rvalue * 1024 - elif mult == 'm': - return rvalue * 1024 * 1024 - elif mult == 'g': - return rvalue * 1024 * 1024 * 1024 - else: - return rvalue - - -def get_gid(val): - """ This takes a group name or gid and returns the corresponding - gid. """ - try: - return int(val) - except ValueError: - return int(grp.getgrnam(val)[2]) - - -def get_uid(val): - """ This takes a group name or gid and returns the corresponding - gid. """ - try: - return int(val) - except ValueError: - return int(pwd.getpwnam(val)[2]) - - -# Options accepts keyword argument list with the following values: -# default: default value for the option -# cmd: command line switch -# odesc: option description -# cf: tuple containing section/option -# cook: method for parsing option -# long_arg: (True|False) specifies whether cmd is a long argument - -# General options -CFILE = \ - Option('Specify configuration file', - default=DEFAULT_CONFIG_LOCATION, - cmd='-C', - odesc='<conffile>', - env="BCFG2_CONFIG") -LOCKFILE = \ - Option('Specify lockfile', - default='/var/lock/bcfg2.run', - odesc='<Path to lockfile>', - cf=('components', 'lockfile')) -HELP = \ - Option('Print this usage message', - default=False, - cmd='-h') -VERSION = \ - Option('Print the version and exit', - default=False, - cmd='--version', long_arg=True) -DAEMON = \ - Option("Daemonize process, storing pid", - default=None, - cmd='-D', - odesc='<pidfile>') -INSTALL_PREFIX = \ - Option('Installation location', - default=DEFAULT_INSTALL_PREFIX, - odesc='</path>', - cf=('server', 'prefix')) -SENDMAIL_PATH = \ - Option('Path to sendmail', - default='/usr/lib/sendmail', - cf=('reports', 'sendmailpath')) -INTERACTIVE = \ - Option('Run interactively, prompting the user for each change', - default=False, - cmd='-I', ) -ENCODING = \ - Option('Encoding of cfg files', - default='UTF-8', - cmd='-E', - odesc='<encoding>', - cf=('components', 'encoding')) -PARANOID_PATH = \ - Option('Specify path for paranoid file backups', - default='/var/cache/bcfg2', - odesc='<paranoid backup path>', - cf=('paranoid', 'path')) -PARANOID_MAX_COPIES = \ - Option('Specify the number of paranoid copies you want', - default=1, - odesc='<max paranoid copies>', - cf=('paranoid', 'max_copies')) -OMIT_LOCK_CHECK = \ - Option('Omit lock check', - default=False, - cmd='-O') -CORE_PROFILE = \ - Option('profile', - default=False, - cmd='-p') -SCHEMA_PATH = \ - Option('Path to XML Schema files', - default='%s/share/bcfg2/schemas' % DEFAULT_INSTALL_PREFIX, - cmd='--schema', - odesc='<schema path>', - cf=('lint', 'schema'), - long_arg=True) -INTERPRETER = \ - Option("Python interpreter to use", - default='best', - cmd="--interpreter", - odesc='<python|bpython|ipython|best>', - cf=('bcfg2-info', 'interpreter'), - long_arg=True) - -# Metadata options (mdata section) -MDATA_OWNER = \ - Option('Default Path owner', - default='root', - odesc='owner permissions', - cf=('mdata', 'owner')) -MDATA_GROUP = \ - Option('Default Path group', - default='root', - odesc='group permissions', - cf=('mdata', 'group')) -MDATA_IMPORTANT = \ - Option('Default Path priority (importance)', - default='False', - odesc='Important entries are installed first', - cf=('mdata', 'important')) -MDATA_MODE = \ - Option('Default mode for Path', - default='644', - odesc='octal file mode', - cf=('mdata', 'mode')) -MDATA_SECONTEXT = \ - Option('Default SELinux context', - default='__default__', - odesc='SELinux context', - cf=('mdata', 'secontext')) -MDATA_PARANOID = \ - Option('Default Path paranoid setting', - default='true', - odesc='Path paranoid setting', - cf=('mdata', 'paranoid')) -MDATA_SENSITIVE = \ - Option('Default Path sensitive setting', - default='false', - odesc='Path sensitive setting', - cf=('mdata', 'sensitive')) - -# Server options -SERVER_REPOSITORY = \ - Option('Server repository path', - default='/var/lib/bcfg2', - cmd='-Q', - odesc='<repository path>', - cf=('server', 'repository')) -SERVER_PLUGINS = \ - Option('Server plugin list', - # default server plugins - default=['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', 'Rules', - 'SSHbase'], - cf=('server', 'plugins'), - cook=list_split) -SERVER_FILEMONITOR = \ - Option('Server file monitor', - default='default', - odesc='File monitoring driver', - cf=('server', 'filemonitor')) -SERVER_FAM_IGNORE = \ - Option('File globs to ignore', - default=['*~', '*#', '.#*', '*.swp', '*.swpx', '.*.swx', - 'SCCS', '.svn', '4913', '.gitignore'], - cf=('server', 'ignore_files'), - cook=list_split) -SERVER_FAM_BLOCK = \ - Option('FAM blocks on startup until all events are processed', - default=False, - cook=get_bool, - cf=('server', 'fam_blocking')) -SERVER_LISTEN_ALL = \ - Option('Listen on all interfaces', - default=False, - cmd='--listen-all', - cf=('server', 'listen_all'), - cook=get_bool, - long_arg=True) -SERVER_LOCATION = \ - Option('Server Location', - default='https://localhost:6789', - cmd='-S', - odesc='https://server:port', - cf=('components', 'bcfg2')) -SERVER_KEY = \ - Option('Path to SSL key', - default="/etc/pki/tls/private/bcfg2.key", - cmd='--ssl-key', - odesc='<ssl key>', - cf=('communication', 'key'), - long_arg=True) -SERVER_CERT = \ - Option('Path to SSL certificate', - default="/etc/pki/tls/certs/bcfg2.crt", - odesc='<ssl cert>', - cf=('communication', 'certificate')) -SERVER_CA = \ - Option('Path to SSL CA Cert', - default=None, - odesc='<ca cert>', - cf=('communication', 'ca')) -SERVER_PASSWORD = \ - Option('Communication Password', - default=None, - cmd='-x', - odesc='<password>', - cf=('communication', 'password')) -SERVER_PROTOCOL = \ - Option('Server Protocol', - default='xmlrpc/ssl', - cf=('communication', 'protocol')) -SERVER_BACKEND = \ - Option('Server Backend', - default='best', - cf=('server', 'backend')) -SERVER_DAEMON_USER = \ - Option('User to run the server daemon as', - default=0, - cf=('server', 'user'), - cook=get_uid) -SERVER_DAEMON_GROUP = \ - Option('Group to run the server daemon as', - default=0, - cf=('server', 'group'), - cook=get_gid) -SERVER_VCS_ROOT = \ - Option('Server VCS repository root', - default=None, - odesc='<VCS repository root>', - cf=('server', 'vcs_root')) -SERVER_UMASK = \ - Option('Server umask', - default='0077', - odesc='<Server umask>', - cf=('server', 'umask')) -SERVER_AUTHENTICATION = \ - Option('Default client authentication method', - default='cert+password', - odesc='{cert|bootstrap|cert+password}', - cf=('communication', 'authentication')) -SERVER_CHILDREN = \ - Option('Spawn this number of children for the multiprocessing core. ' - 'By default spawns children equivalent to the number of processors ' - 'in the machine.', - default=None, - cmd='--children', - odesc='<children>', - cf=('server', 'children'), - cook=get_int, - long_arg=True) -SERVER_PROBE_ALLOWED_GROUPS = \ - Option('Whitespace-separated list of group names (as regex) to which ' - 'probes can assign a client by writing "group:" to stdout.', - default=[re.compile('.*')], - cf=('probes', 'allowed_groups'), - cook=list_split_anchored_regex) - -# database options -DB_ENGINE = \ - Option('Database engine', - default='sqlite3', - cf=('database', 'engine'), - deprecated_cf=('statistics', 'database_engine')) -DB_NAME = \ - Option('Database name', - default=os.path.join(SERVER_REPOSITORY.default, "etc/bcfg2.sqlite"), - cf=('database', 'name'), - deprecated_cf=('statistics', 'database_name')) -DB_USER = \ - Option('Database username', - default=None, - cf=('database', 'user'), - deprecated_cf=('statistics', 'database_user')) -DB_PASSWORD = \ - Option('Database password', - default=None, - cf=('database', 'password'), - deprecated_cf=('statistics', 'database_password')) -DB_HOST = \ - Option('Database host', - default='localhost', - cf=('database', 'host'), - deprecated_cf=('statistics', 'database_host')) -DB_PORT = \ - Option('Database port', - default='', - cf=('database', 'port'), - deprecated_cf=('statistics', 'database_port')) -DB_OPTIONS = \ - Option('Database options', - default=dict(), - cf=('database', 'options'), - cook=dict_split) -DB_SCHEMA = \ - Option('Database schema', - default='public', - cf=('database', 'schema')) - -# Django options -WEB_CFILE = \ - Option('Web interface configuration file', - default="/etc/bcfg2-web.conf", - cmd='-W', - odesc='<conffile>', - cf=('reporting', 'config'), - deprecated_cf=('statistics', 'web_prefix'),) -DJANGO_TIME_ZONE = \ - Option('Django timezone', - default=None, - cf=('reporting', 'time_zone'), - deprecated_cf=('statistics', 'web_prefix'),) -DJANGO_DEBUG = \ - Option('Django debug', - default=None, - cf=('reporting', 'web_debug'), - deprecated_cf=('statistics', 'web_prefix'), - cook=get_bool,) -DJANGO_WEB_PREFIX = \ - Option('Web prefix', - default=None, - cf=('reporting', 'web_prefix'), - deprecated_cf=('statistics', 'web_prefix'),) - -# Reporting options -REPORTING_FILE_LIMIT = \ - Option('Reporting file size limit', - default=get_size('1m'), - cf=('reporting', 'file_limit'), - cook=get_size,) - -# Reporting options -REPORTING_TRANSPORT = \ - Option('Reporting transport', - default='DirectStore', - cf=('reporting', 'transport'),) - -# Client options -CLIENT_KEY = \ - Option('Path to SSL key', - default=None, - cmd='--ssl-key', - odesc='<ssl key>', - cf=('communication', 'key'), - long_arg=True) -CLIENT_CERT = \ - Option('Path to SSL certificate', - default=None, - cmd='--ssl-cert', - odesc='<ssl cert>', - cf=('communication', 'certificate'), - long_arg=True) -CLIENT_CA = \ - Option('Path to SSL CA Cert', - default=None, - cmd='--ca-cert', - odesc='<ca cert>', - cf=('communication', 'ca'), - long_arg=True) -CLIENT_SCNS = \ - Option('List of server commonNames', - default=None, - cmd='--ssl-cns', - odesc='<CN1:CN2>', - cf=('communication', 'serverCommonNames'), - cook=list_split, - long_arg=True) -CLIENT_PROFILE = \ - Option('Assert the given profile for the host', - default=None, - cmd='-p', - odesc='<profile>', - cf=('client', 'profile')) -CLIENT_RETRIES = \ - Option('The number of times to retry network communication', - default='3', - cmd='-R', - odesc='<retry count>', - cf=('communication', 'retries')) -CLIENT_RETRY_DELAY = \ - Option('The time in seconds to wait between retries', - default='1', - cmd='-y', - odesc='<retry delay>', - cf=('communication', 'retry_delay')) -CLIENT_DRYRUN = \ - Option('Do not actually change the system', - default=False, - cmd='-n') -CLIENT_EXTRA_DISPLAY = \ - Option('enable extra entry output', - default=False, - cmd='-e') -CLIENT_PARANOID = \ - Option('Make automatic backups of config files', - default=False, - cmd='-P', - cf=('client', 'paranoid'), - cook=get_bool) -CLIENT_DRIVERS = \ - Option('Specify tool driver set', - default=Bcfg2.Client.Tools.default, - cmd='-D', - odesc='<driver1,driver2>', - cf=('client', 'drivers'), - cook=list_split) -CLIENT_CACHE = \ - Option('Store the configuration in a file', - default=None, - cmd='-c', - odesc='<cache path>') -CLIENT_REMOVE = \ - Option('Force removal of additional configuration items', - default=None, - cmd='-r', - odesc='<entry type|all>') -CLIENT_BUNDLE = \ - Option('Only configure the given bundle(s)', - default=[], - cmd='-b', - odesc='<bundle:bundle>', - cook=colon_split) -CLIENT_SKIPBUNDLE = \ - Option('Configure everything except the given bundle(s)', - default=[], - cmd='-B', - odesc='<bundle:bundle>', - cook=colon_split) -CLIENT_BUNDLEQUICK = \ - Option('Only verify/configure the given bundle(s)', - default=False, - cmd='-Q') -CLIENT_INDEP = \ - Option('Only configure independent entries, ignore bundles', - default=False, - cmd='-z') -CLIENT_SKIPINDEP = \ - Option('Do not configure independent entries', - default=False, - cmd='-Z') -CLIENT_KEVLAR = \ - Option('Run in kevlar (bulletproof) mode', - default=False, - cmd='-k', ) -CLIENT_FILE = \ - Option('Configure from a file rather than querying the server', - default=None, - cmd='-f', - odesc='<specification path>') -CLIENT_QUICK = \ - Option('Disable some checksum verification', - default=False, - cmd='-q') -CLIENT_USER = \ - Option('The user to provide for authentication', - default='root', - cmd='-u', - odesc='<user>', - cf=('communication', 'user')) -CLIENT_SERVICE_MODE = \ - Option('Set client service mode', - default='default', - cmd='-s', - odesc='<default|disabled|build>') -CLIENT_TIMEOUT = \ - Option('Set the client XML-RPC timeout', - default=90, - cmd='-t', - odesc='<timeout>', - cf=('communication', 'timeout')) -CLIENT_DLIST = \ - Option('Run client in server decision list mode', - default='none', - cmd='-l', - odesc='<whitelist|blacklist|none>', - cf=('client', 'decision')) -CLIENT_DECISION_LIST = \ - Option('Decision List', - default=False, - cmd='--decision-list', - odesc='<file>', - long_arg=True) -CLIENT_EXIT_ON_PROBE_FAILURE = \ - Option("The client should exit if a probe fails", - default=True, - cmd='--exit-on-probe-failure', - long_arg=True, - cf=('client', 'exit_on_probe_failure'), - cook=get_bool) -CLIENT_PROBE_TIMEOUT = \ - Option("Timeout when running client probes", - default=None, - cf=('client', 'probe_timeout'), - cook=get_timeout) -CLIENT_COMMAND_TIMEOUT = \ - Option("Timeout when client runs other external commands (not probes)", - default=None, - cf=('client', 'command_timeout'), - cook=get_timeout) - -# bcfg2-test and bcfg2-lint options -TEST_NOSEOPTS = \ - Option('Options to pass to nosetests. Only honored with --children 0', - default=[], - cmd='--nose-options', - odesc='<opts>', - cf=('bcfg2_test', 'nose_options'), - cook=shlex.split, - long_arg=True) -TEST_IGNORE = \ - Option('Ignore these entries if they fail to build.', - default=[], - cmd='--ignore', - odesc='<Type>:<name>,<Type>:<name>', - cf=('bcfg2_test', 'ignore_entries'), - cook=list_split, - long_arg=True) -TEST_CHILDREN = \ - Option('Spawn this number of children for bcfg2-test (python 2.6+)', - default=0, - cmd='--children', - odesc='<children>', - cf=('bcfg2_test', 'children'), - cook=get_int, - long_arg=True) -TEST_XUNIT = \ - Option('Output an XUnit result file with --children', - default=None, - cmd='--xunit', - odesc='<xunit file>', - cf=('bcfg2_test', 'xunit'), - long_arg=True) -LINT_CONFIG = \ - Option('Specify bcfg2-lint configuration file', - default='/etc/bcfg2-lint.conf', - cmd='--lint-config', - odesc='<conffile>', - long_arg=True) -LINT_PLUGINS = \ - Option('bcfg2-lint plugin list', - default=None, # default is Bcfg2.Server.Lint.__all__ - cf=('lint', 'plugins'), - cook=list_split) -LINT_SHOW_ERRORS = \ - Option('Show error handling', - default=False, - cmd='--list-errors', - long_arg=True) -LINT_FILES_ON_STDIN = \ - Option('Operate on a list of files supplied on stdin', - default=False, - cmd='--stdin', - long_arg=True) - -# individual client tool options -CLIENT_APT_TOOLS_INSTALL_PATH = \ - Option('Apt tools install path', - default='/usr', - cf=('APT', 'install_path')) -CLIENT_APT_TOOLS_VAR_PATH = \ - Option('Apt tools var path', - default='/var', - cf=('APT', 'var_path')) -CLIENT_SYSTEM_ETC_PATH = \ - Option('System etc path', - default='/etc', - cf=('APT', 'etc_path')) -CLIENT_PORTAGE_BINPKGONLY = \ - Option('Portage binary packages only', - default=False, - cf=('Portage', 'binpkgonly'), - cook=get_bool) -CLIENT_RPM_INSTALLONLY = \ - Option('RPM install-only packages', - default=['kernel', 'kernel-bigmem', 'kernel-enterprise', - 'kernel-smp', 'kernel-modules', 'kernel-debug', - 'kernel-unsupported', 'kernel-devel', 'kernel-source', - 'kernel-default', 'kernel-largesmp-devel', - 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], - cf=('RPM', 'installonlypackages'), - deprecated_cf=('RPMng', 'installonlypackages'), - cook=list_split) -CLIENT_RPM_PKG_CHECKS = \ - Option("Perform RPM package checks", - default=True, - cf=('RPM', 'pkg_checks'), - deprecated_cf=('RPMng', 'pkg_checks'), - cook=get_bool) -CLIENT_RPM_PKG_VERIFY = \ - Option("Perform RPM package verify", - default=True, - cf=('RPM', 'pkg_verify'), - deprecated_cf=('RPMng', 'pkg_verify'), - cook=get_bool) -CLIENT_RPM_INSTALLED_ACTION = \ - Option("RPM installed action", - default="install", - cf=('RPM', 'installed_action'), - deprecated_cf=('RPMng', 'installed_action')) -CLIENT_RPM_ERASE_FLAGS = \ - Option("RPM erase flags", - default=["allmatches"], - cf=('RPM', 'erase_flags'), - deprecated_cf=('RPMng', 'erase_flags'), - cook=list_split) -CLIENT_RPM_VERSION_FAIL_ACTION = \ - Option("RPM version fail action", - default="upgrade", - cf=('RPM', 'version_fail_action'), - deprecated_cf=('RPMng', 'version_fail_action')) -CLIENT_RPM_VERIFY_FAIL_ACTION = \ - Option("RPM verify fail action", - default="reinstall", - cf=('RPM', 'verify_fail_action'), - deprecated_cf=('RPMng', 'verify_fail_action')) -CLIENT_RPM_VERIFY_FLAGS = \ - Option("RPM verify flags", - default=[], - cf=('RPM', 'verify_flags'), - deprecated_cf=('RPMng', 'verify_flags'), - cook=list_split) -CLIENT_YUM24_INSTALLONLY = \ - Option('YUM24 install-only packages', - default=['kernel', 'kernel-bigmem', 'kernel-enterprise', - 'kernel-smp', 'kernel-modules', 'kernel-debug', - 'kernel-unsupported', 'kernel-devel', 'kernel-source', - 'kernel-default', 'kernel-largesmp-devel', - 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], - cf=('YUM24', 'installonlypackages'), - cook=list_split) -CLIENT_YUM24_PKG_CHECKS = \ - Option("Perform YUM24 package checks", - default=True, - cf=('YUM24', 'pkg_checks'), - cook=get_bool) -CLIENT_YUM24_PKG_VERIFY = \ - Option("Perform YUM24 package verify", - default=True, - cf=('YUM24', 'pkg_verify'), - cook=get_bool) -CLIENT_YUM24_INSTALLED_ACTION = \ - Option("YUM24 installed action", - default="install", - cf=('YUM24', 'installed_action')) -CLIENT_YUM24_ERASE_FLAGS = \ - Option("YUM24 erase flags", - default=["allmatches"], - cf=('YUM24', 'erase_flags'), - cook=list_split) -CLIENT_YUM24_VERSION_FAIL_ACTION = \ - Option("YUM24 version fail action", - cf=('YUM24', 'version_fail_action'), - default="upgrade") -CLIENT_YUM24_VERIFY_FAIL_ACTION = \ - Option("YUM24 verify fail action", - default="reinstall", - cf=('YUM24', 'verify_fail_action')) -CLIENT_YUM24_VERIFY_FLAGS = \ - Option("YUM24 verify flags", - default=[], - cf=('YUM24', 'verify_flags'), - cook=list_split) -CLIENT_YUM24_AUTODEP = \ - Option("YUM24 autodependency processing", - default=True, - cf=('YUM24', 'autodep'), - cook=get_bool) -CLIENT_YUM_PKG_CHECKS = \ - Option("Perform YUM package checks", - default=True, - cf=('YUM', 'pkg_checks'), - deprecated_cf=('YUMng', 'pkg_checks'), - cook=get_bool) -CLIENT_YUM_PKG_VERIFY = \ - Option("Perform YUM package verify", - default=True, - cf=('YUM', 'pkg_verify'), - deprecated_cf=('YUMng', 'pkg_verify'), - cook=get_bool) -CLIENT_YUM_INSTALLED_ACTION = \ - Option("YUM installed action", - default="install", - cf=('YUM', 'installed_action'), - deprecated_cf=('YUMng', 'installed_action')) -CLIENT_YUM_VERSION_FAIL_ACTION = \ - Option("YUM version fail action", - default="upgrade", - cf=('YUM', 'version_fail_action'), - deprecated_cf=('YUMng', 'version_fail_action')) -CLIENT_YUM_VERIFY_FAIL_ACTION = \ - Option("YUM verify fail action", - default="reinstall", - cf=('YUM', 'verify_fail_action'), - deprecated_cf=('YUMng', 'verify_fail_action')) -CLIENT_YUM_VERIFY_FLAGS = \ - Option("YUM verify flags", - default=[], - cf=('YUM', 'verify_flags'), - deprecated_cf=('YUMng', 'verify_flags'), - cook=list_split) -CLIENT_YUM_DISABLED_PLUGINS = \ - Option("YUM disabled plugins", - default=[], - cf=('YUM', 'disabled_plugins'), - cook=list_split) -CLIENT_YUM_ENABLED_PLUGINS = \ - Option("YUM enabled plugins", - default=[], - cf=('YUM', 'enabled_plugins'), - cook=list_split) -CLIENT_POSIX_UID_WHITELIST = \ - Option("UID ranges the POSIXUsers tool will manage", - default=[], - cf=('POSIXUsers', 'uid_whitelist'), - cook=list_split) -CLIENT_POSIX_GID_WHITELIST = \ - Option("GID ranges the POSIXUsers tool will manage", - default=[], - cf=('POSIXUsers', 'gid_whitelist'), - cook=list_split) -CLIENT_POSIX_UID_BLACKLIST = \ - Option("UID ranges the POSIXUsers tool will not manage", - default=[], - cf=('POSIXUsers', 'uid_blacklist'), - cook=list_split) -CLIENT_POSIX_GID_BLACKLIST = \ - Option("GID ranges the POSIXUsers tool will not manage", - default=[], - cf=('POSIXUsers', 'gid_blacklist'), - cook=list_split) - -# Logging options -LOGGING_FILE_PATH = \ - Option('Set path of file log', - default=None, - cmd='-o', - odesc='<path>', - cf=('logging', 'path')) -LOGGING_SYSLOG = \ - Option('Log to syslog', - default=True, - cook=get_bool, - cf=('logging', 'syslog')) -DEBUG = \ - Option("Enable debugging output", - default=False, - cmd='-d', - cook=get_bool, - cf=('logging', 'debug')) -VERBOSE = \ - Option("Enable verbose output", - default=False, - cmd='-v', - cook=get_bool, - cf=('logging', 'verbose')) -LOG_PERFORMANCE = \ - Option("Periodically log performance statistics", - default=False, - cf=('logging', 'performance')) -PERFLOG_INTERVAL = \ - Option("Performance statistics logging interval in seconds", - default=300.0, - cook=get_timeout, - cf=('logging', 'performance_interval')) - -# Plugin-specific options -CFG_VALIDATION = \ - Option('Run validation on Cfg files', - default=True, - cmd='--cfg-validation', - cf=('cfg', 'validation'), - long_arg=True, - cook=get_bool) - -# bcfg2-crypt options -ENCRYPT = \ - Option('Encrypt the specified file', - default=False, - cmd='--encrypt', - long_arg=True) -DECRYPT = \ - Option('Decrypt the specified file', - default=False, - cmd='--decrypt', - long_arg=True) -CRYPT_STDOUT = \ - Option('Decrypt or encrypt the specified file to stdout', - default=False, - cmd='--stdout', - long_arg=True) -CRYPT_PASSPHRASE = \ - Option('Encryption passphrase name', - default=None, - cmd='-p', - odesc='<passphrase>') -CRYPT_XPATH = \ - Option('XPath expression to select elements to encrypt', - default=None, - cmd='--xpath', - odesc='<xpath>', - long_arg=True) -CRYPT_PROPERTIES = \ - Option('Encrypt the specified file as a Properties file', - default=False, - cmd="--properties", - long_arg=True) -CRYPT_CFG = \ - Option('Encrypt the specified file as a Cfg file', - default=False, - cmd="--cfg", - long_arg=True) -CRYPT_REMOVE = \ - Option('Remove the plaintext file after encrypting', - default=False, - cmd="--remove", - long_arg=True) - -# Option groups -CLI_COMMON_OPTIONS = dict(configfile=CFILE, - debug=DEBUG, - help=HELP, - version=VERSION, - verbose=VERBOSE, - encoding=ENCODING, - logging=LOGGING_FILE_PATH, - syslog=LOGGING_SYSLOG) - -DAEMON_COMMON_OPTIONS = dict(daemon=DAEMON, - umask=SERVER_UMASK, - listen_all=SERVER_LISTEN_ALL, - daemon_uid=SERVER_DAEMON_USER, - daemon_gid=SERVER_DAEMON_GROUP) - -SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, - plugins=SERVER_PLUGINS, - password=SERVER_PASSWORD, - filemonitor=SERVER_FILEMONITOR, - ignore=SERVER_FAM_IGNORE, - fam_blocking=SERVER_FAM_BLOCK, - location=SERVER_LOCATION, - key=SERVER_KEY, - cert=SERVER_CERT, - ca=SERVER_CA, - protocol=SERVER_PROTOCOL, - web_configfile=WEB_CFILE, - backend=SERVER_BACKEND, - vcs_root=SERVER_VCS_ROOT, - authentication=SERVER_AUTHENTICATION, - perflog=LOG_PERFORMANCE, - perflog_interval=PERFLOG_INTERVAL, - children=SERVER_CHILDREN, - client_timeout=CLIENT_TIMEOUT, - probe_allowed_groups=SERVER_PROBE_ALLOWED_GROUPS) - -CRYPT_OPTIONS = dict(encrypt=ENCRYPT, - decrypt=DECRYPT, - crypt_stdout=CRYPT_STDOUT, - passphrase=CRYPT_PASSPHRASE, - xpath=CRYPT_XPATH, - properties=CRYPT_PROPERTIES, - cfg=CRYPT_CFG, - remove=CRYPT_REMOVE) - -DRIVER_OPTIONS = \ - dict(apt_install_path=CLIENT_APT_TOOLS_INSTALL_PATH, - apt_var_path=CLIENT_APT_TOOLS_VAR_PATH, - apt_etc_path=CLIENT_SYSTEM_ETC_PATH, - portage_binpkgonly=CLIENT_PORTAGE_BINPKGONLY, - rpm_installonly=CLIENT_RPM_INSTALLONLY, - rpm_pkg_checks=CLIENT_RPM_PKG_CHECKS, - rpm_pkg_verify=CLIENT_RPM_PKG_VERIFY, - rpm_installed_action=CLIENT_RPM_INSTALLED_ACTION, - rpm_erase_flags=CLIENT_RPM_ERASE_FLAGS, - rpm_version_fail_action=CLIENT_RPM_VERSION_FAIL_ACTION, - rpm_verify_fail_action=CLIENT_RPM_VERIFY_FAIL_ACTION, - rpm_verify_flags=CLIENT_RPM_VERIFY_FLAGS, - yum24_installonly=CLIENT_YUM24_INSTALLONLY, - yum24_pkg_checks=CLIENT_YUM24_PKG_CHECKS, - yum24_pkg_verify=CLIENT_YUM24_PKG_VERIFY, - yum24_installed_action=CLIENT_YUM24_INSTALLED_ACTION, - yum24_erase_flags=CLIENT_YUM24_ERASE_FLAGS, - yum24_version_fail_action=CLIENT_YUM24_VERSION_FAIL_ACTION, - yum24_verify_fail_action=CLIENT_YUM24_VERIFY_FAIL_ACTION, - yum24_verify_flags=CLIENT_YUM24_VERIFY_FLAGS, - yum24_autodep=CLIENT_YUM24_AUTODEP, - yum_pkg_checks=CLIENT_YUM_PKG_CHECKS, - yum_pkg_verify=CLIENT_YUM_PKG_VERIFY, - yum_installed_action=CLIENT_YUM_INSTALLED_ACTION, - yum_version_fail_action=CLIENT_YUM_VERSION_FAIL_ACTION, - yum_verify_fail_action=CLIENT_YUM_VERIFY_FAIL_ACTION, - yum_verify_flags=CLIENT_YUM_VERIFY_FLAGS, - yum_disabled_plugins=CLIENT_YUM_DISABLED_PLUGINS, - yum_enabled_plugins=CLIENT_YUM_ENABLED_PLUGINS, - posix_uid_whitelist=CLIENT_POSIX_UID_WHITELIST, - posix_gid_whitelist=CLIENT_POSIX_GID_WHITELIST, - posix_uid_blacklist=CLIENT_POSIX_UID_BLACKLIST, - posix_gid_blacklist=CLIENT_POSIX_GID_BLACKLIST) - -CLIENT_COMMON_OPTIONS = \ - dict(extra=CLIENT_EXTRA_DISPLAY, - quick=CLIENT_QUICK, - lockfile=LOCKFILE, - drivers=CLIENT_DRIVERS, - dryrun=CLIENT_DRYRUN, - paranoid=CLIENT_PARANOID, - protocol=SERVER_PROTOCOL, - ppath=PARANOID_PATH, - max_copies=PARANOID_MAX_COPIES, - bundle=CLIENT_BUNDLE, - skipbundle=CLIENT_SKIPBUNDLE, - bundle_quick=CLIENT_BUNDLEQUICK, - indep=CLIENT_INDEP, - skipindep=CLIENT_SKIPINDEP, - file=CLIENT_FILE, - interactive=INTERACTIVE, - cache=CLIENT_CACHE, - profile=CLIENT_PROFILE, - remove=CLIENT_REMOVE, - server=SERVER_LOCATION, - user=CLIENT_USER, - password=SERVER_PASSWORD, - retries=CLIENT_RETRIES, - retry_delay=CLIENT_RETRY_DELAY, - kevlar=CLIENT_KEVLAR, - omit_lock_check=OMIT_LOCK_CHECK, - decision=CLIENT_DLIST, - servicemode=CLIENT_SERVICE_MODE, - key=CLIENT_KEY, - certificate=CLIENT_CERT, - ca=CLIENT_CA, - serverCN=CLIENT_SCNS, - timeout=CLIENT_TIMEOUT, - decision_list=CLIENT_DECISION_LIST, - probe_exit=CLIENT_EXIT_ON_PROBE_FAILURE, - probe_timeout=CLIENT_PROBE_TIMEOUT, - command_timeout=CLIENT_COMMAND_TIMEOUT) -CLIENT_COMMON_OPTIONS.update(DRIVER_OPTIONS) -CLIENT_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS) - -DATABASE_COMMON_OPTIONS = dict(web_configfile=WEB_CFILE, - configfile=CFILE, - db_engine=DB_ENGINE, - db_name=DB_NAME, - db_user=DB_USER, - db_password=DB_PASSWORD, - db_host=DB_HOST, - db_port=DB_PORT, - db_options=DB_OPTIONS, - db_schema=DB_SCHEMA, - time_zone=DJANGO_TIME_ZONE, - django_debug=DJANGO_DEBUG, - web_prefix=DJANGO_WEB_PREFIX) - -REPORTING_COMMON_OPTIONS = dict(reporting_file_limit=REPORTING_FILE_LIMIT, - reporting_transport=REPORTING_TRANSPORT) - -TEST_COMMON_OPTIONS = dict(noseopts=TEST_NOSEOPTS, - test_ignore=TEST_IGNORE, - children=TEST_CHILDREN, - xunit=TEST_XUNIT, - validate=CFG_VALIDATION) - -INFO_COMMON_OPTIONS = dict(ppath=PARANOID_PATH, - max_copies=PARANOID_MAX_COPIES) -INFO_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS) -INFO_COMMON_OPTIONS.update(SERVER_COMMON_OPTIONS) - - -class OptionParser(OptionSet): - """ - OptionParser bootstraps option parsing, - getting the value of the config file - """ - def __init__(self, args, argv=None, quiet=False): - if argv is None: - argv = sys.argv[1:] - # the bootstrap is always quiet, since it's running with a - # default config file and so might produce warnings otherwise - self.bootstrap = OptionSet([('configfile', CFILE)], quiet=True) - self.bootstrap.parse(argv, do_getopt=False) - OptionSet.__init__(self, args, configfile=self.bootstrap['configfile'], - quiet=quiet) - self.optinfo = copy.copy(args) - # these will be set by parse() and then used by reparse() - self.argv = [] - self.do_getopt = True - - def reparse(self): - """ parse the options again, taking any changes (e.g., to the - config file) into account """ - for key, opt in self.optinfo.items(): - self[key] = opt - if "args" not in self.optinfo: - del self['args'] - self.parse(self.argv, self.do_getopt) - - def parse(self, argv, do_getopt=True): - self.argv = argv - self.do_getopt = do_getopt - OptionSet.parse(self, self.argv, do_getopt=self.do_getopt) - - def add_option(self, name, opt): - """ Add an option to the parser """ - self[name] = opt - self.optinfo[name] = opt - - def update(self, optdict): - dict.update(self, optdict) - self.optinfo.update(optdict) diff --git a/src/lib/Bcfg2/Options/Actions.py b/src/lib/Bcfg2/Options/Actions.py new file mode 100644 index 000000000..8b941f2bb --- /dev/null +++ b/src/lib/Bcfg2/Options/Actions.py @@ -0,0 +1,179 @@ +""" Custom argparse actions """ + +import sys +import argparse +from Bcfg2.Options.Parser import get_parser + +__all__ = ["ConfigFileAction", "ComponentAction", "PluginsAction"] + + +class FinalizableAction(argparse.Action): + """ A FinalizableAction requires some additional action to be taken + when storing the value, and as a result must be finalized if the + default value is used.""" + + def __init__(self, *args, **kwargs): + argparse.Action.__init__(self, *args, **kwargs) + self._final = False + + def finalize(self, parser, namespace): + """ Finalize a default value by calling the action callable. """ + if not self._final: + self.__call__(parser, namespace, getattr(namespace, self.dest, + self.default)) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + self._final = True + + +class ComponentAction(FinalizableAction): + """ ComponentAction automatically imports classes and modules + based on the value of the option, and automatically collects + options from the loaded classes and modules. It cannot be used by + itself, but must be subclassed, with either :attr:`mapping` or + :attr:`bases` overridden. See + :class:`Bcfg2.Options.PluginsAction` for an example. + + ComponentActions expect to be given a list of class names. If + :attr:`bases` is overridden, then it will attempt to import those + classes from identically named modules within the given bases. + For instance: + + .. code-block:: python + + class FooComponentAction(Bcfg2.Options.ComponentAction): + bases = ["Bcfg2.Server.Foo"] + + + class FooLoader(object): + options = [ + Bcfg2.Options.Option( + "--foo", + type=Bcfg2.Options.Types.comma_list, + default=["One"], + action=FooComponentAction)] + + If "--foo One,Two,Three" were given on the command line, then + ``FooComponentAction`` would attempt to import + ``Bcfg2.Server.Foo.One.One``, ``Bcfg2.Server.Foo.Two.Two``, and + ``Bcfg2.Server.Foo.Three.Three``. (It would also call + :func:`Bcfg2.Options.Parser.add_component` with each of those + classes as arguments.) + + Note that, although ComponentActions expect lists of components + (by default; this can be overridden by setting :attr:`islist`), + you must still explicitly specify a ``type`` argument to the + :class:`Bcfg2.Options.Option` constructor to split the value into + a list. + + Note also that, unlike other actions, the default value of a + ComponentAction option does not need to be the actual literal + final value. (I.e., you don't have to import + ``Bcfg2.Server.Foo.One.One`` and set it as the default in the + example above; the string "One" suffices.) + """ + + #: A list of parent modules where modules or classes should be + #: imported from. + bases = [] + + #: A mapping of ``<name> => <object>`` that components will be + #: loaded from. This can be used to permit much more complex + #: behavior than just a list of :attr:`bases`. + mapping = dict() + + #: If ``module`` is True, then only the module will be loaded, not + #: a class from the module. For instance, in the example above, + #: ``FooComponentAction`` would attempt instead to import + #: ``Bcfg2.Server.Foo.One``, ``Bcfg2.Server.Foo.Two``, and + #: ``Bcfg2.Server.Foo.Three``. + module = False + + #: By default, ComponentActions expect a list of components to + #: load. If ``islist`` is False, then it will only expect a + #: single component. + islist = True + + #: If ``fail_silently`` is True, then failures to import modules + #: or classes will not be logged. This is useful when the default + #: is to import everything, some of which are expected to fail. + fail_silently = False + + def __init__(self, *args, **kwargs): + if self.mapping: + if 'choices' not in kwargs: + kwargs['choices'] = self.mapping.keys() + FinalizableAction.__init__(self, *args, **kwargs) + + def _import(self, module, name): + """ Import the given name from the given module, handling + errors """ + try: + return getattr(__import__(module, fromlist=[name]), name) + except (AttributeError, ImportError): + if not self.fail_silently: + print("Failed to load %s from %s: %s" % + (name, module, sys.exc_info()[1])) + return None + + def _load_component(self, name): + """ Import a single class or module, adding it as a component to + the parser. + + :param name: The name of the class or module to import, without + the base prepended. + :type name: string + :returns: the imported class or module + """ + cls = None + if self.mapping and name in self.mapping: + cls = self.mapping[name] + elif "." in name: + cls = self._import(*name.rsplit(".", 1)) + else: + for base in self.bases: + if self.module: + mod = base + else: + mod = "%s.%s" % (base, name) + cls = self._import(mod, name) + if cls is not None: + break + if cls: + get_parser().add_component(cls) + elif not self.fail_silently: + print("Could not load component %s" % name) + return cls + + def __call__(self, parser, namespace, values, option_string=None): + if values is None: + result = None + else: + if self.islist: + result = [] + for val in values: + cls = self._load_component(val) + if cls is not None: + result.append(cls) + else: + result = self._load_component(values) + FinalizableAction.__call__(self, parser, namespace, result, + option_string=option_string) + + +class ConfigFileAction(FinalizableAction): + """ ConfigFileAction automatically loads and parses a + supplementary config file (e.g., ``bcfg2-web.conf`` or + ``bcfg2-lint.conf``). """ + + def __call__(self, parser, namespace, values, option_string=None): + parser.add_config_file(self.dest, values, reparse=False) + FinalizableAction.__call__(self, parser, namespace, values, + option_string=option_string) + + +class PluginsAction(ComponentAction): + """ :class:`Bcfg2.Options.ComponentAction` subclass for loading + Bcfg2 server plugins. """ + bases = ['Bcfg2.Server.Plugins'] diff --git a/src/lib/Bcfg2/Options/Common.py b/src/lib/Bcfg2/Options/Common.py new file mode 100644 index 000000000..620a7604c --- /dev/null +++ b/src/lib/Bcfg2/Options/Common.py @@ -0,0 +1,129 @@ +""" Common options used in multiple different contexts. """ + +from Bcfg2.Utils import classproperty +from Bcfg2.Options import Types +from Bcfg2.Options.Actions import PluginsAction, ComponentAction +from Bcfg2.Options.Parser import repository as _repository_option +from Bcfg2.Options import Option, PathOption, BooleanOption + +__all__ = ["Common"] + + +class ReportingTransportAction(ComponentAction): + """ :class:`Bcfg2.Options.ComponentAction` that loads a single + reporting transport from :mod:`Bcfg2.Reporting.Transport`. """ + islist = False + bases = ['Bcfg2.Reporting.Transport'] + + +class ReportingStorageAction(ComponentAction): + """ :class:`Bcfg2.Options.ComponentAction` that loads a single + reporting storage driver from :mod:`Bcfg2.Reporting.Storage`. """ + islist = False + bases = ['Bcfg2.Reporting.Storage'] + + +class Common(object): + """ Common options used in multiple different contexts. """ + _plugins = None + _filemonitor = None + _reporting_storage = None + _reporting_transport = None + + @classproperty + def plugins(cls): + """ Load a list of Bcfg2 server plugins """ + if cls._plugins is None: + cls._plugins = Option( + cf=('server', 'plugins'), + type=Types.comma_list, help="Server plugin list", + action=PluginsAction, + default=['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', 'Rules', + 'SSHbase']) + return cls._plugins + + @classproperty + def filemonitor(cls): + """ Load a single Bcfg2 file monitor (from + :attr:`Bcfg2.Server.FileMonitor.available`) """ + if cls._filemonitor is None: + import Bcfg2.Server.FileMonitor + + class FileMonitorAction(ComponentAction): + """ ComponentAction for loading a single FAM backend + class """ + islist = False + mapping = Bcfg2.Server.FileMonitor.available + + cls._filemonitor = Option( + cf=('server', 'filemonitor'), action=FileMonitorAction, + default='default', help='Server file monitoring driver') + return cls._filemonitor + + @classproperty + def reporting_storage(cls): + """ Load a Reporting storage backend """ + if cls._reporting_storage is None: + cls._reporting_storage = Option( + cf=('reporting', 'storage'), dest="reporting_storage", + help='Reporting storage engine', + action=ReportingStorageAction, default='DjangoORM') + return cls._reporting_storage + + @classproperty + def reporting_transport(cls): + """ Load a Reporting transport backend """ + if cls._reporting_transport is None: + cls._reporting_transport = Option( + cf=('reporting', 'transport'), dest="reporting_transport", + help='Reporting transport', + action=ReportingTransportAction, default='DirectStore') + return cls._reporting_transport + + #: Set the path to the Bcfg2 repository + repository = _repository_option + + #: Daemonize process, storing PID + daemon = PathOption( + '-D', '--daemon', help="Daemonize process, storing PID") + + #: Run interactively, prompting the user for each change + interactive = BooleanOption( + "-I", "--interactive", + help='Run interactively, prompting the user for each change') + + #: Log to syslog + syslog = BooleanOption( + cf=('logging', 'syslog'), help="Log to syslog", default=True) + + #: Server location + location = Option( + '-S', '--server', cf=('components', 'bcfg2'), + default='https://localhost:6789', metavar='<https://server:port>', + help="Server location") + + #: Communication password + password = Option( + '-x', '--password', cf=('communication', 'password'), + metavar='<password>', help="Communication Password") + + #: Path to SSL CA certificate + ssl_ca = PathOption( + cf=('communication', 'ca'), help='Path to SSL CA Cert') + + #: Communication protocol + protocol = Option( + cf=('communication', 'protocol'), default='xmlrpc/tlsv1', + choices=['xmlrpc/ssl', 'xmlrpc/tlsv1'], + help='Communication protocol to use.') + + #: Default Path paranoid setting + default_paranoid = Option( + cf=('mdata', 'paranoid'), dest="default_paranoid", default='true', + choices=['true', 'false'], help='Default Path paranoid setting') + + #: Client timeout + client_timeout = Option( + "-t", "--timeout", type=float, default=90.0, dest="client_timeout", + cf=('communication', 'timeout'), + help='Set the client XML-RPC timeout') diff --git a/src/lib/Bcfg2/Options/OptionGroups.py b/src/lib/Bcfg2/Options/OptionGroups.py new file mode 100644 index 000000000..465358fab --- /dev/null +++ b/src/lib/Bcfg2/Options/OptionGroups.py @@ -0,0 +1,204 @@ +""" Option grouping classes """ + +import re +import copy +import fnmatch +from Bcfg2.Options import Option +from itertools import chain + +__all__ = ["OptionGroup", "ExclusiveOptionGroup", "Subparser", + "WildcardSectionGroup"] + + +class OptionContainer(list): + """ Parent class of all option groups """ + + def list_options(self): + """ Get a list of all options contained in this group, + including options contained in option groups in this group, + and so on. """ + return list(chain(*[o.list_options() for o in self])) + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, list.__repr__(self)) + + def add_to_parser(self, parser): + """ Add this option group to a :class:`Bcfg2.Options.Parser` + object. """ + for opt in self: + opt.add_to_parser(parser) + + +class OptionGroup(OptionContainer): + """ Generic option group that is used only to organize options. + This uses :meth:`argparse.ArgumentParser.add_argument_group` + behind the scenes. """ + + def __init__(self, *items, **kwargs): + r""" + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param title: The title of the option group + :type title: string + :param description: A longer description of the option group + :param description: string + """ + OptionContainer.__init__(self, items) + self.title = kwargs.pop('title') + self.description = kwargs.pop('description', None) + + def add_to_parser(self, parser): + group = parser.add_argument_group(self.title, self.description) + OptionContainer.add_to_parser(self, group) + + +class ExclusiveOptionGroup(OptionContainer): + """ Option group that ensures that only one argument in the group + is present. This uses + :meth:`argparse.ArgumentParser.add_mutually_exclusive_group` + behind the scenes.""" + + def __init__(self, *items, **kwargs): + r""" + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param required: Exactly one argument in the group *must* be + specified. + :type required: boolean + """ + OptionContainer.__init__(self, items) + self.required = kwargs.pop('required', False) + + def add_to_parser(self, parser): + group = parser.add_mutually_exclusive_group(required=self.required) + OptionContainer.add_to_parser(self, group) + + +class Subparser(OptionContainer): + """ Option group that adds options in it to a subparser. This + uses a lot of functionality tied to `argparse Sub-commands + <http://docs.python.org/dev/library/argparse.html#sub-commands>`_. + + The subcommand string itself is stored in the + :attr:`Bcfg2.Options.setup` namespace as ``subcommand``. + + This is commonly used with :class:`Bcfg2.Options.Subcommand` + groups. + """ + + _subparsers = dict() + + def __init__(self, *items, **kwargs): + r""" + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param name: The name of the subparser. Required. + :type name: string + :param help: A help message for the subparser + :param help: string + """ + self.name = kwargs.pop('name') + self.help = kwargs.pop('help', None) + OptionContainer.__init__(self, items) + + def __repr__(self): + return "%s %s(%s)" % (self.__class__.__name__, + self.name, + list.__repr__(self)) + + def add_to_parser(self, parser): + if parser not in self._subparsers: + self._subparsers[parser] = parser.add_subparsers(dest='subcommand') + subparser = self._subparsers[parser].add_parser(self.name, + help=self.help) + OptionContainer.add_to_parser(self, subparser) + + +class WildcardSectionGroup(OptionContainer, Option): + """ WildcardSectionGroups contain options that may exist in + several different sections of the config that match a glob. It + works by creating options on the fly to match the sections + described in the glob. For example, consider: + + .. code-block:: python + + options = [ + Bcfg2.Options.WildcardSectionGroup( + Bcfg2.Options.Option(cf=("myplugin:*", "number"), type=int), + Bcfg2.Options.Option(cf=("myplugin:*", "description"))] + + If the config file contained ``[myplugin:foo]`` and + ``[myplugin:bar]`` sections, then this would automagically create + options for each of those. The end result would be: + + .. code-block:: python + + >>> Bcfg2.Options.setup + Namespace(myplugin_bar_description='Bar description', myplugin_bar_number=2, myplugin_foo_description='Foo description', myplugin_foo_number=1, myplugin_sections=['myplugin:foo', 'myplugin:bar']) + + All options must have the same section glob. + + The options are stored in an automatically-generated destination + given by:: + + <prefix><section>_<destination> + + ``<destination>`` is the original `dest + <http://docs.python.org/dev/library/argparse.html#dest>`_ of the + option. ``<section>`` is the section that it's found in. + ``<prefix>`` is automatically generated from the section glob by + replacing all consecutive characters disallowed in Python variable + names into underscores. (This can be overridden with the + constructor.) + + This group stores an additional option, the sections themselves, + in an option given by ``<prefix>sections``. + """ + + #: Regex to automatically get a destination for this option + _dest_re = re.compile(r'(\A(_|[^A-Za-z])+)|((_|[^A-Za-z0-9])+)') + + def __init__(self, *items, **kwargs): + r""" + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param prefix: The prefix to use for options generated by this + option group. By default this is generated + automatically from the config glob; see above + for details. + :type prefix: string + :param dest: The destination for the list of known sections + that match the glob. + :param dest: string + """ + OptionContainer.__init__(self, []) + self._section_glob = items[0].cf[0] + # get a default destination + self._prefix = kwargs.get("prefix", + self._dest_re.sub('_', self._section_glob)) + Option.__init__(self, dest=kwargs.get('dest', + self._prefix + "sections")) + self._options = items + + def list_options(self): + return [self] + OptionContainer.list_options(self) + + def from_config(self, cfp): + sections = [] + for section in cfp.sections(): + if fnmatch.fnmatch(section, self._section_glob): + sections.append(section) + newopts = [] + for opt_tmpl in self._options: + option = copy.deepcopy(opt_tmpl) + option.cf = (section, option.cf[1]) + option.dest = self._prefix + section + "_" + option.dest + newopts.append(option) + self.extend(newopts) + for parser in self.parsers: + parser.add_options(newopts) + return sections + + def add_to_parser(self, parser): + Option.add_to_parser(self, parser) + OptionContainer.add_to_parser(self, parser) diff --git a/src/lib/Bcfg2/Options/Options.py b/src/lib/Bcfg2/Options/Options.py new file mode 100644 index 000000000..3874f810d --- /dev/null +++ b/src/lib/Bcfg2/Options/Options.py @@ -0,0 +1,352 @@ +""" The base :class:`Bcfg2.Options.Option` object represents an +option. Unlike options in :mod:`argparse`, an Option object does not +need to be associated with an option parser; it exists on its own.""" + +import os +import copy +import fnmatch +import argparse +from Bcfg2.Options import Types +from Bcfg2.Compat import ConfigParser + + +__all__ = ["Option", "BooleanOption", "PathOption", "PositionalArgument", + "_debug"] + + +def _debug(msg): + """ Option parsing happens before verbose/debug have been set -- + they're options, after all -- so option parsing verbosity is + enabled by changing this to True. The verbosity here is primarily + of use to developers. """ + if os.environ.get('BCFG2_OPTIONS_DEBUG', '0') == '1': + print(msg) + + +#: A dict that records a mapping of argparse action name (e.g., +#: "store_true") to the argparse Action class for it. See +#: :func:`_get_action_class` +_action_map = dict() # pylint: disable=C0103 + + +def _get_action_class(action_name): + """ Given an argparse action name (e.g., "store_true"), get the + related :class:`argparse.Action` class. The mapping that stores + this information in :mod:`argparse` itself is unfortunately + private, so it's an implementation detail that we shouldn't depend + on. So we just instantiate a dummy parser, add a dummy argument, + and determine the class that way. """ + if (isinstance(action_name, type) and + issubclass(action_name, argparse.Action)): + return action_name + if action_name not in _action_map: + action = argparse.ArgumentParser().add_argument(action_name, + action=action_name) + _action_map[action_name] = action.__class__ + return _action_map[action_name] + + +class Option(object): + """ Representation of an option that can be specified on the + command line, as an environment variable, or in a config + file. Precedence is in that order; that is, an option specified on + the command line takes precendence over an option given by the + environment, which takes precedence over an option specified in + the config file. """ + + #: Keyword arguments that should not be passed on to the + #: :class:`argparse.ArgumentParser` constructor + _local_args = ['cf', 'env', 'man'] + + def __init__(self, *args, **kwargs): + """ See :meth:`argparse.ArgumentParser.add_argument` for a + full list of accepted parameters. + + In addition to supporting all arguments and keyword arguments + from :meth:`argparse.ArgumentParser.add_argument`, several + additional keyword arguments are allowed. + + :param cf: A tuple giving the section and option name that + this argument can be referenced as in the config + file. The option name may contain the wildcard + '*', in which case the value will be a dict of all + options matching the glob. (To use a wildcard in + the section, use a + :class:`Bcfg2.Options.WildcardSectionGroup`.) + :type cf: tuple + :param env: An environment variable that the value of this + option can be taken from. + :type env: string + :param man: A detailed description of the option that will be + used to populate automatically-generated manpages. + :type man: string + """ + #: The options by which this option can be called. + #: (Coincidentally, this is also the list of arguments that + #: will be passed to + #: :meth:`argparse.ArgumentParser.add_argument` when this + #: option is added to a parser.) As a result, ``args`` can be + #: tested to see if this argument can be given on the command + #: line at all, or if it is purely a config file option. + self.args = args + self._kwargs = kwargs + + #: The tuple giving the section and option name for this + #: option in the config file + self.cf = None # pylint: disable=C0103 + + #: The environment variable that this option can take its + #: value from + self.env = None + + #: A detailed description of this option that will be used in + #: man pages. + self.man = None + + #: A list of :class:`Bcfg2.Options.Parser` objects to which + #: this option has been added. (There will be more than one + #: parser if this option is added to a subparser, for + #: instance.) + self.parsers = [] + + #: A dict of :class:`Bcfg2.Options.Parser` -> + #: :class:`argparse.Action` that gives the actions that + #: resulted from adding this option to each parser that it was + #: added to. If this option cannot be specified on the + #: command line (i.e., it only takes its value from the config + #: file), then this will be empty. + self.actions = dict() + + self.type = self._kwargs.get("type") + self.help = self._kwargs.get("help") + self._default = self._kwargs.get("default") + for kwarg in self._local_args: + setattr(self, kwarg, self._kwargs.pop(kwarg, None)) + if self.args: + # cli option + self._dest = None + else: + action_cls = _get_action_class(self._kwargs.get('action', 'store')) + # determine the name of this option. use, in order, the + # 'name' kwarg; the option name; the environment variable + # name. + self._dest = None + if 'dest' in self._kwargs: + self._dest = self._kwargs.pop('dest') + elif self.cf is not None: + self._dest = self.cf[1] + elif self.env is not None: + self._dest = self.env + kwargs = copy.copy(self._kwargs) + kwargs.pop("action", None) + self.actions[None] = action_cls(self._dest, self._dest, **kwargs) + + def __repr__(self): + sources = [] + if self.args: + sources.extend(self.args) + if self.cf: + sources.append("%s.%s" % self.cf) + if self.env: + sources.append("$" + self.env) + spec = ["sources=%s" % sources, "default=%s" % self.default] + spec.append("%d parsers" % (len(self.parsers))) + return 'Option(%s: %s)' % (self.dest, ", ".join(spec)) + + def list_options(self): + """ List options contained in this option. This exists to + provide a consistent interface with + :class:`Bcfg2.Options.OptionGroup` """ + return [self] + + def finalize(self, namespace): + """ Finalize the default value for this option. This is used + with actions (such as :class:`Bcfg2.Options.ComponentAction`) + that allow you to specify a default in a different format than + its final storage format; this can be called after it has been + determined that the default will be used (i.e., the option is + not given on the command line or in the config file) to store + the appropriate default value in the appropriate format.""" + for parser, action in self.actions.items(): + if hasattr(action, "finalize"): + if parser: + _debug("Finalizing %s for %s" % (self, parser)) + else: + _debug("Finalizing %s" % self) + action.finalize(parser, namespace) + + def from_config(self, cfp): + """ Get the value of this option from the given + :class:`ConfigParser.ConfigParser`. If it is not found in the + config file, the default is returned. (If there is no + default, None is returned.) + + :param cfp: The config parser to get the option value from + :type cfp: ConfigParser.ConfigParser + :returns: The default value + """ + if not self.cf: + return None + if '*' in self.cf[1]: + if cfp.has_section(self.cf[0]): + # build a list of known options in this section, and + # exclude them + exclude = set() + for parser in self.parsers: + exclude.update(o.cf[1] + for o in parser.option_list + if o.cf and o.cf[0] == self.cf[0]) + rv = dict([(o, cfp.get(self.cf[0], o)) + for o in fnmatch.filter(cfp.options(self.cf[0]), + self.cf[1]) + if o not in exclude]) + else: + rv = dict() + else: + if self.type: + rtype = self.type + else: + rtype = lambda x: x + try: + rv = rtype(cfp.getboolean(*self.cf)) + except ValueError: + rv = rtype(cfp.get(*self.cf)) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + rv = None + _debug("Setting %s from config file(s): %s" % (self, rv)) + return rv + + def default_from_config(self, cfp): + """ Set the default value of this option from the config file + or from the environment. + + :param cfp: The config parser to get the option value from + :type cfp: ConfigParser.ConfigParser + """ + if self.env and self.env in os.environ: + self.default = os.environ[self.env] + _debug("Setting the default of %s from environment: %s" % + (self, self.default)) + else: + val = self.from_config(cfp) + if val is not None: + _debug("Setting the default of %s from config: %s" % + (self, val)) + self.default = val + + def _get_default(self): + """ Getter for the ``default`` property """ + return self._default + + def _set_default(self, value): + """ Setter for the ``default`` property """ + self._default = value + for action in self.actions.values(): + action.default = value + + #: The current default value of this option + default = property(_get_default, _set_default) + + def _get_dest(self): + """ Getter for the ``dest`` property """ + return self._dest + + def _set_dest(self, value): + """ Setter for the ``dest`` property """ + self._dest = value + for action in self.actions.values(): + action.dest = value + + #: The namespace destination of this option (see `dest + #: <http://docs.python.org/dev/library/argparse.html#dest>`_) + dest = property(_get_dest, _set_dest) + + def add_to_parser(self, parser): + """ Add this option to the given parser. + + :param parser: The parser to add the option to. + :type parser: Bcfg2.Options.Parser + :returns: argparse.Action + """ + self.parsers.append(parser) + if self.args: + # cli option + _debug("Adding %s to %s as a CLI option" % (self, parser)) + action = parser.add_argument(*self.args, **self._kwargs) + if not self._dest: + self._dest = action.dest + if self._default: + action.default = self._default + self.actions[parser] = action + else: + # else, config file-only option + _debug("Adding %s to %s as a config file-only option" % + (self, parser)) + + +class PathOption(Option): + """ Shortcut for options that expect a path argument. Uses + :meth:`Bcfg2.Options.Types.path` to transform the argument into a + canonical path. + + The type of a path option can also be overridden to return an + option file-like object. For example: + + .. code-block:: python + + options = [ + Bcfg2.Options.PathOption( + "--input", type=argparse.FileType('r'), + help="The input file")] + """ + + def __init__(self, *args, **kwargs): + kwargs.setdefault('type', Types.path) + kwargs.setdefault('metavar', '<path>') + Option.__init__(self, *args, **kwargs) + + +class _BooleanOptionAction(argparse.Action): + """ BooleanOptionAction sets a boolean value in the following ways: + - if None is passed, store the default + - if the option_string is not None, then the option was passed on the + command line, thus store the opposite of the default (this is the + argparse store_true and store_false behavior) + - if a boolean value is passed, use that + + Defined here instead of :mod:`Bcfg2.Options.Actions` because otherwise + there is a circular import Options -> Actions -> Parser -> Options """ + + def __call__(self, parser, namespace, values, option_string=None): + if values is None: + setattr(namespace, self.dest, self.default) + elif option_string is not None: + setattr(namespace, self.dest, not self.default) + else: + setattr(namespace, self.dest, bool(values)) + + +class BooleanOption(Option): + """ Shortcut for boolean options. The default is False, but this + can easily be overridden: + + .. code-block:: python + + options = [ + Bcfg2.Options.PathOption( + "--dwim", default=True, help="Do What I Mean")] + """ + def __init__(self, *args, **kwargs): + kwargs.setdefault('action', _BooleanOptionAction) + kwargs.setdefault('nargs', 0) + kwargs.setdefault('default', False) + + Option.__init__(self, *args, **kwargs) + + +class PositionalArgument(Option): + """ Shortcut for positional arguments. """ + def __init__(self, *args, **kwargs): + if 'metavar' not in kwargs: + kwargs['metavar'] = '<%s>' % args[0] + Option.__init__(self, *args, **kwargs) diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py new file mode 100644 index 000000000..677a69e4c --- /dev/null +++ b/src/lib/Bcfg2/Options/Parser.py @@ -0,0 +1,352 @@ +""" The option parser """ + +import os +import sys +import argparse +from Bcfg2.version import __version__ +from Bcfg2.Compat import ConfigParser +from Bcfg2.Options import Option, PathOption, BooleanOption, _debug + +__all__ = ["setup", "OptionParserException", "Parser", "get_parser"] + + +#: The repository option. This is specified here (and imported into +#: :module:`Bcfg2.Options.Common`) rather than vice-versa due to +#: circular imports. +repository = PathOption( # pylint: disable=C0103 + '-Q', '--repository', cf=('server', 'repository'), + default='var/lib/bcfg2', help="Server repository path") + + +#: A module-level :class:`argparse.Namespace` object that stores all +#: configuration for Bcfg2. +setup = argparse.Namespace(version=__version__, # pylint: disable=C0103 + name="Bcfg2", + uri='http://trac.mcs.anl.gov/projects/bcfg2') + + +class OptionParserException(Exception): + """ Base exception raised for generic option parser errors """ + + +class Parser(argparse.ArgumentParser): + """ The Bcfg2 option parser. Most interfaces should not need to + instantiate a parser, but should instead use + :func:`Bcfg2.Options.get_parser` to get the parser that already + exists.""" + + #: Option for specifying the path to the Bcfg2 config file + configfile = PathOption('-C', '--config', + env="BCFG2_CONFIG_FILE", + help="Path to configuration file", + default="/etc/bcfg2.conf") + + #: Builtin options that apply to all commands + options = [configfile, + BooleanOption('--version', help="Print the version and exit"), + Option('-E', '--encoding', metavar='<encoding>', + default='UTF-8', help="Encoding of config files", + cf=('components', 'encoding'))] + + #: Flag used in unit tests to disable actual config file reads + unit_test = False + + def __init__(self, **kwargs): + """ See :class:`argparse.ArgumentParser` for a full list of + accepted parameters. + + In addition to supporting all arguments and keyword arguments + from :class:`argparse.ArgumentParser`, several additional + keyword arguments are allowed. + + :param components: A list of components to add to the parser. + :type components: list + :param namespace: The namespace to store options in. Default + is :attr:`Bcfg2.Options.setup`. + :type namespace: argparse.Namespace + :param add_base_options: Whether or not to add the options in + :attr:`Bcfg2.Options.Parser.options` + to the parser. Setting this to False + is default for subparsers. Default is + True. + :type add_base_options: bool + """ + self._cfp = ConfigParser.ConfigParser() + components = kwargs.pop('components', []) + + #: The namespace options will be stored in. + self.namespace = kwargs.pop('namespace', setup) + if self.namespace is None: + self.namespace = setup + add_base_options = kwargs.pop('add_base_options', True) + + #: Flag to indicate that this is the pre-parsing 'early' run + #: for important options like database settings that must be + #: loaded before other components can be. + self._early = kwargs.pop('early', False) + + if 'add_help' not in kwargs: + kwargs['add_help'] = add_base_options + argparse.ArgumentParser.__init__(self, **kwargs) + + #: Whether or not parsing has completed on all current options. + self.parsed = False + + #: The argument list that was parsed. + self.argv = None + + #: Components that have been added to the parser + self.components = [] + + #: Options that have been added to the parser + self.option_list = [] + self._defaults_set = [] + self._config_files = [] + if add_base_options: + self.add_component(self) + if components: + for component in components: + self.add_component(component) + + def add_options(self, options): + """ Add an explicit list of options to the parser. When + possible, prefer :func:`Bcfg2.Options.Parser.add_component` to + add a whole component instead.""" + self.parsed = False + for option in options: + if option not in self.option_list: + self.option_list.extend(option.list_options()) + option.add_to_parser(self) + + def add_component(self, component): + """ Add a component (and all of its options) to the + parser. """ + if component not in self.components: + _debug("Adding component %s to %s" % (component, self)) + self.components.append(component) + if hasattr(component, "options"): + self.add_options(getattr(component, "options")) + + def _set_defaults_from_config(self): + """ Set defaults from the config file for all options that can + come from the config file, but haven't yet had their default + set """ + _debug("Setting defaults on all options") + for opt in self.option_list: + if opt not in self._defaults_set: + opt.default_from_config(self._cfp) + self._defaults_set.append(opt) + + def _parse_config_options(self): + """ populate the namespace with default values for any options + that aren't already in the namespace (i.e., options without + CLI arguments) """ + _debug("Parsing config file-only options") + for opt in self.option_list[:]: + if not opt.args and opt.dest not in self.namespace: + value = opt.default + if value: + for _, action in opt.actions.items(): + _debug("Setting config file-only option %s to %s" % + (opt, value)) + action(self, self.namespace, value) + else: + setattr(self.namespace, opt.dest, value) + + def _finalize(self): + """ Finalize the value of any options that require that + additional post-processing step. (Mostly + :class:`Bcfg2.Options.Actions.ComponentAction` subclasses.) + """ + _debug("Finalizing options") + for opt in self.option_list[:]: + opt.finalize(self.namespace) + + def _reset_namespace(self): + """ Delete all options from the namespace except for a few + predefined values and config file options. """ + self.parsed = False + _debug("Resetting namespace") + for attr in dir(self.namespace): + if (not attr.startswith("_") and + attr not in ['uri', 'version', 'name'] and + attr not in self._config_files): + _debug("Deleting %s" % attr) + delattr(self.namespace, attr) + + def add_config_file(self, dest, cfile, reparse=True): + """ Add a config file, which triggers a full reparse of all + options. """ + if dest not in self._config_files: + _debug("Adding new config file %s for %s" % (cfile, dest)) + self._reset_namespace() + self._cfp.read([cfile]) + self._defaults_set = [] + self._set_defaults_from_config() + if reparse: + self._parse_config_options() + self._config_files.append(dest) + + def reparse(self, argv=None): + """ Reparse options after they have already been parsed. + + :param argv: The argument list to parse. By default, + :attr:`Bcfg2.Options.Parser.argv` is reused. + (I.e., the argument list that was initially + parsed.) :type argv: list + """ + _debug("Reparsing all options") + self._reset_namespace() + self.parse(argv or self.argv) + + def parse(self, argv=None): + """ Parse options. + + :param argv: The argument list to parse. By default, + ``sys.argv[1:]`` is used. This is stored in + :attr:`Bcfg2.Options.Parser.argv` for reuse by + :func:`Bcfg2.Options.Parser.reparse`. + :type argv: list + """ + _debug("Parsing options") + if argv is None: + argv = sys.argv[1:] + if self.parsed and self.argv == argv: + _debug("Returning already parsed namespace") + return self.namespace + self.argv = argv + + # phase 1: get and read config file + _debug("Option parsing phase 1: Get and read main config file") + bootstrap_parser = argparse.ArgumentParser(add_help=False) + self.configfile.add_to_parser(bootstrap_parser) + self.configfile.default_from_config(self._cfp) + bootstrap = bootstrap_parser.parse_known_args(args=self.argv)[0] + + # check whether the specified bcfg2.conf exists + if not self.unit_test and not os.path.exists(bootstrap.config): + self.error("Could not read %s" % bootstrap.config) + self.add_config_file(self.configfile.dest, bootstrap.config, + reparse=False) + + # phase 2: re-parse command line for early options; currently, + # that's database options + _debug("Option parsing phase 2: Parse early options") + if not self._early: + early_opts = argparse.Namespace() + early_parser = Parser(add_help=False, namespace=early_opts, + early=True) + # add the repo option so we can resolve <repository> + # macros + early_parser.add_options([repository]) + early_components = [] + for component in self.components: + if getattr(component, "parse_first", False): + early_components.append(component) + early_parser.add_component(component) + early_parser.parse(self.argv) + _debug("Early parsing complete, calling hooks") + for component in early_components: + if hasattr(component, "component_parsed_hook"): + _debug("Calling component_parsed_hook on %s" % component) + getattr(component, "component_parsed_hook")(early_opts) + + # phase 3: re-parse command line, loading additional + # components, until all components have been loaded. On each + # iteration, set defaults from config file/environment + # variables + _debug("Option parsing phase 3: Main parser loop") + # _set_defaults_from_config must be called before _parse_config_options + # This is due to a tricky interaction between the two methods: + # + # (1) _set_defaults_from_config does what its name implies, it updates + # the "default" property of each Option based on the value that exists + # in the config. + # + # (2) _parse_config_options will look at each option and set it to the + # default value that is _currently_ defined. If the option does not + # exist in the namespace, it will be added. The method carefully + # avoids overwriting the value of an option that is already defined in + # the namespace. + # + # Thus, if _set_defaults_from_config has not been called yet when + # _parse_config_options is called, all config file options will get set + # to their hardcoded defaults. This process defines the options in the + # namespace and _parse_config_options will never look at them again. + self._set_defaults_from_config() + self._parse_config_options() + while not self.parsed: + self.parsed = True + self._set_defaults_from_config() + self.parse_known_args(args=self.argv, namespace=self.namespace) + self._parse_config_options() + self._finalize() + + # phase 4: fix up <repository> macros + _debug("Option parsing phase 4: Fix up macros") + repo = getattr(self.namespace, "repository", repository.default) + for attr in dir(self.namespace): + value = getattr(self.namespace, attr) + if (not attr.startswith("_") and + hasattr(value, "replace") and + "<repository>" in value): + setattr(self.namespace, attr, + value.replace("<repository>", repo, 1)) + _debug("Fixing up macros in %s: %s -> %s" % + (attr, value, getattr(self.namespace, attr))) + + # phase 5: call post-parsing hooks + _debug("Option parsing phase 5: Call hooks") + if not self._early: + for component in self.components: + if hasattr(component, "options_parsed_hook"): + _debug("Calling post-parsing hook on %s" % component) + getattr(component, "options_parsed_hook")() + + return self.namespace + + +#: A module-level :class:`Bcfg2.Options.Parser` object that is used +#: for all parsing +_parser = Parser() # pylint: disable=C0103 + + +def get_parser(description=None, components=None, namespace=None): + """ Get an existing :class:`Bcfg2.Options.Parser` object. (One is + created at the module level when :mod:`Bcfg2.Options` is + imported.) If no arguments are given, then the existing parser is + simply fetched. + + If arguments are given, then one of two things happens: + + * If this is the first ``get_parser`` call with arguments, then + the values given are set accordingly in the parser, and it is + returned. + * If this is not the first such call, then + :class:`Bcfg2.Options.OptionParserException` is raised. + + That is, a ``get_parser`` call with options is considered to + initialize the parser that already exists, and that can only + happen once. + + :param description: Set the parser description + :type description: string + :param components: Load the given components in the parser + :type components: list + :param namespace: Use the given namespace instead of + :attr:`Bcfg2.Options.setup` + :type namespace: argparse.Namespace + :returns: Bcfg2.Options.Parser object + """ + if Parser.unit_test: + return Parser(description=description, components=components, + namespace=namespace) + elif (description or components or namespace): + if description: + _parser.description = description + if components is not None: + for component in components: + _parser.add_component(component) + if namespace: + _parser.namespace = namespace + return _parser diff --git a/src/lib/Bcfg2/Options/Subcommands.py b/src/lib/Bcfg2/Options/Subcommands.py new file mode 100644 index 000000000..660bd5077 --- /dev/null +++ b/src/lib/Bcfg2/Options/Subcommands.py @@ -0,0 +1,238 @@ +""" Classes to make it easier to create commands with large numbers of +subcommands (e.g., bcfg2-admin, bcfg2-info). """ + +import re +import cmd +import sys +import copy +import shlex +import logging +from Bcfg2.Compat import StringIO +from Bcfg2.Options import PositionalArgument +from Bcfg2.Options.OptionGroups import Subparser +from Bcfg2.Options.Parser import Parser, setup as master_setup + +__all__ = ["Subcommand", "HelpCommand", "CommandRegistry", "register_commands"] + + +class Subcommand(object): + """ Base class for subcommands. This must be subclassed to create + commands. + + Specifically, you must override + :func:`Bcfg2.Options.Subcommand.run`. You may want to override: + + * The docstring, which will be used as the short help. + * :attr:`Bcfg2.Options.Subcommand.options` + * :attr:`Bcfg2.Options.Subcommand.help` + * :attr:`Bcfg2.Options.Subcommand.interactive` + * + * :func:`Bcfg2.Options.Subcommand.shutdown` + + You should not need to override + :func:`Bcfg2.Options.Subcommand.__call__` or + :func:`Bcfg2.Options.Subcommand.usage`. + + A ``Subcommand`` subclass constructor must not take any arguments. + """ + + #: Options this command takes + options = [] + + #: Longer help message + help = None + + #: Whether or not to expose this command in an interactive + #: :class:`cmd.Cmd` shell, if one is used. (``bcfg2-info`` uses + #: one, ``bcfg2-admin`` does not.) + interactive = True + + _ws_re = re.compile(r'\s+', flags=re.MULTILINE) + + def __init__(self): + self.core = None + description = "%s: %s" % (self.__class__.__name__.lower(), + self.__class__.__doc__) + + #: The :class:`Bcfg2.Options.Parser` that will be used to + #: parse options if this subcommand is called from an + #: interactive :class:`cmd.Cmd` shell. + self.parser = Parser( + prog=self.__class__.__name__.lower(), + description=description, + components=[self], + add_base_options=False, + epilog=self.help) + self._usage = None + + #: A :class:`logging.Logger` that can be used to produce + #: logging output for this command. + self.logger = logging.getLogger(self.__class__.__name__.lower()) + + def __call__(self, args=None): + """ Perform option parsing and other tasks necessary to + support running ``Subcommand`` objects as part of a + :class:`cmd.Cmd` shell. You should not need to override + ``__call__``. + + :param args: Arguments given in the interactive shell + :type args: list of strings + :returns: The return value of :func:`Bcfg2.Options.Subcommand.run` + """ + if args is not None: + self.parser.namespace = copy.copy(master_setup) + alist = shlex.split(args) + try: + setup = self.parser.parse(alist) + except SystemExit: + return sys.exc_info()[1].code + return self.run(setup) + else: + return self.run(master_setup) + + def usage(self): + """ Get the short usage message. """ + if self._usage is None: + sio = StringIO() + self.parser.print_usage(file=sio) + usage = self._ws_re.sub(' ', sio.getvalue()).strip()[7:] + doc = self._ws_re.sub(' ', getattr(self, "__doc__")).strip() + if doc is None: + self._usage = usage + else: + self._usage = "%s - %s" % (usage, doc) + return self._usage + + def run(self, setup): + """ Run the command. + + :param setup: A namespace giving the options for this command. + This must be used instead of + :attr:`Bcfg2.Options.setup` because this command + may have been called from an interactive + :class:`cmd.Cmd` shell, and thus has its own + option parser and its own (private) namespace. + ``setup`` is guaranteed to contain all of the + options in the global + :attr:`Bcfg2.Options.setup` namespace, in + addition to any local options given to this + command from the interactive shell. + :type setup: argparse.Namespace + """ + raise NotImplementedError + + def shutdown(self): + """ Perform any necessary shtudown tasks for this command This + is called to when the program exits (*not* when this command + is finished executing). """ + pass + + +class HelpCommand(Subcommand): + """ Get help on a specific subcommand. This must be subclassed to + create the actual help command by overriding + :func:`Bcfg2.Options.HelpCommand.command_registry` and giving the + command access to a :class:`Bcfg2.Options.CommandRegistry`. """ + options = [PositionalArgument("command", nargs='?')] + + # the interactive shell has its own help + interactive = False + + def command_registry(self): + """ Return a :class:`Bcfg2.Options.CommandRegistry` class. + All commands registered with the class will be included in the + help message. """ + raise NotImplementedError + + def run(self, setup): + commands = self.command_registry() + if setup.command: + try: + commands[setup.command].parser.print_help() + return 0 + except KeyError: + print("No such command: %s" % setup.command) + for command in sorted(commands.keys()): + print(commands[command].usage()) + + +class CommandRegistry(object): + """ A ``CommandRegistry`` is used to register subcommands and + provides a single interface to run them. It's also used by + :class:`Bcfg2.Options.HelpCommand` to produce help messages for + all available commands. """ + + #: A dict of registered commands. Keys are the class names, + #: lowercased (i.e., the command names), and values are instances + #: of the command objects. + commands = dict() + + options = [] + + def runcommand(self): + """ Run the single command named in + ``Bcfg2.Options.setup.subcommand``, which is where + :class:`Bcfg2.Options.Subparser` groups store the + subcommand. """ + try: + return self.commands[master_setup.subcommand].run(master_setup) + finally: + self.shutdown() + + def shutdown(self): + """ Perform shutdown tasks. This calls the ``shutdown`` + method of all registered subcommands. """ + self.commands[master_setup.subcommand].shutdown() + + @classmethod + def register_command(cls, cmdcls): + """ Register a single command. + + :param cmdcls: The command class to register + :type cmdcls: type + :returns: An instance of ``cmdcls`` + """ + cmd_obj = cmdcls() + name = cmdcls.__name__.lower() + cls.commands[name] = cmd_obj + # py2.5 can't mix *magic and non-magical keyword args, thus + # the **dict(...) + cls.options.append( + Subparser(*cmdcls.options, **dict(name=name, help=cmdcls.__doc__))) + if issubclass(cls, cmd.Cmd) and cmdcls.interactive: + setattr(cls, "do_%s" % name, cmd_obj) + setattr(cls, "help_%s" % name, cmd_obj.parser.print_help) + return cmd_obj + + +def register_commands(registry, candidates, parent=Subcommand): + """ Register all subcommands in ``candidates`` against the + :class:`Bcfg2.Options.CommandRegistry` subclass given in + ``registry``. A command is registered if and only if: + + * It is a subclass of the given ``parent`` (by default, + :class:`Bcfg2.Options.Subcommand`); + * It is not the parent class itself; and + * Its name does not start with an underscore. + + :param registry: The :class:`Bcfg2.Options.CommandRegistry` + subclass against which commands will be + registered. + :type registry: Bcfg2.Options.CommandRegistry + :param candidates: A list of objects that will be considered for + registration. Only objects that meet the + criteria listed above will be registered. + :type candidates: list + :param parent: Specify a parent class other than + :class:`Bcfg2.Options.Subcommand` that all + registered commands must subclass. + :type parent: type + """ + for attr in candidates: + try: + if (issubclass(attr, parent) and + attr != parent and + not attr.__name__.startswith("_")): + registry.register_command(attr) + except TypeError: + pass diff --git a/src/lib/Bcfg2/Options/Types.py b/src/lib/Bcfg2/Options/Types.py new file mode 100644 index 000000000..d11e54fba --- /dev/null +++ b/src/lib/Bcfg2/Options/Types.py @@ -0,0 +1,120 @@ +""" :mod:`Bcfg2.Options` provides a number of useful types for use +with the :class:`Bcfg2.Options.Option` constructor. """ + +import os +import re +import pwd +import grp + +_COMMA_SPLIT_RE = re.compile(r'\s*,\s*') + + +def path(value): + """ A generic path. ``~`` will be expanded with + :func:`os.path.expanduser` and the absolute resulting path will be + used. This does *not* ensure that the path exists. """ + return os.path.abspath(os.path.expanduser(value)) + + +def comma_list(value): + """ Split a comma-delimited list, with optional whitespace around + the commas.""" + return _COMMA_SPLIT_RE.split(value) + + +def colon_list(value): + """ Split a colon-delimited list. Whitespace is not allowed + around the colons. """ + return value.split(':') + + +def comma_dict(value): + """ Split an option string on commas, optionally surrounded by + whitespace, and split the resulting items again on equals signs, + returning a dict """ + result = dict() + if value: + items = comma_list(value) + for item in items: + if '=' in item: + key, value = item.split(r'=', 1) + try: + result[key] = bool(value) + except ValueError: + try: + result[key] = int(value) + except ValueError: + result[key] = value + else: + result[item] = True + return result + + +def anchored_regex_list(value): + """ Split an option string on whitespace and compile each element as + an anchored regex """ + try: + return [re.compile('^' + x + '$') for x in re.split(r'\s+', value)] + except re.error: + raise ValueError("Not a list of regexes", value) + + +def octal(value): + """ Given an octal string, get an integer representation. """ + return int(value, 8) + + +def username(value): + """ Given a username or numeric UID, get a numeric UID. The user + must exist.""" + try: + return int(value) + except ValueError: + return int(pwd.getpwnam(value)[2]) + + +def groupname(value): + """ Given a group name or numeric GID, get a numeric GID. The + user must exist.""" + try: + return int(value) + except ValueError: + return int(grp.getgrnam(value)[2]) + + +def timeout(value): + """ Convert the value into a float or None. """ + if value is None: + return value + rv = float(value) # pass ValueError up the stack + if rv <= 0: + return None + return rv + + +# pylint: disable=C0103 +_bytes_multipliers = dict(k=1, + m=2, + g=3, + t=4) +_suffixes = "".join(_bytes_multipliers.keys()).lower() +_suffixes += _suffixes.upper() +_bytes_re = re.compile(r'(?P<value>\d+)(?P<multiplier>[%s])?' % _suffixes) +# pylint: enable=C0103 + + +def size(value): + """ Given a number of bytes in a human-readable format (e.g., + '512m', '2g'), get the absolute number of bytes as an integer. + """ + if value == -1: + return value + mat = _bytes_re.match(value) + if not mat: + raise ValueError("Not a valid size", value) + rvalue = int(mat.group("value")) + mult = mat.group("multiplier") + if mult: + return rvalue * (1024 ** _bytes_multipliers[mult.lower()]) + else: + return rvalue diff --git a/src/lib/Bcfg2/Options/__init__.py b/src/lib/Bcfg2/Options/__init__.py new file mode 100644 index 000000000..96465ec56 --- /dev/null +++ b/src/lib/Bcfg2/Options/__init__.py @@ -0,0 +1,10 @@ +""" Bcfg2 options parsing. """ + +# pylint: disable=W0611,W0401 +from Bcfg2.Options import Types +from Bcfg2.Options.Options import * +from Bcfg2.Options.Common import * +from Bcfg2.Options.Parser import * +from Bcfg2.Options.Actions import * +from Bcfg2.Options.Subcommands import * +from Bcfg2.Options.OptionGroups import * diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 8e2fe1cb1..90b9f0ec7 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -4,7 +4,6 @@ import atexit import daemon import logging import time -import traceback import threading from lockfile import LockFailed, LockTimeout @@ -16,11 +15,11 @@ except ImportError: # pylint: enable=E0611 import Bcfg2.Logger -from Bcfg2.Reporting.Transport import load_transport_from_config, \ - TransportError, TransportImportError +import Bcfg2.Options +from Bcfg2.Reporting.Transport.base import TransportError from Bcfg2.Reporting.Transport.DirectStore import DirectStore -from Bcfg2.Reporting.Storage import load_storage_from_config, \ - StorageError, StorageImportError +from Bcfg2.Reporting.Storage.base import StorageError + class ReportingError(Exception): @@ -31,7 +30,7 @@ class ReportingError(Exception): class ReportingStoreThread(threading.Thread): """Thread for calling the storage backend""" def __init__(self, interaction, storage, group=None, target=None, - name=None, args=(), kwargs=None): + name=None, semaphore=None, args=(), kwargs=None): """Initialize the thread with a reference to the interaction as well as the storage engine to use""" threading.Thread.__init__(self, group, target, name, args, @@ -39,59 +38,70 @@ class ReportingStoreThread(threading.Thread): self.interaction = interaction self.storage = storage self.logger = logging.getLogger('bcfg2-report-collector') + self.semaphore = semaphore def run(self): """Call the database storage procedure (aka import)""" try: - start = time.time() - self.storage.import_interaction(self.interaction) - self.logger.info("Imported interaction for %s in %ss" % - (self.interaction.get('hostname', '<unknown>'), - time.time() - start)) - except: - #TODO requeue? - self.logger.error("Unhandled exception in import thread %s" % - traceback.format_exc().splitlines()[-1]) + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', + '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + sys.exc_info()[1]) + finally: + if self.semaphore: + self.semaphore.release() class ReportingCollector(object): """The collecting process for reports""" - - def __init__(self, setup): + options = [Bcfg2.Options.Common.reporting_storage, + Bcfg2.Options.Common.reporting_transport, + Bcfg2.Options.Common.daemon, + Bcfg2.Options.Option( + '--max-children', dest="children", + cf=('reporting', 'max_children'), type=int, + default=0, + help='Maximum number of children for the reporting collector')] + + def __init__(self): """Setup the collector. This may be called by the daemon or though bcfg2-admin""" - self.setup = setup - self.datastore = setup['repo'] - self.encoding = setup['encoding'] self.terminate = None self.context = None self.children = [] self.cleanup_threshold = 25 - if setup['debug']: + if Bcfg2.Options.setup.children > 0: + self.semaphore = threading.Semaphore( + value=Bcfg2.Options.setup.children) + + if Bcfg2.Options.setup.debug: level = logging.DEBUG - elif setup['verbose']: + elif Bcfg2.Options.setup.verbose: level = logging.INFO else: level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-report-collector', - to_console=logging.INFO, - to_syslog=setup['syslog'], - to_file=setup['logging'], - level=level) + Bcfg2.Logger.setup_logging() self.logger = logging.getLogger('bcfg2-report-collector') try: - self.transport = load_transport_from_config(setup) - self.storage = load_storage_from_config(setup) + self.transport = Bcfg2.Options.setup.reporting_transport() + self.storage = Bcfg2.Options.setup.reporting_storage() except TransportError: self.logger.error("Failed to load transport: %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) raise ReportingError except StorageError: self.logger.error("Failed to load storage: %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) raise ReportingError if isinstance(self.transport, DirectStore): @@ -102,12 +112,12 @@ class ReportingCollector(object): try: self.logger.debug("Validating storage %s" % - self.storage.__class__.__name__) + self.storage.__class__.__name__) self.storage.validate() except: self.logger.error("Storage backed %s failed to validate: %s" % - (self.storage.__class__.__name__, - traceback.format_exc().splitlines()[-1])) + (self.storage.__class__.__name__, + sys.exc_info()[1])) def run(self): """Startup the processing and go!""" @@ -116,10 +126,10 @@ class ReportingCollector(object): self.context = daemon.DaemonContext(detach_process=True) iter = 0 - if self.setup['daemon']: + if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") - self.context.pidfile = TimeoutPIDLockFile(self.setup['daemon'], - acquire_timeout=5) + self.context.pidfile = TimeoutPIDLockFile( + Bcfg2.Options.setup.daemon, acquire_timeout=5) # Attempt to ensure lockfile is able to be created and not stale try: self.context.pidfile.acquire() @@ -136,7 +146,7 @@ class ReportingCollector(object): else: self.logger.error("Failed to daemonize: " "Failed to acquire lock on %s" % - self.setup['daemon']) + Bcfg2.Options.setup.daemon) self.shutdown() return else: @@ -152,8 +162,10 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - - store_thread = ReportingStoreThread(interaction, self.storage) + if Bcfg2.Options.setup.children > 0: + self.semaphore.acquire() + store_thread = ReportingStoreThread(interaction, self.storage, + semaphore=self.semaphore) store_thread.start() self.children.append(store_thread) @@ -167,7 +179,7 @@ class ReportingCollector(object): self.shutdown() except: self.logger.error("Unhandled exception in main loop %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) def shutdown(self): """Cleanup and go""" diff --git a/src/lib/Bcfg2/Reporting/Reports.py b/src/lib/Bcfg2/Reporting/Reports.py new file mode 100755 index 000000000..219d74584 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Reports.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python +"""Query reporting system for client status.""" + +import sys +import argparse +import datetime +import Bcfg2.DBSettings + + +def hosts_by_entry_type(clients, etype, entryspec): + result = [] + for entry in entryspec: + for client in clients: + items = getattr(client.current_interaction, etype)() + for item in items: + if (item.entry_type == entry[0] and + item.name == entry[1]): + result.append(client) + return result + + +def print_fields(fields, client, fmt, extra=None): + """ Prints the fields specified in fields of client, max_name + specifies the column width of the name column. """ + fdata = [] + if extra is None: + extra = dict() + for field in fields: + if field == 'time': + fdata.append(str(client.current_interaction.timestamp)) + elif field == 'state': + if client.current_interaction.isclean(): + fdata.append("clean") + else: + fdata.append("dirty") + elif field == 'total': + fdata.append(client.current_interaction.total_count) + elif field == 'good': + fdata.append(client.current_interaction.good_count) + elif field == 'modified': + fdata.append(client.current_interaction.modified_count) + elif field == 'extra': + fdata.append(client.current_interaction.extra_count) + elif field == 'bad': + fdata.append((client.current_interaction.bad_count)) + elif field == 'stale': + fdata.append(client.current_interaction.isstale()) + else: + try: + fdata.append(getattr(client, field)) + except AttributeError: + fdata.append(extra.get(field, "N/A")) + + print(fmt % tuple(fdata)) + + +def print_entries(interaction, etype): + items = getattr(interaction, etype)() + for item in items: + print("%-70s %s" % (item.entry_type + ":" + item.name, etype)) + + +class _SingleHostCmd(Bcfg2.Options.Subcommand): # pylint: disable=W0223 + """ Base class for bcfg2-reports modes that take a single host as + a positional argument """ + options = [Bcfg2.Options.PositionalArgument("host")] + + def get_client(self, setup): + from Bcfg2.Reporting.models import Client + try: + return Client.objects.select_related().get(name=setup.host) + except Client.DoesNotExist: + print("No such host: %s" % setup.host) + raise SystemExit(2) + + +class Show(_SingleHostCmd): + """ Show bad, extra, modified, or all entries from a given host """ + + options = _SingleHostCmd.options + [ + Bcfg2.Options.BooleanOption( + "-b", "--bad", help="Show bad entries from HOST"), + Bcfg2.Options.BooleanOption( + "-e", "--extra", help="Show extra entries from HOST"), + Bcfg2.Options.BooleanOption( + "-m", "--modified", help="Show modified entries from HOST")] + + def run(self, setup): + client = self.get_client(setup) + show_all = not setup.bad and not setup.extra and not setup.modified + if setup.bad or show_all: + print_entries(client.current_interaction, "bad") + if setup.modified or show_all: + print_entries(client.current_interaction, "modified") + if setup.extra or show_all: + print_entries(client.current_interaction, "extra") + + +class Total(_SingleHostCmd): + """ Show total number of managed and good entries from HOST """ + + def run(self, setup): + client = self.get_client(setup) + managed = client.current_interaction.total_count + good = client.current_interaction.good_count + print("Total managed entries: %d (good: %d)" % (managed, good)) + + +class Expire(_SingleHostCmd): + """ Toggle the expired/unexpired state of HOST """ + + def run(self, setup): + client = self.get_client(setup) + if client.expiration is None: + client.expiration = datetime.datetime.now() + print("%s expired." % client.name) + else: + client.expiration = None + print("%s un-expired." % client.name) + client.save() + + +class _ClientSelectCmd(Bcfg2.Options.Subcommand): + """ Base class for subcommands that display lists of clients """ + options = [ + Bcfg2.Options.Option("--fields", metavar="FIELD,FIELD,...", + help="Only display the listed fields", + type=Bcfg2.Options.Types.comma_list, + default=['name', 'time', 'state'])] + + def get_clients(self): + from Bcfg2.Reporting.models import Client + return Client.objects.exclude(current_interaction__isnull=True) + + def display(self, result, fields, extra=None): + if 'name' not in fields: + fields.insert(0, "name") + if not result: + print("No match found") + return + if extra is None: + extra = dict() + max_name = max(len(c.name) for c in result) + ffmt = [] + for field in fields: + if field == "name": + ffmt.append("%%-%ds" % max_name) + elif field == "time": + ffmt.append("%-19s") + else: + ffmt.append("%%-%ds" % len(field)) + fmt = " ".join(ffmt) + print(fmt % tuple(f.title() for f in fields)) + for client in result: + if not client.expiration: + print_fields(fields, client, fmt, + extra=extra.get(client, None)) + + +class Clients(_ClientSelectCmd): + """ Query hosts """ + options = _ClientSelectCmd.options + [ + Bcfg2.Options.BooleanOption( + "-c", "--clean", help="Show only clean hosts"), + Bcfg2.Options.BooleanOption( + "-d", "--dirty", help="Show only dirty hosts"), + Bcfg2.Options.BooleanOption( + "--stale", + help="Show hosts that haven't run in the last 24 hours")] + + def run(self, setup): + result = [] + show_all = not setup.stale and not setup.clean and not setup.dirty + for client in self.get_clients(): + interaction = client.current_interaction + if (show_all or + (setup.stale and interaction.isstale()) or + (setup.clean and interaction.isclean()) or + (setup.dirty and not interaction.isclean())): + result.append(client) + + self.display(result, setup.fields) + + +class Entries(_ClientSelectCmd): + """ Query hosts by entries """ + options = _ClientSelectCmd.options + [ + Bcfg2.Options.BooleanOption( + "--badentry", + help="Show hosts that have bad entries that match"), + Bcfg2.Options.BooleanOption( + "--modifiedentry", + help="Show hosts that have modified entries that match"), + Bcfg2.Options.BooleanOption( + "--extraentry", + help="Show hosts that have extra entries that match"), + Bcfg2.Options.PathOption( + "--file", type=argparse.FileType('r'), + help="Read TYPE:NAME pairs from the specified file instead of " + "from the command line"), + Bcfg2.Options.PositionalArgument( + "entries", metavar="TYPE:NAME", nargs="*")] + + def run(self, setup): + result = [] + if setup.file: + try: + entries = [l.strip().split(":") for l in setup.file] + except IOError: + err = sys.exc_info()[1] + print("Cannot read entries from %s: %s" % (setup.file.name, + err)) + return 2 + else: + entries = [a.split(":") for a in setup.entries] + + clients = self.get_clients() + if setup.badentry: + result = hosts_by_entry_type(clients, "bad", entries) + elif setup.modifiedentry: + result = hosts_by_entry_type(clients, "modified", entries) + elif setup.extraentry: + result = hosts_by_entry_type(clients, "extra", entries) + + self.display(result, setup.fields) + + +class Entry(_ClientSelectCmd): + """ Show the status of a single entry on all hosts """ + + options = _ClientSelectCmd.options + [ + Bcfg2.Options.PositionalArgument( + "entry", metavar="TYPE:NAME", nargs=1)] + + def run(self, setup): + from Bcfg2.Reporting.models import BaseEntry + result = [] + fields = setup.fields + if 'state' in fields: + fields.remove('state') + fields.append("entry state") + + etype, ename = setup.entry[0].split(":") + try: + entry_cls = BaseEntry.entry_from_type(etype) + except ValueError: + print("Unhandled/unknown type %s" % etype) + return 2 + + # TODO: batch fetch this. sqlite could break + extra = dict() + for client in self.get_clients(): + ents = entry_cls.objects.filter( + name=ename, + interaction=client.current_interaction) + if len(ents) == 0: + continue + extra[client] = {"entry state": ents[0].get_state_display(), + "reason": ents[0]} + result.append(client) + + self.display(result, fields, extra=extra) + + +class CLI(Bcfg2.Options.CommandRegistry): + """ CLI class for bcfg2-reports """ + + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, globals().values()) + parser = Bcfg2.Options.get_parser( + description="Query the Bcfg2 reporting subsystem", + components=[self]) + parser.parse() + + def run(self): + """ Run bcfg2-reports """ + return self.runcommand() diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 98226dc4e..96226c424 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -2,15 +2,12 @@ The base for the original DjangoORM (DBStats) """ -import os -import traceback from lxml import etree from datetime import datetime +import traceback from time import strptime - -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' -from Bcfg2 import settings - +import Bcfg2.Options +import Bcfg2.DBSettings from Bcfg2.Compat import md5 from Bcfg2.Reporting.Storage.base import StorageBase, StorageError from Bcfg2.Server.Plugin.exceptions import PluginExecutionError @@ -28,9 +25,13 @@ from Bcfg2.Reporting.Compat import transaction class DjangoORM(StorageBase): - def __init__(self, setup): - super(DjangoORM, self).__init__(setup) - self.size_limit = setup.get('reporting_file_limit') + options = StorageBase.options + [ + Bcfg2.Options.Common.repository, + Bcfg2.Options.Option( + cf=('reporting', 'file_limit'), + type=Bcfg2.Options.Types.size, + help='Reporting file size limit', + default=1024 * 1024)] def _import_default(self, entry, state, entrytype=None, defaults=None, mapping=None, boolean=None, xforms=None): @@ -185,7 +186,7 @@ class DjangoORM(StorageBase): act_dict['detail_type'] = PathEntry.DETAIL_DIFF cdata = entry.get('current_bdiff') if cdata: - if len(cdata) > self.size_limit: + if len(cdata) > Bcfg2.Options.setup.file_limit: act_dict['detail_type'] = PathEntry.DETAIL_SIZE_LIMIT act_dict['details'] = md5(cdata).hexdigest() else: @@ -365,7 +366,6 @@ class DjangoORM(StorageBase): def import_interaction(self, interaction): """Import the data into the backend""" - try: try: self._import_interaction(interaction) @@ -380,23 +380,21 @@ class DjangoORM(StorageBase): def validate(self): """Validate backend storage. Should be called once when loaded""" - - settings.read_config(repo=self.setup['repo']) - # verify our database schema try: - if self.setup['debug']: + if Bcfg2.Options.setup.debug: vrb = 2 - elif self.setup['verbose']: + elif Bcfg2.Options.setup.verbose: vrb = 1 else: vrb = 0 - management.call_command("syncdb", verbosity=vrb, interactive=False) - management.call_command("migrate", verbosity=vrb, interactive=False) + Bcfg2.DBSettings.sync_databases(verbosity=vrb, interactive=False) + Bcfg2.DBSettings.migrate_databases(verbosity=vrb, + interactive=False) except: - self.logger.error("Failed to update database schema: %s" % \ - traceback.format_exc().splitlines()[-1]) - raise StorageError + msg = "Failed to update database schema: %s" % sys.exc_info()[1] + self.logger.error(msg) + raise StorageError(msg) def GetExtra(self, client): """Fetch extra entries for a client""" diff --git a/src/lib/Bcfg2/Reporting/Storage/__init__.py b/src/lib/Bcfg2/Reporting/Storage/__init__.py index 85356fcfe..953104d4b 100644 --- a/src/lib/Bcfg2/Reporting/Storage/__init__.py +++ b/src/lib/Bcfg2/Reporting/Storage/__init__.py @@ -1,32 +1,3 @@ """ Public storage routines """ - -import traceback - -from Bcfg2.Reporting.Storage.base import StorageError, \ - StorageImportError - -def load_storage(storage_name, setup): - """ - Try to load the storage. Raise StorageImportError on failure - """ - try: - mod_name = "%s.%s" % (__name__, storage_name) - mod = getattr(__import__(mod_name).Reporting.Storage, storage_name) - except ImportError: - try: - mod = __import__(storage_name) - except: - raise StorageImportError("Unavailable") - try: - cls = getattr(mod, storage_name) - return cls(setup) - except: - raise StorageImportError("Storage unavailable: %s" % - traceback.format_exc().splitlines()[-1]) - -def load_storage_from_config(setup): - """Load the storage in the config... eventually""" - return load_storage('DjangoORM', setup) - diff --git a/src/lib/Bcfg2/Reporting/Storage/base.py b/src/lib/Bcfg2/Reporting/Storage/base.py index 92cc3a68b..771f755a1 100644 --- a/src/lib/Bcfg2/Reporting/Storage/base.py +++ b/src/lib/Bcfg2/Reporting/Storage/base.py @@ -2,28 +2,25 @@ The base for all Storage backends """ -import logging +import logging + class StorageError(Exception): """Generic StorageError""" pass -class StorageImportError(StorageError): - """Raised when a storage module fails to import""" - pass - class StorageBase(object): """The base for all storages""" + options = [] + __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] - def __init__(self, setup): + def __init__(self): """Do something here""" clsname = self.__class__.__name__ self.logger = logging.getLogger(clsname) self.logger.debug("Loading %s storage" % clsname) - self.setup = setup - self.encoding = setup['encoding'] def import_interaction(self, interaction): """Import the data into the backend""" @@ -48,4 +45,3 @@ class StorageBase(object): def GetCurrentEntry(self, client, e_type, e_name): """Get the current status of an entry on the client""" raise NotImplementedError - diff --git a/src/lib/Bcfg2/Reporting/Transport/DirectStore.py b/src/lib/Bcfg2/Reporting/Transport/DirectStore.py index 79d1b5aba..b9d17212e 100644 --- a/src/lib/Bcfg2/Reporting/Transport/DirectStore.py +++ b/src/lib/Bcfg2/Reporting/Transport/DirectStore.py @@ -5,18 +5,20 @@ import os import sys import time import threading +import Bcfg2.Options from Bcfg2.Reporting.Transport.base import TransportBase, TransportError -from Bcfg2.Reporting.Storage import load_storage_from_config from Bcfg2.Compat import Queue, Full, Empty, cPickle class DirectStore(TransportBase, threading.Thread): - def __init__(self, setup): - TransportBase.__init__(self, setup) + options = TransportBase.options + [Bcfg2.Options.Common.reporting_storage] + + def __init__(self): + TransportBase.__init__(self) threading.Thread.__init__(self) self.save_file = os.path.join(self.data, ".saved") - self.storage = load_storage_from_config(setup) + self.storage = Bcfg2.Options.setup.reporting_storage() self.storage.validate() self.queue = Queue(100000) @@ -30,10 +32,9 @@ class DirectStore(TransportBase, threading.Thread): def store(self, hostname, metadata, stats): try: - self.queue.put_nowait(dict( - hostname=hostname, - metadata=metadata, - stats=stats)) + self.queue.put_nowait(dict(hostname=hostname, + metadata=metadata, + stats=stats)) except Full: self.logger.warning("Reporting: Queue is full, " "dropping statistics") diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py index c7d5c512a..189967cb0 100644 --- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -9,6 +9,7 @@ import os import select import time import traceback +import Bcfg2.Options import Bcfg2.Server.FileMonitor from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError from Bcfg2.Reporting.Transport.base import TransportBase, TransportError @@ -16,8 +17,10 @@ from Bcfg2.Compat import cPickle class LocalFilesystem(TransportBase): - def __init__(self, setup): - super(LocalFilesystem, self).__init__(setup) + options = TransportBase.options + [Bcfg2.Options.Common.filemonitor] + + def __init__(self): + super(LocalFilesystem, self).__init__() self.work_path = "%s/work" % self.data self.debug_log("LocalFilesystem: work path %s" % self.work_path) @@ -42,24 +45,16 @@ class LocalFilesystem(TransportBase): def start_monitor(self, collector): """Start the file monitor. Most of this comes from BaseCore""" - setup = self.setup - try: - fmon = Bcfg2.Server.FileMonitor.available[setup['filemonitor']] - except KeyError: - self.logger.error("File monitor driver %s not available; " - "forcing to default" % setup['filemonitor']) - fmon = Bcfg2.Server.FileMonitor.available['default'] - if self.debug_flag: - self.fmon.set_debug(self.debug_flag) try: - self.fmon = fmon(debug=self.debug_flag) - self.logger.info("Using the %s file monitor" % - self.fmon.__class__.__name__) + self.fmon = Bcfg2.Server.FileMonitor.get_fam() except IOError: - msg = "Failed to instantiate file monitor %s" % \ - setup['filemonitor'] + msg = "Failed to instantiate fam driver %s" % \ + Bcfg2.Options.setup.filemonitor self.logger.error(msg, exc_info=1) raise TransportError(msg) + + if self.debug_flag: + self.fmon.set_debug(self.debug_flag) self.fmon.start() self.fmon.AddMonitor(self.work_path, self) @@ -154,7 +149,7 @@ class LocalFilesystem(TransportBase): """ try: if not self._phony_collector: - self._phony_collector = ReportingCollector(self.setup) + self._phony_collector = ReportingCollector() except ReportingError: raise TransportError except: @@ -176,4 +171,3 @@ class LocalFilesystem(TransportBase): self.logger.error("RPC method %s failed: %s" % (method, traceback.format_exc().splitlines()[-1])) raise TransportError - diff --git a/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py b/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py index 22d9af57e..7427c2e1d 100644 --- a/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py +++ b/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py @@ -9,9 +9,9 @@ import signal import platform import traceback import threading +import Bcfg2.Options from Bcfg2.Reporting.Transport.base import TransportBase, TransportError from Bcfg2.Compat import cPickle -from Bcfg2.Options import Option try: import redis @@ -34,9 +34,19 @@ class RedisTransport(TransportBase): STATS_KEY = 'bcfg2_statistics' COMMAND_KEY = 'bcfg2_command' - def __init__(self, setup): - super(RedisTransport, self).__init__(setup) - self._redis = None + options = TransportBase.options + [ + Bcfg2.Options.Option( + cf=('reporting', 'redis_host'), dest="reporting_redis_host", + default='127.0.0.1', help='Reporting Redis host'), + Bcfg2.Options.Option( + cf=('reporting', 'redis_port'), dest="reporting_redis_port", + default=6379, type=int, help='Reporting Redis port'), + Bcfg2.Options.Option( + cf=('reporting', 'redis_db'), dest="reporting_redis_db", + default=0, type=int, help='Reporting Redis DB')] + + def __init__(self): + super(RedisTransport, self).__init__() self._commands = None self.logger.error("Warning: RedisTransport is experimental") @@ -45,36 +55,15 @@ class RedisTransport(TransportBase): self.logger.error("redis python module is not available") raise TransportError - setup.update(dict( - reporting_redis_host=Option( - 'Redis Host', - default='127.0.0.1', - cf=('reporting', 'redis_host')), - reporting_redis_port=Option( - 'Redis Port', - default=6379, - cf=('reporting', 'redis_port')), - reporting_redis_db=Option( - 'Redis DB', - default=0, - cf=('reporting', 'redis_db')), - )) - setup.reparse() - - self._redis_host = setup.get('reporting_redis_host', '127.0.0.1') - try: - self._redis_port = int(setup.get('reporting_redis_port', 6379)) - except ValueError: - self.logger.error("Redis port must be an integer") - raise TransportError - self._redis_db = setup.get('reporting_redis_db', 0) - self._redis = redis.Redis(host=self._redis_host, - port=self._redis_port, db=self._redis_db) + self._redis = redis.Redis( + host=Bcfg2.Options.setup.reporting_redis_host, + port=Bcfg2.Options.setup.reporting_redis_port, + db=Bcfg2.Options.setup.reporting_redis_db) def start_monitor(self, collector): """Start the monitor. Eventaully start the command thread""" - self._commands = threading.Thread(target=self.monitor_thread, + self._commands = threading.Thread(target=self.monitor_thread, args=(self._redis, collector)) self._commands.start() @@ -129,7 +118,7 @@ class RedisTransport(TransportBase): channel = "%s%s" % (platform.node(), int(time.time())) pubsub.subscribe(channel) - self._redis.rpush(RedisTransport.COMMAND_KEY, + self._redis.rpush(RedisTransport.COMMAND_KEY, cPickle.dumps(RedisMessage(channel, method, args, kwargs))) resp = pubsub.listen() @@ -160,7 +149,7 @@ class RedisTransport(TransportBase): continue message = cPickle.loads(payload[1]) if not isinstance(message, RedisMessage): - self.logger.error("Message \"%s\" is not a RedisMessage" % + self.logger.error("Message \"%s\" is not a RedisMessage" % message) if not message.method in collector.storage.__class__.__rmi__ or\ @@ -192,5 +181,3 @@ class RedisTransport(TransportBase): self.logger.error("Unhandled exception in command thread: %s" % traceback.format_exc().splitlines()[-1]) self.logger.info("Command thread shutdown") - - diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py index 73bdd0b3a..04b574ed7 100644 --- a/src/lib/Bcfg2/Reporting/Transport/__init__.py +++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py @@ -1,35 +1,3 @@ """ Public transport routines """ - -import sys -from Bcfg2.Reporting.Transport.base import TransportError, \ - TransportImportError - - -def load_transport(transport_name, setup): - """ - Try to load the transport. Raise TransportImportError on failure - """ - try: - mod_name = "%s.%s" % (__name__, transport_name) - mod = getattr(__import__(mod_name).Reporting.Transport, transport_name) - except ImportError: - try: - mod = __import__(transport_name) - except: - raise TransportImportError("Error importing transport %s: %s" % - (transport_name, sys.exc_info()[1])) - try: - return getattr(mod, transport_name)(setup) - except: - raise TransportImportError("Error instantiating transport %s: %s" % - (transport_name, sys.exc_info()[1])) - - -def load_transport_from_config(setup): - """Load the transport in the config... eventually""" - try: - return load_transport(setup['reporting_transport'], setup) - except KeyError: - raise TransportImportError('Transport missing in config') diff --git a/src/lib/Bcfg2/Reporting/Transport/base.py b/src/lib/Bcfg2/Reporting/Transport/base.py index 530011e47..9a0a4262f 100644 --- a/src/lib/Bcfg2/Reporting/Transport/base.py +++ b/src/lib/Bcfg2/Reporting/Transport/base.py @@ -4,7 +4,8 @@ The base for all server -> collector Transports import os import sys -from Bcfg2.Server.Plugin import Debuggable +import Bcfg2.Options +from Bcfg2.Logger import Debuggable class TransportError(Exception): @@ -12,20 +13,18 @@ class TransportError(Exception): pass -class TransportImportError(TransportError): - """Raised when a transport fails to import""" - pass - - class TransportBase(Debuggable): """The base for all transports""" - def __init__(self, setup): + options = Debuggable.options + + def __init__(self): """Do something here""" clsname = self.__class__.__name__ Debuggable.__init__(self, name=clsname) self.debug_log("Loading %s transport" % clsname) - self.data = os.path.join(setup['repo'], 'Reporting', clsname) + self.data = os.path.join(Bcfg2.Options.setup.repository, 'Reporting', + clsname) if not os.path.exists(self.data): self.logger.info("%s does not exist, creating" % self.data) try: @@ -34,7 +33,6 @@ class TransportBase(Debuggable): self.logger.warning("Could not create %s: %s" % (self.data, sys.exc_info()[1])) self.logger.warning("The transport may not function properly") - self.setup = setup self.timeout = 2 def start_monitor(self, collector): diff --git a/src/lib/Bcfg2/Reporting/migrations/0002_convert_perms_to_mode.py b/src/lib/Bcfg2/Reporting/migrations/0002_convert_perms_to_mode.py index 668094cf5..37cdd146c 100644 --- a/src/lib/Bcfg2/Reporting/migrations/0002_convert_perms_to_mode.py +++ b/src/lib/Bcfg2/Reporting/migrations/0002_convert_perms_to_mode.py @@ -3,8 +3,7 @@ import datetime from south.db import db from south.v2 import SchemaMigration from django.db import models - -from Bcfg2 import settings +from django.conf import settings class Migration(SchemaMigration): diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 71fa66086..ae6f6731b 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -3,7 +3,7 @@ import sys from django.core.exceptions import ImproperlyConfigured try: - from django.db import models, backend, connection + from django.db import models, backend, connections except ImproperlyConfigured: e = sys.exc_info()[1] print("Reports: unable to import django models: %s" % e) @@ -12,6 +12,7 @@ except ImproperlyConfigured: from django.core.cache import cache from datetime import datetime, timedelta from Bcfg2.Compat import cPickle +from Bcfg2.DBSettings import get_db_label TYPE_GOOD = 0 @@ -61,7 +62,8 @@ def _quote(value): global _our_backend if not _our_backend: try: - _our_backend = backend.DatabaseOperations(connection) + _our_backend = backend.DatabaseOperations( + connections[get_db_label('Reporting')]) except TypeError: _our_backend = backend.DatabaseOperations() return _our_backend.quote_name(value) @@ -91,8 +93,8 @@ class InteractionManager(models.Manager): maxdate -- datetime object. Most recent date to pull. (default None) """ - from django.db import connection - cursor = connection.cursor() + from django.db import connections + cursor = connections[get_db_label('Reporting')].cursor() cfilter = "expiration is null" sql = 'select ri.id, x.client_id from ' + \ @@ -381,7 +383,7 @@ class BaseEntry(models.Model): @classmethod def entry_from_type(cls, etype): - for entry_cls in ENTRY_CLASSES: + for entry_cls in ENTRY_TYPES: if etype == entry_cls.ENTRY_TYPE: return entry_cls else: diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index a367d8ccb..8b197231c 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5 <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.5</span> + <span>Bcfg2 Version 1.4.0pre1</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index 33c78a5f0..6a314bd88 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -32,7 +32,7 @@ This is needed for Django versions less than 1.5 <td class='right_column_narrow'>{{ entry.bad_count }}</td> <td class='right_column_narrow'>{{ entry.modified_count }}</td> <td class='right_column_narrow'>{{ entry.extra_count }}</td> - <td class='right_column'><span {% if entry.timestamp|isstale:entry_max %}class='dirty-lineitem'{% endif %}>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</span></td> + <td class='right_column'><span {% if entry.isstale %}class='dirty-lineitem'{% endif %}>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</span></td> <td class='right_column_wide'> {% if entry.server %} <a href='{% add_url_filter server=entry.server %}'>{{ entry.server }}</a> diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 0ee5cd0d6..09aebc7fd 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -200,19 +200,6 @@ def build_metric_list(mdict): @register.filter -def isstale(timestamp, entry_max=None): - """ - Check for a stale timestamp - - Compares two timestamps and returns True if the - difference is greater then 24 hours. - """ - if not entry_max: - entry_max = datetime.now() - return entry_max - timestamp > timedelta(hours=24) - - -@register.filter def sort_interactions_by_name(value): """ Sort an interaction list by client name @@ -329,7 +316,11 @@ def determine_client_state(entry): dirty. If the client is reporting dirty, this will figure out just _how_ dirty and adjust the color accordingly. """ + if entry.isstale(): + return "stale-lineitem" if entry.state == 'clean': + if entry.extra_count > 0: + return "extra-lineitem" return "clean-lineitem" bad_percentage = 100 * (float(entry.bad_count) / entry.total_count) diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index c7c2a503f..0b8ed65cc 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -13,7 +13,7 @@ from django.http import \ from django.shortcuts import render_to_response, get_object_or_404 from django.core.urlresolvers import \ resolve, reverse, Resolver404, NoReverseMatch -from django.db import connection, DatabaseError +from django.db import DatabaseError from django.db.models import Q, Count from Bcfg2.Reporting.models import * diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py new file mode 100644 index 000000000..0807fb2b0 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin.py @@ -0,0 +1,1209 @@ +""" Subcommands and helpers for bcfg2-admin """ + +import os +import sys +import time +import glob +import stat +import random +import socket +import string +import getpass +import difflib +import tarfile +import argparse +import lxml.etree +import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.DBSettings +import Bcfg2.Server.Core +import Bcfg2.Client.Proxy +from Bcfg2.Server.Plugin import PullSource, Generator, MetadataConsistencyError +from Bcfg2.Utils import hostnames2ranges, Executor, safe_input +import Bcfg2.Server.Plugins.Metadata + +try: + from django.core.exceptions import ImproperlyConfigured + from django.core import management + import django.conf + import Bcfg2.Server.models + + HAS_DJANGO = True + try: + import south # pylint: disable=W0611 + HAS_REPORTS = True + except ImportError: + HAS_REPORTS = False +except ImportError: + HAS_DJANGO = False + HAS_REPORTS = False + + +class ccolors: # pylint: disable=C0103,W0232 + """ ANSI color escapes to make colorizing text easier """ + # pylint: disable=W1401 + ADDED = '\033[92m' + CHANGED = '\033[93m' + REMOVED = '\033[91m' + ENDC = '\033[0m' + # pylint: enable=W1401 + + @classmethod + def disable(cls): + """ Disable all coloration """ + cls.ADDED = '' + cls.CHANGED = '' + cls.REMOVED = '' + cls.ENDC = '' + + +def gen_password(length): + """Generates a random alphanumeric password with length characters.""" + chars = string.ascii_letters + string.digits + return "".join(random.choice(chars) for i in range(length)) + + +def print_table(rows, justify='left', hdr=True, vdelim=" ", padding=1): + """Pretty print a table + + rows - list of rows ([[row 1], [row 2], ..., [row n]]) + hdr - if True the first row is treated as a table header + vdelim - vertical delimiter between columns + padding - # of spaces around the longest element in the column + justify - may be left,center,right + + """ + hdelim = "=" + justify = {'left': str.ljust, + 'center': str.center, + 'right': str.rjust}[justify.lower()] + + # Calculate column widths (longest item in each column + # plus padding on both sides) + cols = list(zip(*rows)) + col_widths = [max([len(str(item)) + 2 * padding + for item in col]) for col in cols] + borderline = vdelim.join([w * hdelim for w in col_widths]) + + # Print out the table + print(borderline) + for row in rows: + print(vdelim.join([justify(str(item), width) + for (item, width) in zip(row, col_widths)])) + if hdr: + print(borderline) + hdr = False + + +class AdminCmd(Bcfg2.Options.Subcommand): # pylint: disable=W0223 + """ Base class for all bcfg2-admin modes """ + def setup(self): + """ Perform post-init (post-options parsing), pre-run setup + tasks """ + pass + + def errExit(self, emsg): + """ exit with an error """ + print(emsg) + raise SystemExit(1) + + +class _ServerAdminCmd(AdminCmd): # pylint: disable=W0223 + """ Base class for admin modes that run a Bcfg2 server. """ + __plugin_whitelist__ = None + __plugin_blacklist__ = None + + options = AdminCmd.options + Bcfg2.Server.Core.Core.options + + def __init__(self): + AdminCmd.__init__(self) + self.metadata = None + + def setup(self): + if self.__plugin_whitelist__ is not None: + Bcfg2.Options.setup.plugins = [ + p for p in Bcfg2.Options.setup.plugins + if p.name in self.__plugin_whitelist__] + elif self.__plugin_blacklist__ is not None: + Bcfg2.Options.setup.plugins = [ + p for p in Bcfg2.Options.setup.plugins + if p.name not in self.__plugin_blacklist__] + + try: + self.core = Bcfg2.Server.Core.Core() + except Bcfg2.Server.Core.CoreInitError: + msg = sys.exc_info()[1] + self.errExit("Core load failed: %s" % msg) + self.core.load_plugins() + self.core.fam.handle_event_set() + self.metadata = self.core.metadata + + def shutdown(self): + self.core.shutdown() + + +class _ProxyAdminCmd(AdminCmd): # pylint: disable=W0223 + """ Base class for admin modes that proxy to a running Bcfg2 server """ + + options = AdminCmd.options + Bcfg2.Client.Proxy.ComponentProxy.options + + def __init__(self): + AdminCmd.__init__(self) + self.proxy = None + + def setup(self): + self.proxy = Bcfg2.Client.Proxy.ComponentProxy() + + +class Backup(AdminCmd): + """ Make a backup of the Bcfg2 repository """ + + options = AdminCmd.options + [Bcfg2.Options.Common.repository] + + def run(self, setup): + timestamp = time.strftime('%Y%m%d%H%M%S') + datastore = setup.repository + fmt = 'gz' + mode = 'w:' + fmt + filename = timestamp + '.tar' + '.' + fmt + out = tarfile.open(os.path.join(datastore, filename), mode=mode) + out.add(datastore, os.path.basename(datastore)) + out.close() + print("Archive %s was stored under %s" % (filename, datastore)) + + +class Client(_ServerAdminCmd): + """ Create, modify, delete, or list client entries """ + + __plugin_whitelist__ = ["Metadata"] + options = _ServerAdminCmd.options + [ + Bcfg2.Options.PositionalArgument( + "mode", + choices=["add", "del", "delete", "remove", "rm", "up", "update", + "list"]), + Bcfg2.Options.PositionalArgument("hostname", nargs='?'), + Bcfg2.Options.PositionalArgument("attributes", metavar="KEY=VALUE", + nargs='*')] + + valid_attribs = ['profile', 'uuid', 'password', 'floating', 'secure', + 'address', 'auth'] + + def get_attribs(self, setup): + """ Get attributes for adding or updating a client from the command + line """ + attr_d = {} + for i in setup.attributes: + attr, val = i.split('=', 1) + if attr not in self.valid_attribs: + print("Attribute %s unknown. Valid attributes: %s" % + (attr, self.valid_attribs)) + raise SystemExit(1) + attr_d[attr] = val + return attr_d + + def run(self, setup): + if setup.mode != 'list' and not setup.hostname: + self.parser.error("<hostname> is required in %s mode" % setup.mode) + elif setup.mode == 'list' and setup.hostname: + self.logger.warning("<hostname> is not honored in list mode") + + if setup.mode == 'list': + for client in self.metadata.list_clients(): + print(client) + else: + include_attribs = True + if setup.mode == 'add': + func = self.metadata.add_client + action = "adding" + elif setup.mode in ['up', 'update']: + func = self.metadata.update_client + action = "updating" + elif setup.mode in ['del', 'delete', 'rm', 'remove']: + func = self.metadata.remove_client + include_attribs = False + action = "deleting" + + if include_attribs: + args = (setup.hostname, self.get_attribs(setup)) + else: + args = (setup.hostname,) + try: + func(*args) + except MetadataConsistencyError: + err = sys.exc_info()[1] + self.errExit("Error %s client %s: %s" % (setup.hostname, + action, err)) + + +class Compare(AdminCmd): + """ Compare two hosts or two versions of a host specification """ + + help = "Given two XML files (as produced by bcfg2-info build or bcfg2 " + \ + "-qnc) or two directories containing XML files (as produced by " + \ + "bcfg2-info buildall or bcfg2-info builddir), output a detailed, " + \ + "Bcfg2-centric diff." + + options = AdminCmd.options + [ + Bcfg2.Options.Option( + "-d", "--diff-lines", type=int, + help="Show only N lines of a diff"), + Bcfg2.Options.BooleanOption( + "-c", "--color", help="Use colors even if not run from a TTY"), + Bcfg2.Options.BooleanOption( + "-q", "--quiet", + help="Only show that entries differ, not how they differ"), + Bcfg2.Options.PathOption("path1", metavar="<file-or-dir>"), + Bcfg2.Options.PathOption("path2", metavar="<file-or-dir>")] + + changes = dict() + + def removed(self, msg, host): + """ Record a removed element """ + self.record("%sRemoved: %s%s" % (ccolors.REMOVED, msg, ccolors.ENDC), + host) + + def added(self, msg, host): + """ Record an removed element """ + self.record("%sAdded: %s%s" % (ccolors.ADDED, msg, ccolors.ENDC), host) + + def changed(self, msg, host): + """ Record a changed element """ + self.record("%sChanged: %s%s" % (ccolors.CHANGED, msg, ccolors.ENDC), + host) + + def record(self, msg, host): + """ Record a new removed/added/changed message for the given + host """ + if msg not in self.changes: + self.changes[msg] = [host] + else: + self.changes[msg].append(host) + + def udiff(self, lines1, lines2, **kwargs): + """ get a unified diff with control lines stripped """ + lines = None + if "lines" in kwargs: + if kwargs['lines'] is not None: + lines = int(kwargs['lines']) + del kwargs['lines'] + if lines == 0: + return [] + kwargs['n'] = 0 + diff = [] + for line in difflib.unified_diff(lines1, lines2, **kwargs): + if (line.startswith("--- ") or line.startswith("+++ ") or + line.startswith("@@ ")): + continue + if lines is not None and len(diff) > lines: + diff.append(" ...") + break + if line.startswith("+"): + diff.extend(" %s%s%s" % (ccolors.ADDED, l, ccolors.ENDC) + for l in line.splitlines()) + elif line.startswith("-"): + diff.extend(" %s%s%s" % (ccolors.REMOVED, l, ccolors.ENDC) + for l in line.splitlines()) + return diff + + def _bundletype(self, el): + """ Get a human-friendly representation of the type of the + given bundle -- independent or not """ + if el.get("tag") == "Independent": + return "Independent bundle" + else: + return "Bundle" + + def _get_filelists(self, setup): + """ Get a list of 2-tuples of files to compare """ + files = [] + if os.path.isdir(setup.path1) and os.path.isdir(setup.path1): + for fpath in glob.glob(os.path.join(setup.path1, '*')): + fname = os.path.basename(fpath) + if os.path.exists(os.path.join(setup.path2, fname)): + files.append((os.path.join(setup.path1, fname), + os.path.join(setup.path2, fname))) + else: + if fname.endswith(".xml"): + host = fname[0:-4] + else: + host = fname + self.removed(host, '') + for fpath in glob.glob(os.path.join(setup.path2, '*')): + fname = os.path.basename(fpath) + if not os.path.exists(os.path.join(setup.path1, fname)): + if fname.endswith(".xml"): + host = fname[0:-4] + else: + host = fname + self.added(host, '') + elif os.path.isfile(setup.path1) and os.path.isfile(setup.path2): + files.append((setup.path1, setup.path2)) + else: + self.errExit("Cannot diff a file and a directory") + return files + + def run(self, setup): # pylint: disable=R0912,R0914,R0915 + if not sys.stdout.isatty() and not setup.color: + ccolors.disable() + + files = self._get_filelists(setup) + for file1, file2 in files: + host = None + if os.path.basename(file1) == os.path.basename(file2): + fname = os.path.basename(file1) + if fname.endswith(".xml"): + host = fname[0:-4] + else: + host = fname + + xdata1 = lxml.etree.parse(file1).getroot() + xdata2 = lxml.etree.parse(file2).getroot() + + elements1 = dict() + elements2 = dict() + bundles1 = [el.get("name") for el in xdata1.iterchildren()] + bundles2 = [el.get("name") for el in xdata2.iterchildren()] + for el in xdata1.iterchildren(): + if el.get("name") not in bundles2: + self.removed("%s %s" % (self._bundletype(el), + el.get("name")), + host) + for el in xdata2.iterchildren(): + if el.get("name") not in bundles1: + self.added("%s %s" % (self._bundletype(el), + el.get("name")), + host) + + for bname in bundles1: + bundle = xdata1.find("*[@name='%s']" % bname) + for el in bundle.getchildren(): + elements1["%s:%s" % (el.tag, el.get("name"))] = el + for bname in bundles2: + bundle = xdata2.find("*[@name='%s']" % bname) + for el in bundle.getchildren(): + elements2["%s:%s" % (el.tag, el.get("name"))] = el + + for el in elements1.values(): + elid = "%s:%s" % (el.tag, el.get("name")) + if elid not in elements2: + self.removed("Element %s" % elid, host) + else: + el2 = elements2[elid] + if (el.getparent().get("name") != + el2.getparent().get("name")): + self.changed( + "Element %s was in bundle %s, " + "now in bundle %s" % (elid, + el.getparent().get("name"), + el2.getparent().get("name")), + host) + attr1 = sorted(["%s=\"%s\"" % (attr, el.get(attr)) + for attr in el.attrib]) + attr2 = sorted(["%s=\"%s\"" % (attr, el.get(attr)) + for attr in el2.attrib]) + if attr1 != attr2: + err = ["Element %s has different attributes" % elid] + if not setup.quiet: + err.extend(self.udiff(attr1, attr2)) + self.changed("\n".join(err), host) + + if el.text != el2.text: + if el.text is None: + self.changed("Element %s content was added" % elid, + host) + elif el2.text is None: + self.changed("Element %s content was removed" % + elid, host) + else: + err = ["Element %s has different content" % + elid] + if not setup.quiet: + err.extend( + self.udiff(el.text.splitlines(), + el2.text.splitlines(), + lines=setup.diff_lines)) + self.changed("\n".join(err), host) + + for el in elements2.values(): + elid = "%s:%s" % (el.tag, el.get("name")) + if elid not in elements2: + self.removed("Element %s" % elid, host) + + for change, hosts in self.changes.items(): + hlist = [h for h in hosts if h is not None] + if len(files) > 1 and len(hlist): + print("===== %s =====" % + "\n ".join(hostnames2ranges(hlist))) + print(change) + if len(files) > 1 and len(hlist): + print("") + + +class Help(AdminCmd, Bcfg2.Options.HelpCommand): + """ Get help on a specific subcommand """ + def command_registry(self): + return CLI.commands + + def run(self, setup): + Bcfg2.Options.HelpCommand.run(self, setup) + + +class Init(AdminCmd): + """Interactively initialize a new repository.""" + + options = AdminCmd.options + [ + Bcfg2.Options.Common.repository, Bcfg2.Options.Common.plugins] + + # default config file + config = '''[server] +repository = %s +plugins = %s +# Uncomment the following to listen on all interfaces +#listen_all = true + +[database] +#engine = sqlite3 +# 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. +#name = +# Or path to database file if using sqlite3. +#<repository>/etc/bcfg2.sqlite is default path if left empty +#user = +# Not used with sqlite3. +#password = +# Not used with sqlite3. +#host = +# Not used with sqlite3. +#port = + +[reporting] +transport = LocalFilesystem + +[communication] +password = %s +certificate = %s +key = %s +ca = %s + +[components] +bcfg2 = %s +''' + + # Default groups + groups = '''<Groups> + <Group profile='true' public='true' default='true' name='basic'/> +</Groups> +''' + + # Default contents of clients.xml + clients = '''<Clients> + <Client profile="basic" name="%s"/> +</Clients> +''' + + def __init__(self): + AdminCmd.__init__(self) + self.data = dict() + + def _set_defaults(self, setup): + """Set default parameters.""" + self.data['plugins'] = setup.plugins + self.data['configfile'] = setup.config + self.data['repopath'] = setup.repository + self.data['password'] = gen_password(8) + self.data['shostname'] = socket.getfqdn() + self.data['server_uri'] = "https://%s:6789" % self.data['shostname'] + self.data['country'] = 'US' + self.data['state'] = 'Illinois' + self.data['location'] = 'Argonne' + if os.path.exists("/etc/pki/tls"): + self.data['keypath'] = "/etc/pki/tls/private/bcfg2.key" + self.data['certpath'] = "/etc/pki/tls/certs/bcfg2.crt" + elif os.path.exists("/etc/ssl"): + self.data['keypath'] = "/etc/ssl/bcfg2.key" + self.data['certpath'] = "/etc/ssl/bcfg2.crt" + else: + basepath = os.path.dirname(self.data['configfile']) + self.data['keypath'] = os.path.join(basepath, "bcfg2.key") + self.data['certpath'] = os.path.join(basepath, 'bcfg2.crt') + + def input_with_default(self, msg, default_name): + """ Prompt for input with the given message, taking the + default from ``self.data`` """ + val = safe_input("%s [%s]: " % (msg, self.data[default_name])) + if val: + self.data[default_name] = val + + def run(self, setup): + self._set_defaults(setup) + + # Prompt the user for input + self._prompt_server() + self._prompt_config() + self._prompt_repopath() + self._prompt_password() + self._prompt_keypath() + self._prompt_certificate() + + # Initialize the repository + self.init_repo() + + def _prompt_server(self): + """Ask for the server name and URI.""" + self.input_with_default("What is the server's hostname", 'shostname') + # reset default server URI + self.data['server_uri'] = "https://%s:6789" % self.data['shostname'] + self.input_with_default("Server location", 'server_uri') + + def _prompt_config(self): + """Ask for the configuration file path.""" + self.input_with_default("Path to Bcfg2 configuration", 'configfile') + + def _prompt_repopath(self): + """Ask for the repository path.""" + while True: + self.input_with_default("Location of Bcfg2 repository", 'repopath') + if os.path.isdir(self.data['repopath']): + response = safe_input("Directory %s exists. Overwrite? [y/N]:" + % self.data['repopath']) + if response.lower().strip() == 'y': + break + else: + break + + def _prompt_password(self): + """Ask for a password or generate one if none is provided.""" + newpassword = getpass.getpass( + "Input password used for communication verification " + "(without echoing; leave blank for random): ").strip() + if len(newpassword) != 0: + self.data['password'] = newpassword + + def _prompt_certificate(self): + """Ask for the key details (country, state, and location).""" + print("The following questions affect SSL certificate generation.") + print("If no data is provided, the default values are used.") + self.input_with_default("Country code for certificate", 'country') + self.input_with_default("State or Province Name (full name) for " + "certificate", 'state') + self.input_with_default("Locality Name (e.g., city) for certificate", + 'location') + + def _prompt_keypath(self): + """ Ask for the key pair location. Try to use sensible + defaults depending on the OS """ + self.input_with_default("Path where Bcfg2 server private key will be " + "created", 'keypath') + self.input_with_default("Path where Bcfg2 server cert will be created", + 'certpath') + + def _init_plugins(self): + """Initialize each plugin-specific portion of the repository.""" + for plugin in self.data['plugins']: + kwargs = dict() + if issubclass(plugin, Bcfg2.Server.Plugins.Metadata.Metadata): + kwargs.update( + dict(groups_xml=self.groups, + clients_xml=self.clients % self.data['shostname'])) + plugin.init_repo(self.data['repopath'], **kwargs) + + def create_conf(self): + """ create the config file """ + confdata = self.config % ( + self.data['repopath'], + ','.join(p.__name__ for p in self.data['plugins']), + self.data['password'], + self.data['certpath'], + self.data['keypath'], + self.data['certpath'], + self.data['server_uri']) + + # Don't overwrite existing bcfg2.conf file + if os.path.exists(self.data['configfile']): + result = safe_input("\nWarning: %s already exists. " + "Overwrite? [y/N]: " % self.data['configfile']) + if result not in ['Y', 'y']: + print("Leaving %s unchanged" % self.data['configfile']) + return + try: + open(self.data['configfile'], "w").write(confdata) + os.chmod(self.data['configfile'], + stat.S_IRUSR | stat.S_IWUSR) # 0600 + except: # pylint: disable=W0702 + self.errExit("Error trying to write configuration file '%s': %s" % + (self.data['configfile'], sys.exc_info()[1])) + + def init_repo(self): + """Setup a new repo and create the content of the + configuration file.""" + # Create the repository + path = os.path.join(self.data['repopath'], 'etc') + try: + os.makedirs(path) + self._init_plugins() + print("Repository created successfuly in %s" % + self.data['repopath']) + except OSError: + print("Failed to create %s." % path) + + # Create the configuration file and SSL key + self.create_conf() + self.create_key() + + def create_key(self): + """Creates a bcfg2.key at the directory specifed by keypath.""" + cmd = Executor(timeout=120) + subject = "/C=%s/ST=%s/L=%s/CN=%s'" % ( + self.data['country'], self.data['state'], self.data['location'], + self.data['shostname']) + key = cmd.run(["openssl", "req", "-batch", "-x509", "-nodes", + "-subj", subject, "-days", "1000", + "-newkey", "rsa:2048", + "-keyout", self.data['keypath'], "-noout"]) + if not key.success: + print("Error generating key: %s" % key.error) + return + os.chmod(self.data['keypath'], stat.S_IRUSR | stat.S_IWUSR) # 0600 + csr = cmd.run(["openssl", "req", "-batch", "-new", "-subj", subject, + "-key", self.data['keypath']]) + if not csr.success: + print("Error generating certificate signing request: %s" % + csr.error) + return + cert = cmd.run(["openssl", "x509", "-req", "-days", "1000", + "-signkey", self.data['keypath'], + "-out", self.data['certpath']], + inputdata=csr.stdout) + if not cert.success: + print("Error signing certificate: %s" % cert.error) + return + + +class Minestruct(_ServerAdminCmd): + """ Extract extra entry lists from statistics """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.PathOption( + "-f", "--outfile", type=argparse.FileType('w'), default=sys.stdout, + help="Write to the given file"), + Bcfg2.Options.Option( + "-g", "--groups", help="Only build config for groups", + type=Bcfg2.Options.Types.colon_list, default=[]), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + extra = set() + for source in self.core.plugins_by_type(PullSource): + for item in source.GetExtra(setup.hostname): + extra.add(item) + except: # pylint: disable=W0702 + self.errExit("Failed to find extra entry info for client %s: %s" % + (setup.hostname, sys.exc_info()[1])) + root = lxml.etree.Element("Base") + self.logger.info("Found %d extra entries" % len(extra)) + add_point = root + for grp in setup.groups: + add_point = lxml.etree.SubElement(add_point, "Group", name=grp) + for tag, name in extra: + self.logger.info("%s: %s" % (tag, name)) + lxml.etree.SubElement(add_point, tag, name=name) + + lxml.etree.ElementTree(root).write(setup.outfile, pretty_print=True) + + +class Perf(_ProxyAdminCmd): + """ Get performance data from server """ + + def run(self, setup): + output = [('Name', 'Min', 'Max', 'Mean', 'Count')] + data = self.proxy.get_statistics() + for key in sorted(data.keys()): + output.append( + (key, ) + + tuple(["%.06f" % item + for item in data[key][:-1]] + [data[key][-1]])) + print_table(output) + + +class Pull(_ServerAdminCmd): + """ Retrieves entries from clients and integrates the information + into the repository """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.Common.interactive, + Bcfg2.Options.BooleanOption( + "-s", "--stdin", + help="Read lists of <hostname> <entrytype> <entryname> from stdin " + "instead of the command line"), + Bcfg2.Options.PositionalArgument("hostname", nargs='?'), + Bcfg2.Options.PositionalArgument("entrytype", nargs='?'), + Bcfg2.Options.PositionalArgument("entryname", nargs='?')] + + def __init__(self): + _ServerAdminCmd.__init__(self) + self.interactive = False + + def setup(self): + if (not Bcfg2.Options.setup.stdin and + not (Bcfg2.Options.setup.hostname and + Bcfg2.Options.setup.entrytype and + Bcfg2.Options.setup.entryname)): + print("You must specify either --stdin or a hostname, entry type, " + "and entry name on the command line.") + self.errExit(self.usage()) + _ServerAdminCmd.setup(self) + + def run(self, setup): + self.interactive = setup.interactive + if setup.stdin: + for line in sys.stdin: + try: + self.PullEntry(*line.split(None, 3)) + except SystemExit: + print(" for %s" % line) + except: + print("Bad entry: %s" % line.strip()) + else: + self.PullEntry(setup.hostname, setup.entrytype, setup.entryname) + + def BuildNewEntry(self, client, etype, ename): + """Construct a new full entry for + given client/entry from statistics. + """ + new_entry = {'type': etype, 'name': ename} + pull_sources = self.core.plugins_by_type(PullSource) + for plugin in pull_sources: + try: + (owner, group, mode, contents) = \ + plugin.GetCurrentEntry(client, etype, ename) + break + except Bcfg2.Server.Plugin.PluginExecutionError: + if plugin == pull_sources[-1]: + self.errExit("Pull Source failure; could not fetch " + "current state") + + try: + data = {'owner': owner, + 'group': group, + 'mode': mode, + 'text': contents} + except UnboundLocalError: + self.errExit("Unable to build entry") + for key, val in list(data.items()): + if val: + new_entry[key] = val + return new_entry + + def Choose(self, choices): + """Determine where to put pull data.""" + if self.interactive: + for choice in choices: + print("Plugin returned choice:") + if id(choice) == id(choices[0]): + print("(current entry) ") + if choice.all: + print(" => global entry") + elif choice.group: + print(" => group entry: %s (prio %d)" % + (choice.group, choice.prio)) + else: + print(" => host entry: %s" % (choice.hostname)) + + # flush input buffer + ans = safe_input("Use this entry? [yN]: ") in ['y', 'Y'] + if ans: + return choice + return False + else: + if not choices: + return False + return choices[0] + + def PullEntry(self, client, etype, ename): + """Make currently recorded client state correct for entry.""" + new_entry = self.BuildNewEntry(client, etype, ename) + + meta = self.core.build_metadata(client) + # Find appropriate plugin in core + glist = [gen for gen in self.core.plugins_by_type(Generator) + if ename in gen.Entries.get(etype, {})] + if len(glist) != 1: + self.errExit("Got wrong numbers of matching generators for entry:" + "%s" % ([g.name for g in glist])) + plugin = glist[0] + if not isinstance(plugin, Bcfg2.Server.Plugin.PullTarget): + self.errExit("Configuration upload not supported by plugin %s" % + plugin.name) + try: + choices = plugin.AcceptChoices(new_entry, meta) + specific = self.Choose(choices) + if specific: + plugin.AcceptPullData(specific, new_entry, self.logger) + except Bcfg2.Server.Plugin.PluginExecutionError: + self.errExit("Configuration upload not supported by plugin %s" % + plugin.name) + + # Commit if running under a VCS + for vcsplugin in list(self.core.plugins.values()): + if isinstance(vcsplugin, Bcfg2.Server.Plugin.Version): + files = "%s/%s" % (plugin.data, ename) + comment = 'file "%s" pulled from host %s' % (files, client) + vcsplugin.commit_data([files], comment) + + +class _ReportsCmd(AdminCmd): # pylint: disable=W0223 + """ Base command for all admin modes dealing with the reporting + subsystem """ + def __init__(self): + AdminCmd.__init__(self) + self.reports_entries = () + self.reports_classes = () + + def setup(self): + # this has to be imported after options are parsed, because + # Django finalizes its settings as soon as it's loaded, which + # means that if we import this before Bcfg2.DBSettings has + # been populated, Django gets a null configuration, and + # subsequent updates to Bcfg2.DBSettings won't help. + import Bcfg2.Reporting.models # pylint: disable=W0621 + self.reports_entries = (Bcfg2.Reporting.models.Group, + Bcfg2.Reporting.models.Bundle, + Bcfg2.Reporting.models.FailureEntry, + Bcfg2.Reporting.models.ActionEntry, + Bcfg2.Reporting.models.PathEntry, + Bcfg2.Reporting.models.PackageEntry, + Bcfg2.Reporting.models.PathEntry, + Bcfg2.Reporting.models.ServiceEntry) + self.reports_classes = self.reports_entries + ( + Bcfg2.Reporting.models.Client, + Bcfg2.Reporting.models.Interaction, + Bcfg2.Reporting.models.Performance) + + +if HAS_DJANGO: + class _DjangoProxyCmd(AdminCmd): + """ Base for admin modes that proxy a command through the + Django management system """ + command = None + args = [] + + def run(self, _): + '''Call a django command''' + if self.command is not None: + command = self.command + else: + command = self.__class__.__name__.lower() + args = [command] + self.args + management.call_command(*args) + + class DBShell(_DjangoProxyCmd): + """ Call the Django 'dbshell' command on the database """ + + class Shell(_DjangoProxyCmd): + """ Call the Django 'shell' command on the database """ + + class ValidateDB(_DjangoProxyCmd): + """ Call the Django 'validate' command on the database """ + command = "validate" + + class Syncdb(AdminCmd): + """ Sync the Django ORM with the configured database """ + + def run(self, setup): + Bcfg2.Server.models.load_models() + try: + Bcfg2.DBSettings.sync_databases( + interactive=False, + verbosity=setup.verbose + setup.debug) + except ImproperlyConfigured: + err = sys.exc_info()[1] + self.logger.error("Django configuration problem: %s" % err) + raise SystemExit(1) + except: + err = sys.exc_info()[1] + self.logger.error("Database update failed: %s" % err) + raise SystemExit(1) + + +if HAS_REPORTS: + import datetime + + class ScrubReports(_ReportsCmd): + """ Perform a thorough scrub and cleanup of the Reporting + database """ + + def setup(self): + _ReportsCmd.setup(self) + # this has to be imported after options are parsed, + # because Django finalizes its settings as soon as it's + # loaded, which means that if we import this before + # Bcfg2.DBSettings has been populated, Django gets a null + # configuration, and subsequent updates to + # Bcfg2.DBSettings won't help. + from Bcfg2.Reporting.Compat import transaction + self.run = transaction.atomic(self.run) + + def run(self, _): # pylint: disable=E0202 + # Cleanup unused entries + for cls in self.reports_entries: + try: + start_count = cls.objects.count() + cls.prune_orphans() + self.logger.info("Pruned %d %s records" % + (start_count - cls.objects.count(), + cls.__name__)) + except: # pylint: disable=W0702 + print("Failed to prune %s: %s" % + (cls.__name__, sys.exc_info()[1])) + + class InitReports(AdminCmd): + """ Initialize the Reporting database """ + def run(self, setup): + verbose = setup.verbose + setup.debug + try: + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=verbose) + Bcfg2.DBSettings.migrate_databases(interactive=False, + verbosity=verbose) + except: # pylint: disable=W0702 + self.errExit("%s failed: %s" % + (self.__class__.__name__.title(), + sys.exc_info()[1])) + + class UpdateReports(InitReports): + """ Apply updates to the reporting database """ + + class ReportsStats(_ReportsCmd): + """ Print Reporting database statistics """ + def run(self, _): + for cls in self.reports_classes: + print("%s has %s records" % (cls.__name__, + cls.objects.count())) + + class PurgeReports(_ReportsCmd): + """ Purge records from the Reporting database """ + + options = AdminCmd.options + [ + Bcfg2.Options.Option("--client", help="Client to operate on"), + Bcfg2.Options.Option("--days", type=int, metavar='N', + help="Records older than N days"), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption("--expired", + help="Expired clients only"), + Bcfg2.Options.Option("--state", help="Purge entries in state", + choices=['dirty', 'clean', 'modified']), + required=False)] + + def run(self, setup): + if setup.days: + maxdate = datetime.datetime.now() - \ + datetime.timedelta(days=setup.days) + else: + maxdate = None + + starts = {} + for cls in self.reports_classes: + starts[cls] = cls.objects.count() + if setup.expired: + self.purge_expired(maxdate) + else: + self.purge(setup.client, maxdate, setup.state) + for cls in self.reports_classes: + self.logger.info("Purged %s %s records" % + (starts[cls] - cls.objects.count(), + cls.__name__)) + + def purge(self, client=None, maxdate=None, state=None): + '''Purge historical data from the database''' + # indicates whether or not a client should be deleted + filtered = False + + if not client and not maxdate and not state: + self.errExit("Refusing to prune all data. Specify an option " + "to %s" % self.__class__.__name__.lower()) + + ipurge = Bcfg2.Reporting.models.Interaction.objects + if client: + try: + cobj = Bcfg2.Reporting.models.Client.objects.get( + name=client) + ipurge = ipurge.filter(client=cobj) + except Bcfg2.Reporting.models.Client.DoesNotExist: + self.errExit("Client %s not in database" % client) + self.logger.debug("Filtering by client: %s" % client) + + if maxdate: + filtered = True + self.logger.debug("Filtering by maxdate: %s" % maxdate) + ipurge = ipurge.filter(timestamp__lt=maxdate) + + if django.conf.settings.DATABASES['default']['ENGINE'] == \ + 'django.db.backends.sqlite3': + grp_limit = 100 + else: + grp_limit = 1000 + if state: + filtered = True + self.logger.debug("Filtering by state: %s" % state) + ipurge = ipurge.filter(state=state) + + count = ipurge.count() + rnum = 0 + try: + while rnum < count: + grp = list(ipurge[:grp_limit].values("id")) + # just in case... + if not grp: + break + Bcfg2.Reporting.models.Interaction.objects.filter( + id__in=[x['id'] for x in grp]).delete() + rnum += len(grp) + self.logger.debug("Deleted %s of %s" % (rnum, count)) + except: # pylint: disable=W0702 + self.logger.error("Failed to remove interactions: %s" % + sys.exc_info()[1]) + + # Prune any orphaned ManyToMany relations + for m2m in self.reports_entries: + self.logger.debug("Pruning any orphaned %s objects" % + m2m.__name__) + m2m.prune_orphans() + + if client and not filtered: + # Delete the client, ping data is automatic + try: + self.logger.debug("Purging client %s" % client) + cobj.delete() + except: # pylint: disable=W0702 + self.logger.error("Failed to delete client %s: %s" % + (client, sys.exc_info()[1])) + + def purge_expired(self, maxdate=None): + """ Purge expired clients from the Reporting database """ + + if maxdate: + if not isinstance(maxdate, datetime.datetime): + raise TypeError("maxdate is not a DateTime object") + self.logger.debug("Filtering by maxdate: %s" % maxdate) + clients = Bcfg2.Reporting.models.Client.objects.filter( + expiration__lt=maxdate) + else: + clients = Bcfg2.Reporting.models.Client.objects.filter( + expiration__isnull=False) + + for client in clients: + self.logger.debug("Purging client %s" % client) + Bcfg2.Reporting.models.Interaction.objects.filter( + client=client).delete() + client.delete() + + class ReportsSQLAll(_DjangoProxyCmd): + """ Call the Django 'sqlall' command on the Reporting database """ + args = ["Reporting"] + + +class Viz(_ServerAdminCmd): + """ Produce graphviz diagrams of metadata structures """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.BooleanOption( + "-H", "--includehosts", + help="Include hosts in the viz output"), + Bcfg2.Options.BooleanOption( + "-b", "--includebundles", + help="Include bundles in the viz output"), + Bcfg2.Options.BooleanOption( + "-k", "--includekey", + help="Show a key for different digraph shapes"), + Bcfg2.Options.Option( + "-c", "--only-client", metavar="<hostname>", + help="Only show groups and bundles for the named client"), + Bcfg2.Options.PathOption( + "-o", "--outfile", + help="Write viz output to an output file")] + + colors = ['steelblue1', 'chartreuse', 'gold', 'magenta', + 'indianred1', 'limegreen', 'orange1', 'lightblue2', + 'green1', 'blue1', 'yellow1', 'darkturquoise', 'gray66'] + + __plugin_blacklist__ = ['DBStats', 'Cfg', 'Pkgmgr', 'Packages', 'Rules', + 'Decisions', 'Deps', 'Git', 'Svn', 'Fossil', 'Bzr', + 'Bundler'] + + def run(self, setup): + if setup.outfile: + fmt = setup.outfile.split('.')[-1] + else: + fmt = 'png' + + exc = Executor() + cmd = ["dot", "-T", fmt] + if setup.outfile: + cmd.extend(["-o", setup.outfile]) + inputlist = ["digraph groups {", + '\trankdir="LR";', + self.metadata.viz(setup.includehosts, + setup.includebundles, + setup.includekey, + setup.only_client, + self.colors)] + if setup.includekey: + inputlist.extend( + ["\tsubgraph cluster_key {", + '\tstyle="filled";', + '\tcolor="lightblue";', + '\tBundle [ shape="septagon" ];', + '\tGroup [shape="ellipse"];', + '\tGroup Category [shape="trapezium"];\n', + '\tProfile [style="bold", shape="ellipse"];', + '\tHblock [label="Host1|Host2|Host3",shape="record"];', + '\tlabel="Key";', + "\t}"]) + inputlist.append("}") + idata = "\n".join(inputlist) + try: + result = exc.run(cmd, inputdata=idata) + except OSError: + # on some systems (RHEL 6), you cannot run dot with + # shell=True. on others (Gentoo with Python 2.7), you + # must. In yet others (RHEL 5), either way works. I have + # no idea what the difference is, but it's kind of a PITA. + result = exc.run(cmd, shell=True, inputdata=idata) + if not result.success: + self.errExit("Error running %s: %s" % (cmd, result.error)) + if not setup.outfile: + print(result.stdout) + + +class Xcmd(_ProxyAdminCmd): + """ XML-RPC Command Interface """ + + options = _ProxyAdminCmd.options + [ + Bcfg2.Options.PositionalArgument("command"), + Bcfg2.Options.PositionalArgument("arguments", nargs='*')] + + def run(self, setup): + try: + data = getattr(self.proxy, setup.command)(*setup.arguments) + except Bcfg2.Client.Proxy.ProxyError: + self.errExit("Proxy Error: %s" % sys.exc_info()[1]) + + if data is not None: + print(data) + + +class CLI(Bcfg2.Options.CommandRegistry): + """ CLI class for bcfg2-admin """ + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, globals().values(), + parent=AdminCmd) + parser = Bcfg2.Options.get_parser( + description="Manage a running Bcfg2 server", + components=[self]) + parser.parse() + + def run(self): + """ Run bcfg2-admin """ + self.commands[Bcfg2.Options.setup.subcommand].setup() + return self.runcommand() diff --git a/src/lib/Bcfg2/Server/Admin/Backup.py b/src/lib/Bcfg2/Server/Admin/Backup.py deleted file mode 100644 index 0a04df98b..000000000 --- a/src/lib/Bcfg2/Server/Admin/Backup.py +++ /dev/null @@ -1,22 +0,0 @@ -""" Make a backup of the Bcfg2 repository """ - -import os -import time -import tarfile -import Bcfg2.Server.Admin -import Bcfg2.Options - - -class Backup(Bcfg2.Server.Admin.MetadataCore): - """ Make a backup of the Bcfg2 repository """ - - def __call__(self, args): - datastore = self.setup['repo'] - timestamp = time.strftime('%Y%m%d%H%M%S') - fmt = 'gz' - mode = 'w:' + fmt - filename = timestamp + '.tar' + '.' + fmt - out = tarfile.open(os.path.join(datastore, filename), mode=mode) - out.add(datastore, os.path.basename(datastore)) - out.close() - print("Archive %s was stored under %s" % (filename, datastore)) diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py deleted file mode 100644 index 325b7ae6e..000000000 --- a/src/lib/Bcfg2/Server/Admin/Client.py +++ /dev/null @@ -1,50 +0,0 @@ -""" Create, delete, or list client entries """ - -import sys -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugin import MetadataConsistencyError - - -def get_attribs(args): - """ Get a list of attributes to set on a client when adding/updating it """ - attr_d = {} - for i in args[2:]: - attr, val = i.split('=', 1) - if attr not in ['profile', 'uuid', 'password', 'floating', 'secure', - 'address', 'auth']: - print("Attribute %s unknown" % attr) - raise SystemExit(1) - attr_d[attr] = val - return attr_d - - -class Client(Bcfg2.Server.Admin.MetadataCore): - """ Create, delete, or list client entries """ - __usage__ = "[options] [add|del|update|list] [attr=val]" - __plugin_whitelist__ = ["Metadata"] - - def __call__(self, args): - if len(args) == 0: - self.errExit("No argument specified.\n" - "Usage: %s" % self.__usage__) - if args[0] == 'add': - try: - self.metadata.add_client(args[1], get_attribs(args)) - except MetadataConsistencyError: - self.errExit("Error adding client: %s" % sys.exc_info()[1]) - elif args[0] in ['update', 'up']: - try: - self.metadata.update_client(args[1], get_attribs(args)) - except MetadataConsistencyError: - self.errExit("Error updating client: %s" % sys.exc_info()[1]) - elif args[0] in ['delete', 'remove', 'del', 'rm']: - try: - self.metadata.remove_client(args[1]) - except MetadataConsistencyError: - self.errExit("Error deleting client: %s" % - sys.exc_info()[1]) - elif args[0] in ['list', 'ls']: - for client in self.metadata.list_clients(): - print(client) - else: - self.errExit("No command specified") diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py deleted file mode 100644 index d7285284a..000000000 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ /dev/null @@ -1,147 +0,0 @@ -import lxml.etree -import os -import Bcfg2.Server.Admin - - -class Compare(Bcfg2.Server.Admin.Mode): - """ Determine differences between files or directories of client - specification instances """ - __usage__ = ("<old> <new>\n\n" - " -r\trecursive") - - def __init__(self, setup): - Bcfg2.Server.Admin.Mode.__init__(self, setup) - self.important = {'Path': ['name', 'type', 'owner', 'group', 'mode', - 'important', 'paranoid', 'sensitive', - 'dev_type', 'major', 'minor', 'prune', - 'encoding', 'empty', 'to', 'recursive', - 'vcstype', 'sourceurl', 'revision', - 'secontext'], - 'Package': ['name', 'type', 'version', 'simplefile', - 'verify'], - 'Service': ['name', 'type', 'status', 'mode', - 'target', 'sequence', 'parameters'], - 'Action': ['name', 'timing', 'when', 'status', - 'command'], - 'PostInstall': ['name'] - } - - def compareStructures(self, new, old): - if new.tag == 'Independent': - bundle = 'Base' - else: - bundle = new.get('name') - - identical = True - - for child in new.getchildren(): - if child.tag not in self.important: - print(" %s in (new) bundle %s:\n tag type not handled!" % - (child.tag, bundle)) - continue - equiv = old.xpath('%s[@name="%s"]' % - (child.tag, child.get('name'))) - if len(equiv) == 0: - print(" %s %s in bundle %s:\n only in new configuration" % - (child.tag, child.get('name'), bundle)) - identical = False - continue - diff = [] - if child.tag == 'Path' and child.get('type') == 'file' and \ - child.text != equiv[0].text: - diff.append('contents') - attrdiff = [field for field in self.important[child.tag] if \ - child.get(field) != equiv[0].get(field)] - if attrdiff: - diff.append('attributes (%s)' % ', '.join(attrdiff)) - if diff: - print(" %s %s in bundle %s:\n %s differ" % (child.tag, \ - child.get('name'), bundle, ' and '.join(diff))) - identical = False - - for child in old.getchildren(): - if child.tag not in self.important: - print(" %s in (old) bundle %s:\n tag type not handled!" % - (child.tag, bundle)) - elif len(new.xpath('%s[@name="%s"]' % - (child.tag, child.get('name')))) == 0: - print(" %s %s in bundle %s:\n only in old configuration" % - (child.tag, child.get('name'), bundle)) - identical = False - - return identical - - def compareSpecifications(self, path1, path2): - try: - new = lxml.etree.parse(path1).getroot() - except IOError: - print("Failed to read %s" % (path1)) - raise SystemExit(1) - - try: - old = lxml.etree.parse(path2).getroot() - except IOError: - print("Failed to read %s" % (path2)) - raise SystemExit(1) - - for src in [new, old]: - for bundle in src.findall('./Bundle'): - if bundle.get('name')[-4:] == '.xml': - bundle.set('name', bundle.get('name')[:-4]) - - identical = True - - for bundle in old.findall('./Bundle'): - if len(new.xpath('Bundle[@name="%s"]' % (bundle.get('name')))) == 0: - print(" Bundle %s only in old configuration" % - bundle.get('name')) - identical = False - for bundle in new.findall('./Bundle'): - equiv = old.xpath('Bundle[@name="%s"]' % (bundle.get('name'))) - if len(equiv) == 0: - print(" Bundle %s only in new configuration" % - bundle.get('name')) - identical = False - elif not self.compareStructures(bundle, equiv[0]): - identical = False - - i1 = lxml.etree.Element('Independent') - i2 = lxml.etree.Element('Independent') - i1.extend(new.findall('./Independent/*')) - i2.extend(old.findall('./Independent/*')) - if not self.compareStructures(i1, i2): - identical = False - - return identical - - def __call__(self, args): - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin compare help for usage.") - if '-r' in args: - args = list(args) - args.remove('-r') - (oldd, newd) = args - (old, new) = [os.listdir(spot) for spot in args] - old_extra = [] - for item in old: - if item not in new: - old_extra.append(item) - continue - print("File: %s" % item) - state = self.__call__([oldd + '/' + item, newd + '/' + item]) - new.remove(item) - if state: - print("File %s is good" % item) - else: - print("File %s is bad" % item) - if new: - print("%s has extra files: %s" % (newd, ', '.join(new))) - if old_extra: - print("%s has extra files: %s" % (oldd, ', '.join(old_extra))) - return - try: - (old, new) = args - return self.compareSpecifications(new, old) - except IndexError: - self.errExit(self.__call__.__doc__) diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py deleted file mode 100644 index fdab5abca..000000000 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ /dev/null @@ -1,349 +0,0 @@ -""" Interactively initialize a new repository. """ - -import os -import sys -import stat -import select -import random -import socket -import string -import getpass -import subprocess - -import Bcfg2.Server.Admin -import Bcfg2.Server.Plugin -import Bcfg2.Options -import Bcfg2.Server.Plugins.Metadata -from Bcfg2.Compat import input # pylint: disable=W0622 - -# default config file -CONFIG = '''[server] -repository = %s -plugins = %s -# Uncomment the following to listen on all interfaces -#listen_all = true - -[statistics] -sendmailpath = %s -#web_debug = False -#time_zone = - -[database] -#engine = sqlite3 -# 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. -#name = -# Or path to database file if using sqlite3. -#<repository>/etc/bcfg2.sqlite is default path if left empty -#user = -# Not used with sqlite3. -#password = -# Not used with sqlite3. -#host = -# Not used with sqlite3. -#port = - -[reporting] -transport = LocalFilesystem - -[communication] -protocol = %s -password = %s -certificate = %s -key = %s -ca = %s - -[components] -bcfg2 = %s -''' - -# Default groups -GROUPS = '''<Groups version='3.0'> - <Group profile='true' public='true' default='true' name='basic'> - <Group name='%s'/> - </Group> - <Group name='ubuntu'/> - <Group name='debian'/> - <Group name='freebsd'/> - <Group name='gentoo'/> - <Group name='redhat'/> - <Group name='suse'/> - <Group name='mandrake'/> - <Group name='solaris'/> - <Group name='arch'/> -</Groups> -''' - -# Default contents of clients.xml -CLIENTS = '''<Clients version="3.0"> - <Client profile="basic" name="%s"/> -</Clients> -''' - -# Mapping of operating system names to groups -OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/CentOS', 'redhat'), - ('SUSE/SLES', 'suse'), - ('Mandrake', 'mandrake'), - ('Debian', 'debian'), - ('Ubuntu', 'ubuntu'), - ('Gentoo', 'gentoo'), - ('FreeBSD', 'freebsd'), - ('Arch', 'arch')] - - -def safe_input(prompt): - """ input() that flushes the input buffer before accepting input """ - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - return input(prompt) - - -def gen_password(length): - """Generates a random alphanumeric password with length characters.""" - chars = string.letters + string.digits - return "".join(random.choice(chars) for i in range(length)) - - -def create_key(hostname, keypath, certpath, country, state, location): - """Creates a bcfg2.key at the directory specifed by keypath.""" - kcstr = ("openssl req -batch -x509 -nodes -subj '/C=%s/ST=%s/L=%s/CN=%s' " - "-days 1000 -newkey rsa:2048 -keyout %s -noout" % (country, - state, - location, - hostname, - keypath)) - subprocess.call((kcstr), shell=True) - ccstr = ("openssl req -batch -new -subj '/C=%s/ST=%s/L=%s/CN=%s' -key %s " - "| openssl x509 -req -days 1000 -signkey %s -out %s" % (country, - state, - location, - hostname, - keypath, - keypath, - certpath)) - subprocess.call((ccstr), shell=True) - os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 - - -def create_conf(confpath, confdata): - """ create the config file """ - # Don't overwrite existing bcfg2.conf file - if os.path.exists(confpath): - result = safe_input("\nWarning: %s already exists. " - "Overwrite? [y/N]: " % confpath) - if result not in ['Y', 'y']: - print("Leaving %s unchanged" % confpath) - return - try: - open(confpath, "w").write(confdata) - os.chmod(confpath, stat.S_IRUSR | stat.S_IWUSR) # 0600 - except Exception: - err = sys.exc_info()[1] - print("Error trying to write configuration file '%s': %s" % - (confpath, err)) - raise SystemExit(1) - - -class Init(Bcfg2.Server.Admin.Mode): - """Interactively initialize a new repository.""" - options = {'configfile': Bcfg2.Options.CFILE, - 'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'proto': Bcfg2.Options.SERVER_PROTOCOL, - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - 'sendmail': Bcfg2.Options.SENDMAIL_PATH} - - def __init__(self, setup): - Bcfg2.Server.Admin.Mode.__init__(self, setup) - self.data = dict() - self.plugins = Bcfg2.Options.SERVER_PLUGINS.default - - def _set_defaults(self, opts): - """Set default parameters.""" - self.data['configfile'] = opts['configfile'] - self.data['repopath'] = opts['repo'] - self.data['password'] = gen_password(8) - self.data['server_uri'] = "https://%s:6789" % socket.getfqdn() - self.data['sendmail'] = opts['sendmail'] - self.data['proto'] = opts['proto'] - if os.path.exists("/etc/pki/tls"): - self.data['keypath'] = "/etc/pki/tls/private/bcfg2.key" - self.data['certpath'] = "/etc/pki/tls/certs/bcfg2.crt" - elif os.path.exists("/etc/ssl"): - self.data['keypath'] = "/etc/ssl/bcfg2.key" - self.data['certpath'] = "/etc/ssl/bcfg2.crt" - else: - basepath = os.path.dirname(self.configfile) - self.data['keypath'] = os.path.join(basepath, "bcfg2.key") - self.data['certpath'] = os.path.join(basepath, 'bcfg2.crt') - - def __call__(self, args): - # Parse options - opts = Bcfg2.Options.OptionParser(self.options) - opts.parse(args) - self._set_defaults(opts) - - # Prompt the user for input - self._prompt_config() - self._prompt_repopath() - self._prompt_password() - self._prompt_hostname() - self._prompt_server() - self._prompt_groups() - self._prompt_keypath() - self._prompt_certificate() - - # Initialize the repository - self.init_repo() - - def _prompt_hostname(self): - """Ask for the server hostname.""" - data = safe_input("What is the server's hostname [%s]: " % - socket.getfqdn()) - if data != '': - self.data['shostname'] = data - else: - self.data['shostname'] = socket.getfqdn() - - def _prompt_config(self): - """Ask for the configuration file path.""" - newconfig = safe_input("Store Bcfg2 configuration in [%s]: " % - self.configfile) - if newconfig != '': - self.data['configfile'] = os.path.abspath(newconfig) - - def _prompt_repopath(self): - """Ask for the repository path.""" - while True: - newrepo = safe_input("Location of Bcfg2 repository [%s]: " % - self.data['repopath']) - if newrepo != '': - self.data['repopath'] = os.path.abspath(newrepo) - if os.path.isdir(self.data['repopath']): - response = safe_input("Directory %s exists. Overwrite? [y/N]:" - % self.data['repopath']) - if response.lower().strip() == 'y': - break - else: - break - - def _prompt_password(self): - """Ask for a password or generate one if none is provided.""" - newpassword = getpass.getpass( - "Input password used for communication verification " - "(without echoing; leave blank for a random): ").strip() - if len(newpassword) != 0: - self.data['password'] = newpassword - - def _prompt_server(self): - """Ask for the server name.""" - newserver = safe_input( - "Input the server location (the server listens on a single " - "interface by default) [%s]: " % self.data['server_uri']) - if newserver != '': - self.data['server_uri'] = newserver - - def _prompt_groups(self): - """Create the groups.xml file.""" - prompt = '''Input base Operating System for clients:\n''' - for entry in OS_LIST: - prompt += "%d: %s\n" % (OS_LIST.index(entry) + 1, entry[0]) - prompt += ': ' - while True: - try: - osidx = int(safe_input(prompt)) - self.data['os_sel'] = OS_LIST[osidx - 1][1] - break - except ValueError: - continue - - def _prompt_certificate(self): - """Ask for the key details (country, state, and location).""" - print("The following questions affect SSL certificate generation.") - print("If no data is provided, the default values are used.") - newcountry = safe_input("Country name (2 letter code) for " - "certificate: ") - if newcountry != '': - if len(newcountry) == 2: - self.data['country'] = newcountry - else: - while len(newcountry) != 2: - newcountry = safe_input("2 letter country code (eg. US): ") - if len(newcountry) == 2: - self.data['country'] = newcountry - break - else: - self.data['country'] = 'US' - - newstate = safe_input("State or Province Name (full name) for " - "certificate: ") - if newstate != '': - self.data['state'] = newstate - else: - self.data['state'] = 'Illinois' - - newlocation = safe_input("Locality Name (eg, city) for certificate: ") - if newlocation != '': - self.data['location'] = newlocation - else: - self.data['location'] = 'Argonne' - - def _prompt_keypath(self): - """ Ask for the key pair location. Try to use sensible - defaults depending on the OS """ - keypath = safe_input("Path where Bcfg2 server private key will be " - "created [%s]: " % self.data['keypath']) - if keypath: - self.data['keypath'] = keypath - certpath = safe_input("Path where Bcfg2 server cert will be created " - "[%s]: " % self.data['certpath']) - if certpath: - self.data['certpath'] = certpath - - def _init_plugins(self): - """Initialize each plugin-specific portion of the repository.""" - for plugin in self.plugins: - if plugin == 'Metadata': - Bcfg2.Server.Plugins.Metadata.Metadata.init_repo( - self.data['repopath'], - groups_xml=GROUPS % self.data['os_sel'], - clients_xml=CLIENTS % socket.getfqdn()) - else: - try: - module = __import__("Bcfg2.Server.Plugins.%s" % plugin, '', - '', ["Bcfg2.Server.Plugins"]) - cls = getattr(module, plugin) - cls.init_repo(self.data['repopath']) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - print("Plugin setup for %s failed: %s\n" - "Check that dependencies are installed" % (plugin, - err)) - - def init_repo(self): - """Setup a new repo and create the content of the - configuration file.""" - # Create the repository - path = os.path.join(self.data['repopath'], 'etc') - try: - os.makedirs(path) - self._init_plugins() - print("Repository created successfuly in %s" % - self.data['repopath']) - except OSError: - print("Failed to create %s." % path) - - confdata = CONFIG % (self.data['repopath'], - ','.join(self.plugins), - self.data['sendmail'], - self.data['proto'], - self.data['password'], - self.data['certpath'], - self.data['keypath'], - self.data['certpath'], - self.data['server_uri']) - - # Create the configuration file and SSL key - create_conf(self.data['configfile'], confdata) - create_key(self.data['shostname'], self.data['keypath'], - self.data['certpath'], self.data['country'], - self.data['state'], self.data['location']) diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py deleted file mode 100644 index 65f99a213..000000000 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ /dev/null @@ -1,56 +0,0 @@ -""" Extract extra entry lists from statistics """ -import getopt -import lxml.etree -import sys -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugin import PullSource - - -class Minestruct(Bcfg2.Server.Admin.StructureMode): - """ Extract extra entry lists from statistics """ - __usage__ = ("[options] <client>\n\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-f <filename>", "build a particular file", - "-g <groups>", "only build config for groups")) - - def __call__(self, args): - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin minestruct help for usage.") - try: - (opts, args) = getopt.getopt(args, 'f:g:h') - except getopt.GetoptError: - self.errExit(self.__doc__) - - client = args[0] - output = sys.stdout - groups = [] - - for (opt, optarg) in opts: - if opt == '-f': - try: - output = open(optarg, 'w') - except IOError: - self.errExit("Failed to open file: %s" % (optarg)) - elif opt == '-g': - groups = optarg.split(':') - - try: - extra = set() - for source in self.bcore.plugins_by_type(PullSource): - for item in source.GetExtra(client): - extra.add(item) - except: # pylint: disable=W0702 - self.errExit("Failed to find extra entry info for client %s" % - client) - root = lxml.etree.Element("Base") - self.log.info("Found %d extra entries" % (len(extra))) - add_point = root - for grp in groups: - add_point = lxml.etree.SubElement(add_point, "Group", name=grp) - for tag, name in extra: - self.log.info("%s: %s" % (tag, name)) - lxml.etree.SubElement(add_point, tag, name=name) - - lxml.etree.ElementTree(root).write(output, pretty_print=True) diff --git a/src/lib/Bcfg2/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py deleted file mode 100644 index f6bc22959..000000000 --- a/src/lib/Bcfg2/Server/Admin/Perf.py +++ /dev/null @@ -1,37 +0,0 @@ -""" Get performance data from server """ - -import sys -import Bcfg2.Options -import Bcfg2.Proxy -import Bcfg2.Server.Admin - - -class Perf(Bcfg2.Server.Admin.Mode): - """ Get performance data from server """ - - def __call__(self, args): - output = [('Name', 'Min', 'Max', 'Mean', 'Count')] - optinfo = { - 'ca': Bcfg2.Options.CLIENT_CA, - 'certificate': Bcfg2.Options.CLIENT_CERT, - 'key': Bcfg2.Options.SERVER_KEY, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'server': Bcfg2.Options.SERVER_LOCATION, - 'user': Bcfg2.Options.CLIENT_USER, - 'timeout': Bcfg2.Options.CLIENT_TIMEOUT} - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) - data = proxy.get_statistics() - for key in sorted(data.keys()): - output.append( - (key, ) + - tuple(["%.06f" % item - for item in data[key][:-1]] + [data[key][-1]])) - self.print_table(output) diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py deleted file mode 100644 index fccdb2d94..000000000 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ /dev/null @@ -1,147 +0,0 @@ -""" Retrieves entries from clients and integrates the information into -the repository """ - -import os -import sys -import getopt -import select -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugin import PullSource, Generator -from Bcfg2.Compat import input # pylint: disable=W0622 - - -class Pull(Bcfg2.Server.Admin.MetadataCore): - """ Retrieves entries from clients and integrates the information - into the repository """ - __usage__ = ("[options] <client> <entry type> <entry name>\n\n" - " %-25s%s\n" - " %-25s%s\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-v", "be verbose", - "-f", "force", - "-I", "interactive", - "-s", "stdin")) - - def __init__(self, setup): - Bcfg2.Server.Admin.MetadataCore.__init__(self, setup) - self.log = False - self.mode = 'interactive' - - def __call__(self, args): - use_stdin = False - try: - opts, gargs = getopt.getopt(args, 'vfIs') - except getopt.GetoptError: - self.errExit(self.__doc__) - for opt in opts: - if opt[0] == '-v': - self.log = True - elif opt[0] == '-f': - self.mode = 'force' - elif opt[0] == '-I': - self.mode = 'interactive' - elif opt[0] == '-s': - use_stdin = True - - if use_stdin: - for line in sys.stdin: - try: - self.PullEntry(*line.split(None, 3)) - except SystemExit: - print(" for %s" % line) - except: - print("Bad entry: %s" % line.strip()) - elif len(gargs) < 3: - self.usage() - else: - self.PullEntry(gargs[0], gargs[1], gargs[2]) - - def BuildNewEntry(self, client, etype, ename): - """Construct a new full entry for - given client/entry from statistics. - """ - new_entry = {'type': etype, 'name': ename} - pull_sources = self.bcore.plugins_by_type(PullSource) - for plugin in pull_sources: - try: - (owner, group, mode, contents) = \ - plugin.GetCurrentEntry(client, etype, ename) - break - except Bcfg2.Server.Plugin.PluginExecutionError: - if plugin == pull_sources[-1]: - print("Pull Source failure; could not fetch current state") - raise SystemExit(1) - - try: - data = {'owner': owner, - 'group': group, - 'mode': mode, - 'text': contents} - except UnboundLocalError: - print("Unable to build entry. " - "Do you have a statistics plugin enabled?") - raise SystemExit(1) - for key, val in list(data.items()): - if val: - new_entry[key] = val - return new_entry - - def Choose(self, choices): - """Determine where to put pull data.""" - if self.mode == 'interactive': - for choice in choices: - print("Plugin returned choice:") - if id(choice) == id(choices[0]): - print("(current entry) ") - if choice.all: - print(" => global entry") - elif choice.group: - print(" => group entry: %s (prio %d)" % - (choice.group, choice.prio)) - else: - print(" => host entry: %s" % (choice.hostname)) - - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], - 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - ans = input("Use this entry? [yN]: ") in ['y', 'Y'] - if ans: - return choice - return False - else: - # mode == 'force' - if not choices: - return False - return choices[0] - - def PullEntry(self, client, etype, ename): - """Make currently recorded client state correct for entry.""" - new_entry = self.BuildNewEntry(client, etype, ename) - - meta = self.bcore.build_metadata(client) - # Find appropriate plugin in bcore - glist = [gen for gen in self.bcore.plugins_by_type(Generator) - if ename in gen.Entries.get(etype, {})] - if len(glist) != 1: - self.errExit("Got wrong numbers of matching generators for entry:" - "%s" % ([g.name for g in glist])) - plugin = glist[0] - if not isinstance(plugin, Bcfg2.Server.Plugin.PullTarget): - self.errExit("Configuration upload not supported by plugin %s" % - plugin.name) - try: - choices = plugin.AcceptChoices(new_entry, meta) - specific = self.Choose(choices) - if specific: - plugin.AcceptPullData(specific, new_entry, self.log) - except Bcfg2.Server.Plugin.PluginExecutionError: - self.errExit("Configuration upload not supported by plugin %s" % - plugin.name) - # Commit if running under a VCS - for vcsplugin in list(self.bcore.plugins.values()): - if isinstance(vcsplugin, Bcfg2.Server.Plugin.Version): - files = "%s/%s" % (plugin.data, ename) - comment = 'file "%s" pulled from host %s' % (files, client) - vcsplugin.commit_data([files], comment) diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py deleted file mode 100644 index eb97123f7..000000000 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ /dev/null @@ -1,262 +0,0 @@ -'''Admin interface for dynamic reports''' -import Bcfg2.Logger -import Bcfg2.Server.Admin -import datetime -import os -import sys -import traceback -from Bcfg2 import settings - -# Load django and reports stuff _after_ we know we can load settings -from django.core import management -from Bcfg2.Reporting.utils import * - -project_directory = os.path.dirname(settings.__file__) -project_name = os.path.basename(project_directory) -sys.path.append(os.path.join(project_directory, '..')) -project_module = __import__(project_name, '', '', ['']) -sys.path.pop() - -# Set DJANGO_SETTINGS_MODULE appropriately. -os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name - -from Bcfg2.Reporting.models import Client, Interaction, \ - Performance, Bundle, Group, FailureEntry, PathEntry, \ - PackageEntry, ServiceEntry, ActionEntry -from Bcfg2.Reporting.Compat import transaction - - -def printStats(fn): - """ - Print db stats. - - Decorator for purging. Prints database statistics after a run. - """ - def print_stats(self, *data): - classes = (Client, Interaction, Performance, \ - FailureEntry, ActionEntry, PathEntry, PackageEntry, \ - ServiceEntry, Group, Bundle) - - starts = {} - for cls in classes: - starts[cls] = cls.objects.count() - - fn(self, *data) - - for cls in classes: - print("%s removed: %s" % (cls().__class__.__name__, - starts[cls] - cls.objects.count())) - - return print_stats - - -class Reports(Bcfg2.Server.Admin.Mode): - """ Manage dynamic reports """ - django_commands = ['dbshell', 'shell', 'sqlall', 'validate'] - __usage__ = ("[command] [options]\n" - " Commands:\n" - " init Initialize the database\n" - " purge Purge records\n" - " --client [n] Client to operate on\n" - " --days [n] Records older then n days\n" - " --expired Expired clients only\n" - " scrub Scrub the database for duplicate " - "reasons and orphaned entries\n" - " stats print database statistics\n" - " update Apply any updates to the reporting " - "database\n" - "\n" - " Django commands:\n " \ - + "\n ".join(django_commands)) - - def __init__(self, setup): - Bcfg2.Server.Admin.Mode.__init__(self, setup) - try: - import south - except ImportError: - print("Django south is required for Reporting") - raise SystemExit(-3) - - def __call__(self, args): - if len(args) == 0 or args[0] == '-h': - self.errExit(self.__usage__) - - # FIXME - dry run - - if args[0] in self.django_commands: - self.django_command_proxy(args[0]) - elif args[0] == 'scrub': - self.scrub() - elif args[0] == 'stats': - self.stats() - elif args[0] in ['init', 'update', 'syncdb']: - if self.setup['debug']: - vrb = 2 - elif self.setup['verbose']: - vrb = 1 - else: - vrb = 0 - try: - management.call_command("syncdb", verbosity=vrb) - management.call_command("migrate", verbosity=vrb) - except: - self.errExit("Update failed: %s" % sys.exc_info()[1]) - elif args[0] == 'purge': - expired = False - client = None - maxdate = None - state = None - i = 1 - while i < len(args): - if args[i] == '-c' or args[i] == '--client': - if client: - self.errExit("Only one client per run") - client = args[i + 1] - print(client) - i = i + 1 - elif args[i] == '--days': - if maxdate: - self.errExit("Max date specified multiple times") - try: - maxdate = datetime.datetime.now() - \ - datetime.timedelta(days=int(args[i + 1])) - except: - self.errExit("Invalid number of days: %s" % - args[i + 1]) - i = i + 1 - elif args[i] == '--expired': - expired = True - i = i + 1 - if expired: - if state: - self.errExit("--state is not valid with --expired") - self.purge_expired(maxdate) - else: - self.purge(client, maxdate, state) - else: - self.errExit("Unknown command: %s" % args[0]) - - @transaction.atomic - def scrub(self): - ''' Perform a thorough scrub and cleanup of the database ''' - - # Cleanup unused entries - for cls in (Group, Bundle, FailureEntry, ActionEntry, PathEntry, - PackageEntry, PathEntry): - try: - start_count = cls.objects.count() - cls.prune_orphans() - self.log.info("Pruned %d %s records" % \ - (start_count - cls.objects.count(), cls.__class__.__name__)) - except: - print("Failed to prune %s: %s" % - (cls.__class__.__name__, sys.exc_info()[1])) - - def django_command_proxy(self, command): - '''Call a django command''' - if command == 'sqlall': - management.call_command(command, 'Reporting') - else: - management.call_command(command) - - @printStats - def purge(self, client=None, maxdate=None, state=None): - '''Purge historical data from the database''' - - filtered = False # indicates whether or not a client should be deleted - - if not client and not maxdate and not state: - self.errExit("Reports.prune: Refusing to prune all data") - - ipurge = Interaction.objects - if client: - try: - cobj = Client.objects.get(name=client) - ipurge = ipurge.filter(client=cobj) - except Client.DoesNotExist: - self.errExit("Client %s not in database" % client) - self.log.debug("Filtering by client: %s" % client) - - if maxdate: - filtered = True - if not isinstance(maxdate, datetime.datetime): - raise TypeError("maxdate is not a DateTime object") - self.log.debug("Filtering by maxdate: %s" % maxdate) - ipurge = ipurge.filter(timestamp__lt=maxdate) - - if settings.DATABASES['default']['ENGINE'] == \ - 'django.db.backends.sqlite3': - grp_limit = 100 - else: - grp_limit = 1000 - if state: - filtered = True - if state not in ('dirty', 'clean', 'modified'): - raise TypeError("state is not one of the following values: " - "dirty, clean, modified") - self.log.debug("Filtering by state: %s" % state) - ipurge = ipurge.filter(state=state) - - count = ipurge.count() - rnum = 0 - try: - while rnum < count: - grp = list(ipurge[:grp_limit].values("id")) - # just in case... - if not grp: - break - Interaction.objects.filter(id__in=[x['id'] - for x in grp]).delete() - rnum += len(grp) - self.log.debug("Deleted %s of %s" % (rnum, count)) - except: - self.log.error("Failed to remove interactions") - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - del a, b, c - self.log.error(msg) - - # Prune any orphaned ManyToMany relations - for m2m in (ActionEntry, PackageEntry, PathEntry, ServiceEntry, \ - FailureEntry, Group, Bundle): - self.log.debug("Pruning any orphaned %s objects" % \ - m2m().__class__.__name__) - m2m.prune_orphans() - - if client and not filtered: - # Delete the client, ping data is automatic - try: - self.log.debug("Purging client %s" % client) - cobj.delete() - except: - self.log.error("Failed to delete client %s" % client) - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - del a, b, c - self.log.error(msg) - - @printStats - def purge_expired(self, maxdate=None): - '''Purge expired clients from the database''' - - if maxdate: - if not isinstance(maxdate, datetime.datetime): - raise TypeError("maxdate is not a DateTime object") - self.log.debug("Filtering by maxdate: %s" % maxdate) - clients = Client.objects.filter(expiration__lt=maxdate) - else: - clients = Client.objects.filter(expiration__isnull=False) - - for client in clients: - self.log.debug("Purging client %s" % client) - Interaction.objects.filter(client=client).delete() - client.delete() - - def stats(self): - classes = (Client, Interaction, Performance, \ - FailureEntry, ActionEntry, PathEntry, PackageEntry, \ - ServiceEntry, Group, Bundle) - - for cls in classes: - print("%s has %s records" % (cls().__class__.__name__, - cls.objects.count())) diff --git a/src/lib/Bcfg2/Server/Admin/Snapshots.py b/src/lib/Bcfg2/Server/Admin/Snapshots.py deleted file mode 100644 index fcb240352..000000000 --- a/src/lib/Bcfg2/Server/Admin/Snapshots.py +++ /dev/null @@ -1,162 +0,0 @@ -from datetime import date -import sys - -# Prereq issues can be signaled with ImportError, so no try needed -import sqlalchemy, sqlalchemy.orm -import Bcfg2.Server.Admin -import Bcfg2.Server.Snapshots -import Bcfg2.Server.Snapshots.model -from Bcfg2.Server.Snapshots.model import Snapshot, Client, Metadata, Base, \ - File, Group, Package, Service -# Compatibility import -from Bcfg2.Compat import u_str - -class Snapshots(Bcfg2.Server.Admin.Mode): - """ Interact with the Snapshots system """ - __usage__ = "[init|query qtype]" - - q_dispatch = {'client': Client, - 'group': Group, - 'metadata': Metadata, - 'package': Package, - 'snapshot': Snapshot} - - def __init__(self, setup): - Bcfg2.Server.Admin.Mode.__init__(self, setup) - self.session = Bcfg2.Server.Snapshots.setup_session(self.configfile) - self.cfile = self.configfile - - def __call__(self, args): - if len(args) == 0 or args[0] == '-h': - print(self.__usage__) - raise SystemExit(0) - - if args[0] == 'query': - if args[1] in self.q_dispatch: - q_obj = self.q_dispatch[args[1]] - if q_obj == Client: - rows = [] - labels = ('Client', 'Active') - for host in \ - self.session.query(q_obj).filter(q_obj.active == False): - rows.append([host.name, 'No']) - for host in \ - self.session.query(q_obj).filter(q_obj.active == True): - rows.append([host.name, 'Yes']) - self.print_table([labels]+rows, - justify='left', - hdr=True, - vdelim=" ", - padding=1) - elif q_obj == Group: - print("Groups:") - for group in self.session.query(q_obj).all(): - print(" %s" % group.name) - else: - results = self.session.query(q_obj).all() - else: - print('error') - raise SystemExit(1) - elif args[0] == 'init': - # Initialize the Snapshots database - dbpath = Bcfg2.Server.Snapshots.db_from_config(self.cfile) - engine = sqlalchemy.create_engine(dbpath, echo=True) - metadata = Base.metadata - metadata.create_all(engine) - Session = sqlalchemy.orm.sessionmaker() - Session.configure(bind=engine) - session = Session() - session.commit() - elif args[0] == 'dump': - client = args[1] - snap = Snapshot.get_current(self.session, u_str(client)) - if not snap: - print("Current snapshot for %s not found" % client) - sys.exit(1) - print("Client %s last run at %s" % (client, snap.timestamp)) - for pkg in snap.packages: - print("C:", pkg.correct, 'M:', pkg.modified) - print("start", pkg.start.name, pkg.start.version) - print("end", pkg.end.name, pkg.end.version) - elif args[0] == 'reports': - # bcfg2-admin reporting interface for Snapshots - if '-a' in args[1:]: - # Query all hosts for Name, Status, Revision, Timestamp - q = self.session.query(Client.name, - Snapshot.correct, - Snapshot.revision, - Snapshot.timestamp)\ - .filter(Client.id==Snapshot.client_id)\ - .group_by(Client.id) - rows = [] - labels = ('Client', 'Correct', 'Revision', 'Time') - for item in q.all(): - cli, cor, time, rev = item - rows.append([cli, cor, time, rev]) - self.print_table([labels]+rows, - justify='left', - hdr=True, vdelim=" ", - padding=1) - elif '-b' in args[1:]: - # Query a single host for bad entries - if len(args) < 3: - print("Usage: bcfg2-admin snapshots -b <client>") - return - client = args[2] - snap = Snapshot.get_current(self.session, u_str(client)) - if not snap: - print("Current snapshot for %s not found" % client) - sys.exit(1) - print("Bad entries:") - bad_pkgs = [self.session.query(Package) - .filter(Package.id==p.start_id).one().name \ - for p in snap.packages if p.correct == False] - for p in bad_pkgs: - print(" Package:%s" % p) - bad_files = [self.session.query(File) - .filter(File.id==f.start_id).one().name \ - for f in snap.files if f.correct == False] - for filename in bad_files: - print(" File:%s" % filename) - bad_svcs = [self.session.query(Service) - .filter(Service.id==s.start_id).one().name \ - for s in snap.services if s.correct == False] - for svc in bad_svcs: - print(" Service:%s" % svc) - elif '-e' in args[1:]: - # Query a single host for extra entries - client = args[2] - snap = Snapshot.get_current(self.session, u_str(client)) - if not snap: - print("Current snapshot for %s not found" % client) - sys.exit(1) - print("Extra entries:") - for pkg in snap.extra_packages: - print(" Package:%s" % pkg.name) - # FIXME: Do we know about extra files yet? - for f in snap.extra_files: - print(" File:%s" % f.name) - for svc in snap.extra_services: - print(" Service:%s" % svc.name) - elif '--date' in args[1:]: - year, month, day = args[2:] - timestamp = date(int(year), int(month), int(day)) - snaps = [] - for client in self.session.query(Client).filter(Client.active == True): - snaps.append(Snapshot.get_by_date(self.session, - client.name, - timestamp)) - rows = [] - labels = ('Client', 'Correct', 'Revision', 'Time') - for snap in snaps: - rows.append([snap.client.name, - snap.correct, - snap.revision, - snap.timestamp]) - self.print_table([labels]+rows, - justify='left', - hdr=True, - vdelim=" ", - padding=1) - else: - print("Unknown options: ", args[1:]) diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py deleted file mode 100644 index eb417966d..000000000 --- a/src/lib/Bcfg2/Server/Admin/Syncdb.py +++ /dev/null @@ -1,29 +0,0 @@ -import sys -import Bcfg2.settings -import Bcfg2.Options -import Bcfg2.Server.Admin -import Bcfg2.Server.models -from django.core.exceptions import ImproperlyConfigured -from django.core.management import setup_environ, call_command - - -class Syncdb(Bcfg2.Server.Admin.Mode): - """ Sync the Django ORM with the configured database """ - options = {'configfile': Bcfg2.Options.WEB_CFILE} - - def __call__(self, args): - # Parse options - opts = Bcfg2.Options.OptionParser(self.options) - opts.parse(args) - - setup_environ(Bcfg2.settings) - Bcfg2.Server.models.load_models(cfile=opts['configfile']) - - try: - call_command("syncdb", interactive=False, verbosity=0) - self._database_available = True - except ImproperlyConfigured: - self.errExit("Django configuration problem: %s" % - sys.exc_info()[1]) - except: - self.errExit("Database update failed: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Admin/Viz.py b/src/lib/Bcfg2/Server/Admin/Viz.py deleted file mode 100644 index cb2470e17..000000000 --- a/src/lib/Bcfg2/Server/Admin/Viz.py +++ /dev/null @@ -1,113 +0,0 @@ -""" Produce graphviz diagrams of metadata structures """ - -import getopt -from subprocess import Popen, PIPE -import pipes -import Bcfg2.Server.Admin - - -class Viz(Bcfg2.Server.Admin.MetadataCore): - """ Produce graphviz diagrams of metadata structures """ - __usage__ = ("[options]\n\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" % - ("-H, --includehosts", - "include hosts in the viz output", - "-b, --includebundles", - "include bundles in the viz output", - "-k, --includekey", - "show a key for different digraph shapes", - "-c, --only-client <clientname>", - "show only the groups, bundles for the named client", - "-o, --outfile <file>", - "write viz output to an output file")) - - colors = ['steelblue1', 'chartreuse', 'gold', 'magenta', - 'indianred1', 'limegreen', 'orange1', 'lightblue2', - 'green1', 'blue1', 'yellow1', 'darkturquoise', 'gray66'] - - __plugin_blacklist__ = ['DBStats', 'Snapshots', 'Cfg', 'Pkgmgr', - 'Packages', 'Rules', 'Account', 'Decisions', - 'Deps', 'Git', 'Svn', 'Fossil', 'Bzr', 'Bundler', - 'TGenshi', 'Base'] - - def __call__(self, args): - # First get options to the 'viz' subcommand - try: - opts, args = getopt.getopt(args, 'Hbkc:o:', - ['includehosts', 'includebundles', - 'includekey', 'only-client=', - 'outfile=']) - except getopt.GetoptError: - self.usage() - - hset = False - bset = False - kset = False - only_client = None - outputfile = False - for opt, arg in opts: - if opt in ("-H", "--includehosts"): - hset = True - elif opt in ("-b", "--includebundles"): - bset = True - elif opt in ("-k", "--includekey"): - kset = True - elif opt in ("-c", "--only-client"): - only_client = arg - elif opt in ("-o", "--outfile"): - outputfile = arg - - data = self.Visualize(hset, bset, kset, only_client, outputfile) - if data: - print(data) - - def Visualize(self, hosts=False, bundles=False, key=False, - only_client=None, output=None): - """Build visualization of groups file.""" - if output: - fmt = output.split('.')[-1] - else: - fmt = 'png' - - cmd = ["dot", "-T", fmt] - if output: - cmd.extend(["-o", output]) - try: - dotpipe = Popen(cmd, stdin=PIPE, stdout=PIPE, close_fds=True) - except OSError: - # on some systems (RHEL 6), you cannot run dot with - # shell=True. on others (Gentoo with Python 2.7), you - # must. In yet others (RHEL 5), either way works. I have - # no idea what the difference is, but it's kind of a PITA. - cmd = ["dot", "-T", pipes.quote(fmt)] - if output: - cmd.extend(["-o", pipes.quote(output)]) - dotpipe = Popen(cmd, shell=True, - stdin=PIPE, stdout=PIPE, close_fds=True) - try: - dotpipe.stdin.write("digraph groups {\n") - except: - print("write to dot process failed. Is graphviz installed?") - raise SystemExit(1) - dotpipe.stdin.write('\trankdir="LR";\n') - dotpipe.stdin.write(self.metadata.viz(hosts, bundles, - key, only_client, self.colors)) - if key: - dotpipe.stdin.write("\tsubgraph cluster_key {\n") - dotpipe.stdin.write('\tstyle="filled";\n') - dotpipe.stdin.write('\tcolor="lightblue";\n') - dotpipe.stdin.write('\tBundle [ shape="septagon" ];\n') - dotpipe.stdin.write('\tGroup [shape="ellipse"];\n') - dotpipe.stdin.write('\tGroup Category [shape="trapezium"];\n') - dotpipe.stdin.write('\tProfile [style="bold", shape="ellipse"];\n') - dotpipe.stdin.write('\tHblock [label="Host1|Host2|Host3", ' - 'shape="record"];\n') - dotpipe.stdin.write('\tlabel="Key";\n') - dotpipe.stdin.write("\t}\n") - dotpipe.stdin.write("}\n") - dotpipe.stdin.close() - return dotpipe.stdout.read() diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py deleted file mode 100644 index 036129a1b..000000000 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ /dev/null @@ -1,44 +0,0 @@ -""" XML-RPC Command Interface for bcfg2-admin""" - -import sys -import Bcfg2.Options -import Bcfg2.Proxy -import Bcfg2.Server.Admin - - -class Xcmd(Bcfg2.Server.Admin.Mode): - """ XML-RPC Command Interface """ - __usage__ = "<command>" - - def __call__(self, args): - optinfo = { - 'server': Bcfg2.Options.SERVER_LOCATION, - 'user': Bcfg2.Options.CLIENT_USER, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'key': Bcfg2.Options.SERVER_KEY, - 'certificate': Bcfg2.Options.CLIENT_CERT, - 'ca': Bcfg2.Options.CLIENT_CA, - 'timeout': Bcfg2.Options.CLIENT_TIMEOUT} - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(args) - Bcfg2.Proxy.RetryMethod.max_retries = 1 - proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) - if len(setup['args']) == 0: - self.errExit("Usage: xcmd <xmlrpc method> <optional arguments>") - cmd = setup['args'][0] - args = () - if len(setup['args']) > 1: - args = tuple(setup['args'][1:]) - try: - data = getattr(proxy, cmd)(*args) - except Bcfg2.Proxy.ProxyError: - self.errExit("Proxy Error: %s" % sys.exc_info()[1]) - - if data is not None: - print(data) diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py deleted file mode 100644 index ef5b2a08c..000000000 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ /dev/null @@ -1,142 +0,0 @@ -""" Base classes for admin modes """ - -import re -import sys -import logging -import lxml.etree -import Bcfg2.Server.Core -import Bcfg2.Options -from Bcfg2.Compat import ConfigParser, walk_packages - -__all__ = [m[1] for m in walk_packages(path=__path__)] - - -class Mode(object): - """ Base object for admin modes. Docstrings are used as help - messages, so if you are seeing this, a help message has not yet - been added for this mode. """ - __usage__ = None - __args__ = [] - - def __init__(self, setup): - self.setup = setup - self.configfile = setup['configfile'] - self.__cfp = False - self.log = logging.getLogger('Bcfg2.Server.Admin.Mode') - usage = "bcfg2-admin %s" % self.__class__.__name__.lower() - if self.__usage__ is not None: - usage += " " + self.__usage__ - setup.hm = usage - - def getCFP(self): - """ get a config parser for the Bcfg2 config file """ - if not self.__cfp: - self.__cfp = ConfigParser.ConfigParser() - self.__cfp.read(self.configfile) - return self.__cfp - - cfp = property(getCFP) - - def __call__(self, args): - raise NotImplementedError - - @classmethod - def usage(cls, rv=1): - """ Exit with a long usage message """ - print(re.sub(r'\s{2,}', ' ', cls.__doc__.strip())) - print("") - print("Usage:") - usage = "bcfg2-admin %s" % cls.__name__.lower() - if cls.__usage__ is not None: - usage += " " + cls.__usage__ - print(" %s" % usage) - raise SystemExit(rv) - - def shutdown(self): - """ Perform any necessary shtudown tasks for this mode """ - pass - - def errExit(self, emsg): - """ exit with an error """ - sys.stderr.write('%s\n' % emsg) - raise SystemExit(1) - - def load_stats(self, client): - """ Load static statistics from the repository """ - stats = lxml.etree.parse("%s/etc/statistics.xml" % self.setup['repo']) - hostent = stats.xpath('//Node[@name="%s"]' % client) - if not hostent: - self.errExit("Could not find stats for client %s" % (client)) - return hostent[0] - - def print_table(self, rows, justify='left', hdr=True, vdelim=" ", - padding=1): - """Pretty print a table - - rows - list of rows ([[row 1], [row 2], ..., [row n]]) - hdr - if True the first row is treated as a table header - vdelim - vertical delimiter between columns - padding - # of spaces around the longest element in the column - justify - may be left,center,right - - """ - hdelim = "=" - justify = {'left': str.ljust, - 'center': str.center, - 'right': str.rjust}[justify.lower()] - - # Calculate column widths (longest item in each column - # plus padding on both sides) - cols = list(zip(*rows)) - col_widths = [max([len(str(item)) + 2 * padding - for item in col]) for col in cols] - borderline = vdelim.join([w * hdelim for w in col_widths]) - - # Print out the table - print(borderline) - for row in rows: - print(vdelim.join([justify(str(item), width) - for (item, width) in zip(row, col_widths)])) - if hdr: - print(borderline) - hdr = False - - -# pylint wants MetadataCore and StructureMode to be concrete classes -# and implement __call__, but they aren't and they don't, so we -# disable that warning -# pylint: disable=W0223 - -class MetadataCore(Mode): - """Base class for admin-modes that handle metadata.""" - __plugin_whitelist__ = None - __plugin_blacklist__ = None - - def __init__(self, setup): - Mode.__init__(self, setup) - if self.__plugin_whitelist__ is not None: - setup['plugins'] = [p for p in setup['plugins'] - if p in self.__plugin_whitelist__] - elif self.__plugin_blacklist__ is not None: - setup['plugins'] = [p for p in setup['plugins'] - if p not in self.__plugin_blacklist__] - - # admin modes don't need to watch for changes. one shot is fine here. - setup['filemonitor'] = 'pseudo' - try: - self.bcore = Bcfg2.Server.Core.BaseCore(setup) - except Bcfg2.Server.Core.CoreInitError: - msg = sys.exc_info()[1] - self.errExit("Core load failed: %s" % msg) - self.bcore.load_plugins() - self.bcore.fam.handle_event_set() - self.metadata = self.bcore.metadata - - def shutdown(self): - if hasattr(self, 'bcore'): - self.bcore.shutdown() - - -class StructureMode(MetadataCore): # pylint: disable=W0223 - """ Base class for admin modes that handle structure plugins """ - pass diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 29beb35d5..e138c57e4 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -5,10 +5,11 @@ import sys import time import socket import daemon -import Bcfg2.Statistics -from Bcfg2.Server.Core import BaseCore, NoExposedMethod +import Bcfg2.Options +import Bcfg2.Server.Statistics +from Bcfg2.Server.Core import NetworkCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse -from Bcfg2.SSLServer import XMLRPCServer +from Bcfg2.Server.SSLServer import XMLRPCServer from lockfile import LockFailed, LockTimeout # pylint: disable=E0611 @@ -19,29 +20,29 @@ except ImportError: # pylint: enable=E0611 -class Core(BaseCore): +class BuiltinCore(NetworkCore): """ The built-in server core """ name = 'bcfg2-server' - def __init__(self, setup): - BaseCore.__init__(self, setup) + def __init__(self): + NetworkCore.__init__(self) - #: The :class:`Bcfg2.SSLServer.XMLRPCServer` instance powering - #: this server core + #: The :class:`Bcfg2.Server.SSLServer.XMLRPCServer` instance + #: powering this server core self.server = None - daemon_args = dict(uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid'], - umask=int(self.setup['umask'], 8), + daemon_args = dict(uid=Bcfg2.Options.setup.daemon_uid, + gid=Bcfg2.Options.setup.daemon_gid, + umask=int(Bcfg2.Options.setup.umask, 8), detach_process=True) - if self.setup['daemon']: - daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'], - acquire_timeout=5) + if Bcfg2.Options.setup.daemon: + daemon_args['pidfile'] = TimeoutPIDLockFile( + Bcfg2.Options.setup.daemon, acquire_timeout=5) #: The :class:`daemon.DaemonContext` used to drop #: privileges, write the PID file (with :class:`PidFile`), #: and daemonize this core. self.context = daemon.DaemonContext(**daemon_args) - __init__.__doc__ = BaseCore.__init__.__doc__.split('.. -----')[0] + __init__.__doc__ = NetworkCore.__init__.__doc__.split('.. -----')[0] def _dispatch(self, method, args, dispatch_dict): """ Dispatch XML-RPC method calls @@ -71,8 +72,9 @@ class Core(BaseCore): try: return method_func(*args) finally: - Bcfg2.Statistics.stats.add_value(method, - time.time() - method_start) + Bcfg2.Server.Statistics.stats.add_value( + method, + time.time() - method_start) except xmlrpclib.Fault: raise except Exception: @@ -100,7 +102,7 @@ class Core(BaseCore): err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: Failed to acquire" "lock on %s" % (self.name, - self.setup['daemon'])) + Bcfg2.Options.setup.daemon)) return False else: self.context.pidfile.release() @@ -111,19 +113,19 @@ class Core(BaseCore): def _run(self): """ Create :attr:`server` to start the server listening. """ - hostname, port = urlparse(self.setup['location'])[1].split(':') + hostname, port = urlparse(Bcfg2.Options.setup.server)[1].split(':') server_address = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM)[0][4] try: - self.server = XMLRPCServer(self.setup['listen_all'], + self.server = XMLRPCServer(Bcfg2.Options.setup.listen_all, server_address, - keyfile=self.setup['key'], - certfile=self.setup['cert'], + keyfile=Bcfg2.Options.setup.key, + certfile=Bcfg2.Options.setup.cert, register=False, - ca=self.setup['ca'], - protocol=self.setup['protocol']) + ca=Bcfg2.Options.setup.ca, + protocol=Bcfg2.Options.setup.protocol) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Server startup failed: %s" % err) diff --git a/src/lib/Bcfg2/Server/Cache.py b/src/lib/Bcfg2/Server/Cache.py new file mode 100644 index 000000000..d05eb0bf6 --- /dev/null +++ b/src/lib/Bcfg2/Server/Cache.py @@ -0,0 +1,180 @@ +""" ``Bcfg2.Server.Cache`` is an implementation of a simple +memory-backed cache. Right now this doesn't provide many features, but +more (time-based expiration, etc.) can be added as necessary. + +The normal workflow is to get a Cache object, which is simply a dict +interface to the unified cache that automatically uses a certain tag +set. For instance: + +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + groupcache['foo.example.com'] = ['group1', 'group2'] + +This would create a Cache object that automatically tags its entries +with ``frozenset(["Probes", "probegroups"])``, and store the list +``['group1', 'group1']`` with the *additional* tag +``foo.example.com``. So the unified backend cache would then contain +a single entry: + +.. code-block:: python + + {frozenset(["Probes", "probegroups", "foo.example.com"]): + ['group1', 'group2']} + +In addition to the dict interface, Cache objects (returned from +:func:`Bcfg2.Server.Cache.Cache`) have one additional method, +``expire()``, which is mostly identical to +:func:`Bcfg2.Server.Cache.expire`, except that it is specific to the +tag set of the cache object. E.g., to expire all ``foo.example.com`` +records for a given cache, you could do: + +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + groupcache.expire("foo.example.com") + +This is mostly functionally identical to: + +.. code-block:: python + + Bcfg2.Server.Cache.expire("Probes", "probegroups", "foo.example.com") + +It's not completely identical, though; the first example will expire, +at most, exactly one item from the cache. The second example will +expire all items that are tagged with a superset of the given tags. +To illustrate the difference, consider the following two examples: + +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes") + groupcache.expire("probegroups") + + Bcfg2.Server.Cache.expire("Probes", "probegroups") + +The former will not expire any data, because there is no single datum +tagged with ``"Probes", "probegroups"``. The latter will expire *all* +items tagged with ``"Probes", "probegroups"`` -- i.e., the entire +cache. In this case, the latter call is equivalent to: + +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + groupcache.expire() + +""" + +from Bcfg2.Compat import MutableMapping + + +class _Cache(MutableMapping): + """ The object returned by :func:`Bcfg2.Server.Cache.Cache` that + presents a dict-like interface to the portion of the unified cache + that uses the specified tags. """ + def __init__(self, registry, tags): + self._registry = registry + self._tags = tags + + def __getitem__(self, key): + return self._registry[self._tags | set([key])] + + def __setitem__(self, key, value): + self._registry[self._tags | set([key])] = value + + def __delitem__(self, key): + del self._registry[self._tags | set([key])] + + def __iter__(self): + for item in self._registry.iterate(*self._tags): + yield list(item.difference(self._tags))[0] + + def keys(self): + """ List cache keys """ + return list(iter(self)) + + def __len__(self): + return len(list(iter(self))) + + def expire(self, key=None): + """ expire all items, or a specific item, from the cache """ + if key is None: + expire(*self._tags) + else: + tags = self._tags | set([key]) + # py 2.5 doesn't support mixing *args and explicit keyword + # args + kwargs = dict(exact=True) + expire(*tags, **kwargs) + + def __repr__(self): + return repr(dict(self)) + + def __str__(self): + return str(dict(self)) + + +class _CacheRegistry(dict): + """ The grand unified cache backend which contains all cache + items. """ + + def iterate(self, *tags): + """ Iterate over all items that match the given tags *and* + have exactly one additional tag. This is used to get items + for :class:`Bcfg2.Server.Cache._Cache` objects that have been + instantiated via :func:`Bcfg2.Server.Cache.Cache`. """ + tags = frozenset(tags) + for key in self.keys(): + if key.issuperset(tags) and len(key.difference(tags)) == 1: + yield key + + def iter_all(self, *tags): + """ Iterate over all items that match the given tags, + regardless of how many additional tags they have (or don't + have). This is used to expire all cache data that matches a + set of tags. """ + tags = frozenset(tags) + for key in list(self.keys()): + if key.issuperset(tags): + yield key + + +_cache = _CacheRegistry() # pylint: disable=C0103 +_hooks = [] # pylint: disable=C0103 + + +def Cache(*tags): # pylint: disable=C0103 + """ A dict interface to the cache data tagged with the given + tags. """ + return _Cache(_cache, frozenset(tags)) + + +def expire(*tags, **kwargs): + """ Expire all items, a set of items, or one specific item from + the cache. If ``exact`` is set to True, then if the given tag set + doesn't match exactly one item in the cache, nothing will be + expired. """ + exact = kwargs.pop("exact", False) + count = 0 + if not tags: + count = len(_cache) + _cache.clear() + elif exact: + if frozenset(tags) in _cache: + count = 1 + del _cache[frozenset(tags)] + else: + for match in _cache.iter_all(*tags): + count += 1 + del _cache[match] + + for hook in _hooks: + hook(tags, exact, count) + + +def add_expire_hook(func): + """ Add a hook that will be called when an item is expired from + the cache. The callable passed in must take three options: the + first will be the tag set that was expired; the second will be the + state of the ``exact`` flag (True or False); and the third will be + the number of items that were expired from the cache. """ + _hooks.append(func) diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherrypyCore.py index d097fd08f..3cb0e291b 100644 --- a/src/lib/Bcfg2/Server/CherryPyCore.py +++ b/src/lib/Bcfg2/Server/CherrypyCore.py @@ -3,9 +3,9 @@ server. """ import sys import time -import Bcfg2.Statistics +import Bcfg2.Server.Statistics from Bcfg2.Compat import urlparse, xmlrpclib, b64decode -from Bcfg2.Server.Core import BaseCore +from Bcfg2.Server.Core import NetworkCore import cherrypy from cherrypy.lib import xmlrpcutil from cherrypy._cptools import ErrorTool @@ -27,7 +27,7 @@ def on_error(*args, **kwargs): # pylint: disable=W0613 cherrypy.tools.xmlrpc_error = ErrorTool(on_error) -class Core(BaseCore): +class CherrypyCore(NetworkCore): """ The CherryPy-based server core. """ #: Base CherryPy config for this class. We enable the @@ -36,8 +36,8 @@ class Core(BaseCore): _cp_config = {'tools.xmlrpc_error.on': True, 'tools.bcfg2_authn.on': True} - def __init__(self, setup): - BaseCore.__init__(self, setup) + def __init__(self): + NetworkCore.__init__(self) cherrypy.tools.bcfg2_authn = cherrypy.Tool('on_start_resource', self.do_authn) @@ -45,11 +45,11 @@ class Core(BaseCore): #: List of exposed plugin RMI self.rmi = self._get_rmi() cherrypy.engine.subscribe('stop', self.shutdown) - __init__.__doc__ = BaseCore.__init__.__doc__.split('.. -----')[0] + __init__.__doc__ = NetworkCore.__init__.__doc__.split('.. -----')[0] def do_authn(self): """ Perform authentication by calling - :func:`Bcfg2.Server.Core.BaseCore.authenticate`. This is + :func:`Bcfg2.Server.Core.NetworkCore.authenticate`. This is implemented as a CherryPy tool.""" try: header = cherrypy.request.headers['Authorization'] @@ -65,8 +65,15 @@ class Core(BaseCore): # FIXME: Get client cert cert = None - address = (cherrypy.request.remote.ip, cherrypy.request.remote.name) - return self.authenticate(cert, username, password, address) + address = (cherrypy.request.remote.ip, cherrypy.request.remote.port) + + rpcmethod = xmlrpcutil.process_body()[1] + if rpcmethod == 'ERRORMETHOD': + raise Exception("Unknown error processing XML-RPC request body") + + if (not self.check_acls(address[0], rpcmethod) or + not self.authenticate(cert, username, password, address)): + raise cherrypy.HTTPError(401) @cherrypy.expose def default(self, *args, **params): # pylint: disable=W0613 @@ -96,8 +103,8 @@ class Core(BaseCore): try: body = handler(*rpcparams, **params) finally: - Bcfg2.Statistics.stats.add_value(rpcmethod, - time.time() - method_start) + Bcfg2.Server.Statistics.stats.add_value(rpcmethod, + time.time() - method_start) xmlrpcutil.respond(body, 'utf-8', True) return cherrypy.serving.response.body @@ -108,36 +115,36 @@ class Core(BaseCore): with :class:`cherrypy.process.plugins.Daemonizer`, and write a PID file with :class:`cherrypy.process.plugins.PIDFile`. """ DropPrivileges(cherrypy.engine, - uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid'], - umask=int(self.setup['umask'], 8)).subscribe() + uid=Bcfg2.Options.setup.daemon_uid, + gid=Bcfg2.Options.setup.daemon_gid, + umask=int(Bcfg2.Options.setup.umask, 8)).subscribe() Daemonizer(cherrypy.engine).subscribe() - PIDFile(cherrypy.engine, self.setup['daemon']).subscribe() + PIDFile(cherrypy.engine, Bcfg2.Options.setup.daemon).subscribe() return True def _run(self): """ Start the server listening. """ - hostname, port = urlparse(self.setup['location'])[1].split(':') - if self.setup['listen_all']: + hostname, port = urlparse(Bcfg2.Options.setup.server)[1].split(':') + if Bcfg2.Options.setup.listen_all: hostname = '0.0.0.0' config = {'engine.autoreload.on': False, 'server.socket_port': int(port), 'server.socket_host': hostname} - if self.setup['cert'] and self.setup['key']: + if Bcfg2.Options.setup.cert and Bcfg2.Options.setup.key: config.update({'server.ssl_module': 'pyopenssl', - 'server.ssl_certificate': self.setup['cert'], - 'server.ssl_private_key': self.setup['key']}) - if self.setup['debug']: + 'server.ssl_certificate': Bcfg2.Options.setup.cert, + 'server.ssl_private_key': Bcfg2.Options.setup.key}) + if Bcfg2.Options.setup.debug: config['log.screen'] = True cherrypy.config.update(config) - cherrypy.tree.mount(self, '/', {'/': self.setup}) + cherrypy.tree.mount(self, '/', {'/': Bcfg2.Options.setup}) cherrypy.engine.start() return True def _block(self): """ Enter the blocking infinite server - loop. :func:`Bcfg2.Server.Core.BaseCore.shutdown` is called on + loop. :func:`Bcfg2.Server.Core.NetworkCore.shutdown` is called on exit by a :meth:`subscription <cherrypy.process.wspbus.Bus.subscribe>` on the top-level CherryPy engine.""" diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 6dfe4df1f..bc305e47a 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -11,26 +11,32 @@ import threading import time import inspect import lxml.etree -import Bcfg2.settings import Bcfg2.Server import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.DBSettings +import Bcfg2.Server.Statistics import Bcfg2.Server.FileMonitor -from Bcfg2.Cache import Cache -import Bcfg2.Statistics from itertools import chain +from Bcfg2.Server.Cache import Cache from Bcfg2.Compat import xmlrpclib, wraps # pylint: disable=W0622 from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin import track_statistics try: + from django.core.exceptions import ImproperlyConfigured + import django.conf + HAS_DJANGO = True +except ImportError: + HAS_DJANGO = False + +try: import psyco psyco.full() except ImportError: pass -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' - def exposed(func): """ Decorator that sets the ``exposed`` attribute of a function to @@ -95,46 +101,65 @@ class NoExposedMethod (Exception): method exposed with the given name. """ -# pylint: disable=W0702 +class DefaultACL(Plugin, ClientACLs): + """ Default ACL 'plugin' that provides security by default. This + is only loaded if no other ClientACLs plugin is enabled. """ + create = False + + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) + Bcfg2.Server.Plugin.ClientACLs.__init__(self) + + def check_acl_ip(self, address, rmi): + return (("." not in rmi and + not rmi.endswith("_debug") and + rmi != 'get_statistics') or + address[0] == "127.0.0.1") + # in core we frequently want to catch all exceptions, regardless of # type, so disable the pylint rule that catches that. +# pylint: disable=W0702 -class BaseCore(object): +class Core(object): """ The server core is the container for all Bcfg2 server logic and modules. All core implementations must inherit from - ``BaseCore``. """ - name = "core" - - def __init__(self, setup): # pylint: disable=R0912,R0915 + ``Core``. """ + + options = [ + Bcfg2.Options.Common.plugins, + Bcfg2.Options.Common.repository, + Bcfg2.Options.Common.filemonitor, + Bcfg2.Options.BooleanOption( + "--no-fam-blocking", cf=('server', 'fam_blocking'), + dest="fam_blocking", default=True, + help='FAM blocks on startup until all events are processed'), + Bcfg2.Options.BooleanOption( + cf=('logging', 'performance'), dest="perflog", + help="Periodically log performance statistics"), + Bcfg2.Options.Option( + cf=('logging', 'performance_interval'), default=300.0, + type=Bcfg2.Options.Types.timeout, + help="Performance statistics logging interval in seconds"), + Bcfg2.Options.Option( + cf=('caching', 'client_metadata'), dest='client_metadata_cache', + default='off', + choices=['off', 'on', 'initial', 'cautious', 'aggressive'])] + + #: The name of this server core. This can be overridden by core + #: implementations to provide a more specific name. + name = "Core" + + def __init__(self): # pylint: disable=R0912,R0915 """ - :param setup: A Bcfg2 options dict - :type setup: Bcfg2.Options.OptionParser - - .. automethod:: _daemonize .. automethod:: _run .. automethod:: _block .. ----- .. automethod:: _file_monitor_thread .. automethod:: _perflog_thread """ - #: The Bcfg2 repository directory - self.datastore = setup['repo'] - - if setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - # we set a higher log level for the console by default. we - # assume that if someone is running bcfg2-server in such a way - # that it _can_ log to console, they want more output. if - # level is set to DEBUG, that will get handled by - # setup_logging and the console will get DEBUG output. - Bcfg2.Logger.setup_logging('bcfg2-server', - to_console=logging.INFO, - to_syslog=setup['syslog'], - to_file=setup['logging'], - level=level) + # Record the core as a module variable + Bcfg2.Server.core = self #: A :class:`logging.Logger` object for use by the core self.logger = logging.getLogger('bcfg2-server') @@ -146,43 +171,32 @@ class BaseCore(object): #: special, and will be used for any log handlers whose name #: does not appear elsewhere in the dict. At a minimum, #: ``default`` must be provided. - self._loglevels = {True: dict(default=logging.DEBUG), - False: dict(console=logging.INFO, - default=level)} + self._loglevels = { + True: dict(default=logging.DEBUG), + False: dict(console=logging.INFO, + default=Bcfg2.Logger.default_log_level())} #: Used to keep track of the current debug state of the core. self.debug_flag = False # enable debugging on the core now. debugging is enabled on # everything else later - if setup['debug']: - self.set_core_debug(None, setup['debug']) - - try: - filemonitor = \ - Bcfg2.Server.FileMonitor.available[setup['filemonitor']] - except KeyError: - self.logger.error("File monitor driver %s not available; " - "forcing to default" % setup['filemonitor']) - filemonitor = Bcfg2.Server.FileMonitor.available['default'] - famargs = dict(ignore=[], debug=False) - if 'ignore' in setup: - famargs['ignore'] = setup['ignore'] - if 'debug' in setup: - famargs['debug'] = setup['debug'] + if Bcfg2.Options.setup.debug: + self.set_core_debug(None, Bcfg2.Options.setup.debug) try: #: The :class:`Bcfg2.Server.FileMonitor.FileMonitor` #: object used by the core to monitor for Bcfg2 data #: changes. - self.fam = filemonitor(**famargs) + self.fam = Bcfg2.Server.FileMonitor.get_fam() except IOError: - msg = "Failed to instantiate fam driver %s" % setup['filemonitor'] + msg = "Failed to instantiate fam driver %s" % \ + Bcfg2.Options.setup.filemonitor self.logger.error(msg, exc_info=1) raise CoreInitError(msg) #: Path to bcfg2.conf - self.cfile = setup['configfile'] + self.cfile = Bcfg2.Options.setup.config #: Dict of plugins that are enabled. Keys are the plugin #: names (just the plugin name, in the correct case; e.g., @@ -203,9 +217,6 @@ class BaseCore(object): #: :class:`Bcfg2.Server.Plugin.interfaces.Version` plugin. self.revision = '-1' - #: The Bcfg2 options dict - self.setup = setup - atexit.register(self.shutdown) #: if :func:`Bcfg2.Server.Core.shutdown` is called explicitly, #: then :mod:`atexit` calls it *again*, so it gets called @@ -221,65 +232,21 @@ class BaseCore(object): #: RLock to be held on writes to the backend db self.db_write_lock = threading.RLock() - # generate Django ORM settings. this must be done _before_ we - # load plugins - Bcfg2.settings.read_config(repo=self.datastore) - # mapping of group name => plugin name to record where groups # that are created by Connector plugins came from self._dynamic_groups = dict() - #: Whether or not it's possible to use the Django database - #: backend for plugins that have that capability - self._database_available = False - if Bcfg2.settings.HAS_DJANGO: - db_settings = Bcfg2.settings.DATABASES['default'] - if ('daemon' in self.setup and 'daemon_uid' in self.setup and - self.setup['daemon'] and self.setup['daemon_uid'] and - db_settings['ENGINE'].endswith(".sqlite3") and - not os.path.exists(db_settings['NAME'])): - # syncdb will create the sqlite database, and we're - # going to daemonize, dropping privs to a non-root - # user, so we need to chown the database after - # creating it - do_chown = True - else: - do_chown = False - - from django.core.exceptions import ImproperlyConfigured - from django.core import management - try: - management.call_command("syncdb", interactive=False, - verbosity=0) - self._database_available = True - except ImproperlyConfigured: - self.logger.error("Django configuration problem: %s" % - sys.exc_info()[1]) - except: - self.logger.error("Database update failed: %s" % - sys.exc_info()[1]) - - if do_chown and self._database_available: - try: - os.chown(db_settings['NAME'], - self.setup['daemon_uid'], - self.setup['daemon_gid']) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Failed to set ownership of database " - "at %s: %s" % (db_settings['NAME'], err)) - - #: The CA that signed the server cert - self.ca = setup['ca'] - #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ - threading.Thread(name="%sFAMThread" % setup['filemonitor'], + threading.Thread(name="%sFAMThread" % + Bcfg2.Options.setup.filemonitor.__name__, target=self._file_monitor_thread) + #: The :class:`threading.Thread` that reports performance + #: statistics to syslog. self.perflog_thread = None - if self.setup['perflog']: + if Bcfg2.Options.setup.perflog: self.perflog_thread = \ threading.Thread(name="PerformanceLoggingThread", target=self._perflog_thread) @@ -288,23 +255,28 @@ class BaseCore(object): #: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set` self.lock = threading.Lock() - #: A :class:`Bcfg2.Cache.Cache` object for caching client + #: A :class:`Bcfg2.Server.Cache.Cache` object for caching client #: metadata - self.metadata_cache = Cache() + self.metadata_cache = Cache("Metadata") - def expire_caches_by_type(self, base_cls, key=None): - """ Expire caches for all - :class:`Bcfg2.Server.Plugin.interfaces.Caching` plugins that - are instances of ``base_cls``. + #: Whether or not it's possible to use the Django database + #: backend for plugins that have that capability + self._database_available = False + if HAS_DJANGO: + try: + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=0) + self._database_available = True + except ImproperlyConfigured: + self.logger.error("Django configuration problem: %s" % + sys.exc_info()[1]) + except: + self.logger.error("Updating database %s failed: %s" % + (Bcfg2.Options.setup.db_name, + sys.exc_info()[1])) - :param base_cls: The base plugin interface class to match (see - :mod:`Bcfg2.Server.Plugin.interfaces`) - :type base_cls: type - :param key: The cache key to expire - """ - for plugin in self.plugins_by_type(base_cls): - if isinstance(plugin, Bcfg2.Server.Plugin.Caching): - plugin.expire_cache(key) + def __str__(self): + return self.__class__.__name__ def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -331,7 +303,7 @@ class BaseCore(object): to syslog. """ self.logger.debug("Performance logging thread starting") while not self.terminate.isSet(): - self.terminate.wait(self.setup['perflog_interval']) + self.terminate.wait(Bcfg2.Options.setup.performance_interval) if not self.terminate.isSet(): for name, stats in self.get_statistics(None).items(): self.logger.info("Performance statistics: " @@ -348,17 +320,21 @@ class BaseCore(object): famfd = self.fam.fileno() terminate = self.terminate while not terminate.isSet(): - try: - if famfd: - select.select([famfd], [], [], 2) - else: - if not self.fam.pending(): - terminate.wait(15) - if self.fam.pending(): + if famfd: + select.select([famfd], [], [], 2) + elif not self.fam.pending(): + terminate.wait(15) + if self.fam.pending(): + try: self._update_vcs_revision() + except: + self.logger.error("Error updating VCS revision: %s" % + sys.exc_info()[1]) + try: self.fam.handle_event_set(self.lock) except: - continue + self.logger.error("Error handling event set: %s" % + sys.exc_info()[1]) self.logger.info("File monitor thread terminated") @track_statistics() @@ -383,10 +359,7 @@ class BaseCore(object): :attr:`Bcfg2.Server.Core.BaseCore.metadata` as side effects. This does not start plugin threads; that is done later, in :func:`Bcfg2.Server.Core.BaseCore.run` """ - while '' in self.setup['plugins']: - self.setup['plugins'].remove('') - - for plugin in self.setup['plugins']: + for plugin in Bcfg2.Options.setup.plugins: if plugin not in self.plugins: self.init_plugin(plugin) @@ -398,17 +371,18 @@ class BaseCore(object): for plug in blacklist: del self.plugins[plug] - # Log experimental plugins - expl = [plug for plug in list(self.plugins.values()) - if plug.experimental] + # Log deprecated and experimental plugins + expl = [] + depr = [] + for plug in list(self.plugins.values()): + if plug.experimental: + expl.append(plug) + if plug.deprecated: + depr.append(plug) if expl: self.logger.info("Loading experimental plugin(s): %s" % (" ".join([x.name for x in expl]))) self.logger.info("NOTE: Interfaces subject to change") - - # Log deprecated plugins - depr = [plug for plug in list(self.plugins.values()) - if plug.deprecated] if depr: self.logger.info("Loading deprecated plugin(s): %s" % (" ".join([x.name for x in depr]))) @@ -425,43 +399,25 @@ class BaseCore(object): "failed to instantiate Core") raise CoreInitError("No Metadata Plugin") - if self.debug_flag: - # enable debugging on plugins - self.plugins[plugin].set_debug(self.debug_flag) + # ensure that an ACL plugin is loaded + if not self.plugins_by_type(Bcfg2.Server.Plugin.ClientACLs): + self.init_plugin(DefaultACL) def init_plugin(self, plugin): """ Import and instantiate a single plugin. The plugin is stored to :attr:`plugins`. - :param plugin: The name of the plugin. This is just the name - of the plugin, in the appropriate case. I.e., - ``Cfg``, not ``Bcfg2.Server.Plugins.Cfg``. - :type plugin: string + :param plugin: The plugin class to load. + :type plugin: type :returns: None """ - self.logger.debug("%s: Loading plugin %s" % (self.name, plugin)) - try: - mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % - (plugin)).Server.Plugins, plugin) - except ImportError: - try: - mod = __import__(plugin, globals(), locals(), - [plugin.split('.')[-1]]) - except: - self.logger.error("Failed to load plugin %s" % plugin) - return - try: - plug = getattr(mod, plugin.split('.')[-1]) - except AttributeError: - self.logger.error("Failed to load plugin %s: %s" % - (plugin, sys.exc_info()[1])) - return + self.logger.debug("Loading plugin %s" % plugin.name) # Blacklist conflicting plugins - cplugs = [conflict for conflict in plug.conflicts + cplugs = [conflict for conflict in plugin.conflicts if conflict in self.plugins] - self.plugin_blacklist[plug.name] = cplugs + self.plugin_blacklist[plugin.name] = cplugs try: - self.plugins[plugin] = plug(self, self.datastore) + self.plugins[plugin.name] = plugin(self) except PluginInitError: self.logger.error("Failed to instantiate plugin %s" % plugin, exc_info=1) @@ -497,8 +453,7 @@ class BaseCore(object): """ Get the client :attr:`metadata_cache` mode. Options are off, initial, cautious, aggressive, on (synonym for cautious). See :ref:`server-caching` for more details. """ - mode = self.setup.cfp.get("caching", "client_metadata", - default="off").lower() + mode = Bcfg2.Options.setup.client_metadata_cache if mode == "on": return "cautious" else: @@ -535,11 +490,12 @@ class BaseCore(object): self.logger.error("%s: Error invoking hook %s: %s" % (plugin, hook, err)) finally: - Bcfg2.Statistics.stats.add_value("%s:client_run_hook:%s" % - (self.__class__.__name__, hook), - time.time() - start) + Bcfg2.Server.Statistics.stats.add_value( + "%s:client_run_hook:%s" % + (self.__class__.__name__, hook), + time.time() - start) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def validate_structures(self, metadata, data): """ Checks the data structures by calling the :func:`Bcfg2.Server.Plugin.interfaces.StructureValidator.validate_structures` @@ -566,7 +522,7 @@ class BaseCore(object): self.logger.error("Plugin %s: unexpected structure validation " "failure" % plugin.name, exc_info=1) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def validate_goals(self, metadata, data): """ Checks that the config matches the goals enforced by :class:`Bcfg2.Server.Plugin.interfaces.GoalValidator` plugins @@ -592,7 +548,7 @@ class BaseCore(object): self.logger.error("Plugin %s: unexpected goal validation " "failure" % plugin.name, exc_info=1) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def GetStructures(self, metadata): """ Get all structures (i.e., bundles) for the given client @@ -611,7 +567,7 @@ class BaseCore(object): (metadata.hostname, ':'.join(missing))) return structures - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def BindStructures(self, structures, metadata, config): """ Given a list of structures (i.e. bundles), bind all the entries in them and add the structures to the config. @@ -632,7 +588,7 @@ class BaseCore(object): except: self.logger.error("error in BindStructure", exc_info=1) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def BindStructure(self, structure, metadata): """ Bind all elements in a single structure (i.e., bundle). @@ -708,10 +664,10 @@ class BaseCore(object): raise PluginExecutionError("No matching generator: %s:%s" % (entry.tag, entry.get('name'))) finally: - Bcfg2.Statistics.stats.add_value("%s:Bind:%s" % - (self.__class__.__name__, - entry.tag), - time.time() - start) + Bcfg2.Server.Statistics.stats.add_value("%s:Bind:%s" % + (self.__class__.__name__, + entry.tag), + time.time() - start) def BuildConfiguration(self, client): """ Build the complete configuration for a client. @@ -777,10 +733,13 @@ class BaseCore(object): self.logger.error("Got event for unknown file: %s" % event.filename) return - if event.code2str() == 'deleted': + if event.code2str() in ['deleted', 'exists']: + # ignore config file deletion, and ignore the initial + # 'exists' event as well. we've already parsed options on + # startup, we don't need to parse them twice. return - self.setup.reparse() - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + Bcfg2.Options.get_parser().reparse() + self.metadata_cache.expire() def block_for_fam_events(self, handle_events=False): """ Block until all fam events have been handleed, optionally @@ -792,7 +751,7 @@ class BaseCore(object): if handle_events: self.fam.handle_events_in_interval(1) slept += 1 - if self.setup['fam_blocking']: + if Bcfg2.Options.setup.fam_blocking: time.sleep(1) slept += 1 while self.fam.pending() != 0: @@ -803,35 +762,12 @@ class BaseCore(object): self.logger.debug("Slept %s seconds while handling FAM events" % slept) def run(self): - """ Run the server core. This calls :func:`_daemonize`, - :func:`_run`, starts the :attr:`fam_thread`, and calls - :func:`_block`, but note that it is the responsibility of the - server core implementation to call :func:`shutdown` under - normal operation. This also handles creation of the directory - containing the pidfile, if necessary. """ - if self.setup['daemon']: - # if we're dropping privs, then the pidfile is likely - # /var/run/bcfg2-server/bcfg2-server.pid or similar. - # since some OSes clean directories out of /var/run on - # reboot, we need to ensure that the directory containing - # the pidfile exists and has the appropriate permissions - piddir = os.path.dirname(self.setup['daemon']) - if not os.path.exists(piddir): - os.makedirs(piddir) - os.chown(piddir, - self.setup['daemon_uid'], - self.setup['daemon_gid']) - os.chmod(piddir, 493) # 0775 - if not self._daemonize(): - return False - - # rewrite $HOME. pulp stores its auth creds in ~/.pulp, so - # this is necessary to make that work when privileges are - # dropped - os.environ['HOME'] = pwd.getpwuid(self.setup['daemon_uid'])[5] - else: - os.umask(int(self.setup['umask'], 8)) - + """ Run the server core. This calls :func:`_run`, starts the + :attr:`fam_thread`, and calls :func:`_block`, but note that it + is the responsibility of the server core implementation to + call :func:`shutdown` under normal operation. This also + handles creation of the directory containing the pidfile, if + necessary.""" if not self._run(): self.shutdown() return False @@ -848,19 +784,12 @@ class BaseCore(object): for plug in self.plugins_by_type(Threaded): plug.start_threads() - if self.debug_flag: - self.set_debug(None, self.debug_flag) self.block_for_fam_events() self._block() except: self.shutdown() raise - def _daemonize(self): - """ Daemonize the server and write the pidfile. This must be - overridden by a core implementation. """ - raise NotImplementedError - def _run(self): """ Start up the server; this method should return immediately. This must be overridden by a core @@ -892,7 +821,62 @@ class BaseCore(object): % plugin.name, exc_info=1) return result - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() + def check_acls(self, address, rmi): + """ Check client IP address and metadata object against all + :class:`Bcfg2.Server.Plugin.interfaces.ClientACLs` plugins. + If any ACL plugin denies access, then access is denied. ACLs + are checked in two phases: First, with the client IP address; + and second, with the client metadata object. This lets an ACL + interface do a quick rejection based on IP before metadata is + ever built. + + :param address: The address pair of the client to check ACLs for + :type address: tuple of (<ip address>, <port>) + :param rmi: The fully-qualified name of the RPC call + :param rmi: string + :returns: bool + """ + plugins = self.plugins_by_type(Bcfg2.Server.Plugin.ClientACLs) + try: + ip_checks = [p.check_acl_ip(address, rmi) for p in plugins] + except: + self.logger.error("Unexpected error checking ACLs for %s for %s: " + "%s" % (address[0], rmi, sys.exc_info()[1])) + return False # failsafe + + if all(ip_checks): + # if all ACL plugins return True (allow), then allow + self.logger.debug("Client %s passed IP-based ACL checks for %s" % + (address[0], rmi)) + return True + elif False in ip_checks: + # if any ACL plugin returned False (deny), then deny + self.logger.warning("Client %s failed IP-based ACL checks for %s" % + (address[0], rmi)) + return False + # else, no plugins returned False, but not all plugins + # returned True, so some plugin returned None (defer), so + # defer. + + client, metadata = self.resolve_client(address) + try: + rv = all(p.check_acl_metadata(metadata, rmi) for p in plugins) + if rv: + self.logger.debug( + "Client %s passed metadata ACL checks for %s" % + (metadata.hostname, rmi)) + else: + self.logger.warning( + "Client %s failed metadata ACL checks for %s" % + (metadata.hostname, rmi)) + return rv + except: + self.logger.error("Unexpected error checking ACLs for %s for %s: " + "%s" % (client, rmi, sys.exc_info()[1])) + return False # failsafe + + @Bcfg2.Server.Statistics.track_statistics() def build_metadata(self, client_name): """ Build initial client metadata for a client @@ -992,7 +976,7 @@ class BaseCore(object): :param address: The address pair of the client to get the canonical hostname for. - :type address: tuple of (<ip address>, <hostname>) + :type address: tuple of (<ip address>, <port>) :param cleanup_cache: Tell the :class:`Bcfg2.Server.Plugin.interfaces.Metadata` plugin in :attr:`metadata` to clean up @@ -1075,21 +1059,23 @@ class BaseCore(object): def listMethods(self, address): # pylint: disable=W0613 """ List all exposed methods, including plugin RMI. - :param address: Client (address, hostname) pair + :param address: Client (address, port) pair :type address: tuple :returns: list of exposed method names """ methods = [name for name, func in inspect.getmembers(self, callable) - if getattr(func, "exposed", False)] - methods.extend(self._get_rmi().keys()) + if (getattr(func, "exposed", False) and + self.check_acls(address, name))] + methods.extend([m for m in self._get_rmi().keys() + if self.check_acls(address, m)]) return methods @exposed def methodHelp(self, address, method_name): # pylint: disable=W0613 """ Get help from the docstring of an exposed method - :param address: Client (address, hostname) pair + :param address: Client (address, port) pair :type address: tuple :param method_name: The name of the method to get help on :type method_name: string @@ -1107,7 +1093,7 @@ class BaseCore(object): def DeclareVersion(self, address, version): """ Declare the client version. - :param address: Client (address, hostname) pair + :param address: Client (address, port) pair :type address: tuple :param version: The client's declared version :type version: string @@ -1130,7 +1116,7 @@ class BaseCore(object): def GetProbes(self, address): """ Fetch probes for the client. - :param address: Client (address, hostname) pair + :param address: Client (address, port) pair :type address: tuple :returns: lxml.etree._Element - XML tree describing probes for this client @@ -1156,7 +1142,7 @@ class BaseCore(object): def RecvProbeData(self, address, probedata): """ Receive probe data from clients. - :param address: Client (address, hostname) pair + :param address: Client (address, port) pair :type address: tuple :returns: bool - True on success :raises: :exc:`xmlrpclib.Fault` @@ -1170,7 +1156,7 @@ class BaseCore(object): # that's created for RecvProbeData doesn't get cached. # I.e., the next metadata object that's built, after probe # data is processed, is cached. - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + self.metadata_cache.expire(client) try: xpdata = lxml.etree.XML(probedata.encode('utf-8'), parser=Bcfg2.Server.XMLParser) @@ -1204,7 +1190,7 @@ class BaseCore(object): def AssertProfile(self, address, profile): """ Set profile for a client. - :param address: Client (address, hostname) pair + :param address: Client (address, port) pair :type address: tuple :returns: bool - True on success :raises: :exc:`xmlrpclib.Fault` @@ -1225,7 +1211,7 @@ class BaseCore(object): """ Build config for a client by calling :func:`BuildConfiguration`. - :param address: Client (address, hostname) pair + :param address: Client (address, port) pair :type address: tuple :returns: lxml.etree._Element - The full configuration document for the client @@ -1244,7 +1230,7 @@ class BaseCore(object): def RecvStats(self, address, stats): """ Act on statistics upload with :func:`process_statistics`. - :param address: Client (address, hostname) pair + :param address: Client (address, port) pair :type address: tuple :returns: bool - True on success :raises: :exc:`xmlrpclib.Fault` @@ -1255,34 +1241,12 @@ class BaseCore(object): self.process_statistics(client, sdata) return True - def authenticate(self, cert, user, password, address): - """ Authenticate a client connection with - :func:`Bcfg2.Server.Plugin.interfaces.Metadata.AuthenticateConnection`. - - :param cert: an x509 certificate - :type cert: dict - :param user: The username of the user trying to authenticate - :type user: string - :param password: The password supplied by the client - :type password: string - :param address: An address pair of ``(<ip address>, <hostname>)`` - :type address: tuple - :return: bool - True if the authenticate succeeds, False otherwise - """ - if self.ca: - acert = cert - else: - # No ca, so no cert validation can be done - acert = None - return self.metadata.AuthenticateConnection(acert, user, password, - address) - @exposed @close_db_connection def GetDecisionList(self, address, mode): """ Get the decision list for the client with :func:`GetDecisions`. - :param address: Client (address, hostname) pair + :param address: Client (address, port) pair :type address: tuple :returns: list of decision tuples :raises: :exc:`xmlrpclib.Fault` @@ -1299,17 +1263,17 @@ class BaseCore(object): @exposed def get_statistics(self, _): """ Get current statistics about component execution from - :attr:`Bcfg2.Statistics.stats`. + :attr:`Bcfg2.Server.Statistics.stats`. :returns: dict - The statistics data as returned by - :func:`Bcfg2.Statistics.Statistics.display` """ - return Bcfg2.Statistics.stats.display() + :func:`Bcfg2.Server.Statistics.Statistics.display` """ + return Bcfg2.Server.Statistics.stats.display() @exposed def toggle_debug(self, address): """ Toggle debug status of the FAM and all plugins - :param address: Client (address, hostname) pair + :param address: Client (address, port) pair :type address: tuple :returns: bool - The new debug state of the FAM """ @@ -1399,3 +1363,125 @@ class BaseCore(object): address[0]) return "This method is deprecated and will be removed in a future " + \ "release\n%s" % self.fam.set_debug(debug) + + +class NetworkCore(Core): + """ A server core that actually listens on the network, can be + daemonized, etc.""" + options = Core.options + [ + Bcfg2.Options.Common.daemon, Bcfg2.Options.Common.syslog, + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.protocol, + Bcfg2.Options.PathOption( + '--ssl-key', cf=('communication', 'key'), dest="key", + help='Path to SSL key', + default="/etc/pki/tls/private/bcfg2.key"), + Bcfg2.Options.PathOption( + cf=('communication', 'certificate'), dest="cert", + help='Path to SSL certificate', + default="/etc/pki/tls/certs/bcfg2.crt"), + Bcfg2.Options.BooleanOption( + '--listen-all', cf=('server', 'listen_all'), default=False, + help="Listen on all interfaces"), + Bcfg2.Options.Option( + cf=('server', 'umask'), default='0077', help='Server umask', + type=Bcfg2.Options.Types.octal), + Bcfg2.Options.Option( + cf=('server', 'user'), default=0, dest='daemon_uid', + type=Bcfg2.Options.Types.username, + help="User to run the server daemon as"), + Bcfg2.Options.Option( + cf=('server', 'group'), default=0, dest='daemon_gid', + type=Bcfg2.Options.Types.groupname, + help="Group to run the server daemon as")] + + def __init__(self): + Core.__init__(self) + + #: The CA that signed the server cert + self.ca = Bcfg2.Options.setup.ca + + if self._database_available: + db_settings = django.conf.settings.DATABASES['default'] + if (Bcfg2.Options.setup.daemon and + Bcfg2.Options.setup.daemon_uid and + db_settings['ENGINE'].endswith(".sqlite3") and + not os.path.exists(db_settings['NAME'])): + # syncdb will create the sqlite database, and we're + # going to daemonize, dropping privs to a non-root + # user, so we need to chown the database after + # creating it + try: + os.chown(db_settings['NAME'], + Bcfg2.Options.setup.daemon_uid, + Bcfg2.Options.setup.daemon_gid) + except OSError: + err = sys.exc_info()[1] + self.logger.error("Failed to set ownership of database " + "at %s: %s" % (db_settings['NAME'], err)) + __init__.__doc__ = Core.__init__.__doc__.split(".. -----")[0] + \ + "\n.. automethod:: _daemonize\n" + + def __str__(self): + if hasattr(Bcfg2.Options.setup, "location"): + return "%s(%s)" % (self.__class__.__name__, + Bcfg2.Options.setup.location) + else: + return Core.__str__(self) + + def run(self): + """ Run the server core. This calls :func:`_daemonize` before + calling :func:`Bcfg2.Server.Core.Core.run` to run the server + core. """ + if Bcfg2.Options.setup.daemon: + # if we're dropping privs, then the pidfile is likely + # /var/run/bcfg2-server/bcfg2-server.pid or similar. + # since some OSes clean directories out of /var/run on + # reboot, we need to ensure that the directory containing + # the pidfile exists and has the appropriate permissions + piddir = os.path.dirname(Bcfg2.Options.setup.daemon) + if not os.path.exists(piddir): + os.makedirs(piddir) + os.chown(piddir, + Bcfg2.Options.setup.daemon_uid, + Bcfg2.Options.setup.daemon_gid) + os.chmod(piddir, 493) # 0775 + if not self._daemonize(): + return False + + # rewrite $HOME. pulp stores its auth creds in ~/.pulp, so + # this is necessary to make that work when privileges are + # dropped + os.environ['HOME'] = \ + pwd.getpwuid(Bcfg2.Options.setup.daemon_uid)[5] + else: + os.umask(int(Bcfg2.Options.setup.umask, 8)) + + Core.run(self) + + def authenticate(self, cert, user, password, address): + """ Authenticate a client connection with + :func:`Bcfg2.Server.Plugin.interfaces.Metadata.AuthenticateConnection`. + + :param cert: an x509 certificate + :type cert: dict + :param user: The username of the user trying to authenticate + :type user: string + :param password: The password supplied by the client + :type password: string + :param address: An address pair of ``(<ip address>, <port>)`` + :type address: tuple + :return: bool - True if the authenticate succeeds, False otherwise + """ + if self.ca: + acert = cert + else: + # No ca, so no cert validation can be done + acert = None + return self.metadata.AuthenticateConnection(acert, user, password, + address) + + def _daemonize(self): + """ Daemonize the server and write the pidfile. This must be + overridden by a core implementation. """ + raise NotImplementedError diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py new file mode 100755 index 000000000..b60302871 --- /dev/null +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -0,0 +1,681 @@ +""" Bcfg2.Server.Encryption provides a number of convenience methods +for handling encryption in Bcfg2. See :ref:`server-encryption` for +more details. """ + +import os +import sys +import copy +import logging +import lxml.etree +import Bcfg2.Logger +import Bcfg2.Options +from M2Crypto import Rand +from M2Crypto.EVP import Cipher, EVPError +from Bcfg2.Utils import safe_input +from Bcfg2.Server import XMLParser +from Bcfg2.Compat import md5, b64encode, b64decode, StringIO + +#: Constant representing the encryption operation for +#: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This +#: makes our code more readable. +ENCRYPT = 1 + +#: Constant representing the decryption operation for +#: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This +#: makes our code more readable. +DECRYPT = 0 + +#: Default initialization vector. For best security, you should use a +#: unique IV for each message. :func:`ssl_encrypt` does this in an +#: automated fashion. +IV = r'\0' * 16 + + +class _OptionContainer(object): + """ Container for options loaded at import-time to configure + encryption """ + options = [ + Bcfg2.Options.BooleanOption( + cf=("encryption", "lax_decryption"), + help="Decryption failures should cause warnings, not errors"), + Bcfg2.Options.Option( + cf=("encryption", "algorithm"), default="aes_256_cbc", + type=lambda v: v.lower().replace("-", "_"), + help="The encryption algorithm to use"), + Bcfg2.Options.Option( + cf=("encryption", "*"), dest='passphrases', default=dict(), + help="Encryption passphrases")] + + +Bcfg2.Options.get_parser().add_component(_OptionContainer) + +Rand.rand_seed(os.urandom(1024)) + + +def _cipher_filter(cipher, instr): + """ M2Crypto reads and writes file-like objects, so this uses + StringIO to pass data through it """ + inbuf = StringIO(instr) + outbuf = StringIO() + while 1: + buf = inbuf.read() + if not buf: + break + outbuf.write(cipher.update(buf)) + outbuf.write(cipher.final()) + rv = outbuf.getvalue() + inbuf.close() + outbuf.close() + return rv + + +def str_encrypt(plaintext, key, iv=IV, algorithm=None, salt=None): + """ Encrypt a string with a key. For a higher-level encryption + interface, see :func:`ssl_encrypt`. + + :param plaintext: The plaintext data to encrypt + :type plaintext: string + :param key: The key to encrypt the data with + :type key: string + :param iv: The initialization vector + :type iv: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :param salt: The salt to use + :type salt: string + :returns: string - The decrypted data + """ + if algorithm is None: + algorithm = Bcfg2.Options.setup.algorithm + cipher = Cipher(alg=algorithm, key=key, iv=iv, op=ENCRYPT, salt=salt) + return _cipher_filter(cipher, plaintext) + + +def str_decrypt(crypted, key, iv=IV, algorithm=None): + """ Decrypt a string with a key. For a higher-level decryption + interface, see :func:`ssl_decrypt`. + + :param crypted: The raw binary encrypted data + :type crypted: string + :param key: The encryption key to decrypt with + :type key: string + :param iv: The initialization vector + :type iv: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :returns: string - The decrypted data + """ + if algorithm is None: + algorithm = Bcfg2.Options.setup.algorithm + cipher = Cipher(alg=algorithm, key=key, iv=iv, op=DECRYPT) + return _cipher_filter(cipher, crypted) + + +def ssl_decrypt(data, passwd, algorithm=None): + """ Decrypt openssl-encrypted data. This can decrypt data + encrypted by :func:`ssl_encrypt`, or ``openssl enc``. It performs + a base64 decode first if the data is base64 encoded, and + automatically determines the salt and initialization vector (both + of which are embedded in the encrypted data). + + :param data: The encrypted data (either base64-encoded or raw + binary) to decrypt + :type data: string + :param passwd: The password to use to decrypt the data + :type passwd: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :returns: string - The decrypted data + """ + # base64-decode the data + data = b64decode(data) + salt = data[8:16] + # pylint: disable=E1101,E1121 + hashes = [md5(passwd + salt).digest()] + for i in range(1, 3): + hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + # pylint: enable=E1101,E1121 + key = hashes[0] + hashes[1] + iv = hashes[2] + + return str_decrypt(data[16:], key=key, iv=iv, algorithm=algorithm) + + +def ssl_encrypt(plaintext, passwd, algorithm=None, salt=None): + """ Encrypt data in a format that is openssl compatible. + + :param plaintext: The plaintext data to encrypt + :type plaintext: string + :param passwd: The password to use to encrypt the data + :type passwd: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :param salt: The salt to use. If none is provided, one will be + randomly generated. + :type salt: bytes + :returns: string - The base64-encoded, salted, encrypted string. + The string includes a trailing newline to make it fully + compatible with openssl command-line tools. + """ + if salt is None: + salt = Rand.rand_bytes(8) + + # pylint: disable=E1101,E1121 + hashes = [md5(passwd + salt).digest()] + for i in range(1, 3): + hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + # pylint: enable=E1101,E1121 + key = hashes[0] + hashes[1] + iv = hashes[2] + + crypted = str_encrypt(plaintext, key=key, salt=salt, iv=iv, + algorithm=algorithm) + return b64encode("Salted__" + salt + crypted) + "\n" + + +def is_encrypted(val): + """ Make a best guess if the value is encrypted or not. This just + checks to see if ``val`` is a base64-encoded string whose content + starts with "Salted__", so it may have (rare) false positives. It + will not have false negatives. """ + try: + return b64decode(val).startswith("Salted__") + except: # pylint: disable=W0702 + return False + + +def bruteforce_decrypt(crypted, passphrases=None, algorithm=None): + """ Convenience method to decrypt the given encrypted string by + trying the given passphrases or all passphrases sequentially until + one is found that works. + + :param crypted: The data to decrypt + :type crypted: string + :param passphrases: The passphrases to try. + :type passphrases: list + :param algorithm: The cipher algorithm to use + :type algorithm: string + :returns: string - The decrypted data + :raises: :class:`M2Crypto.EVP.EVPError`, if the data cannot be decrypted + """ + if passphrases is None: + passphrases = Bcfg2.Options.setup.passphrases.values() + for passwd in passphrases: + try: + return ssl_decrypt(crypted, passwd, algorithm=algorithm) + except EVPError: + pass + raise EVPError("Failed to decrypt") + + +def print_xml(element, keep_text=False): + """ Render an XML element for error output. This prefixes the + line number and removes children for nicer display. + + :param element: The element to render + :type element: lxml.etree._Element + :param keep_text: Do not discard text content from the element for + display + :type keep_text: boolean + """ + xml = None + if len(element) or element.text: + el = copy.copy(element) + if el.text and not keep_text: + el.text = '...' + for child in el.iterchildren(): + el.remove(child) + xml = lxml.etree.tostring( + el, + xml_declaration=False).decode("UTF-8").strip() + else: + xml = lxml.etree.tostring( + element, + xml_declaration=False).decode("UTF-8").strip() + return "%s (line %s)" % (xml, element.sourceline) + + +class PassphraseError(Exception): + """ Exception raised when there's a problem determining the + passphrase to encrypt or decrypt with """ + + +class DecryptError(Exception): + """ Exception raised when decryption fails. """ + + +class EncryptError(Exception): + """ Exception raised when encryption fails. """ + + +class CryptoTool(object): + """ Generic decryption/encryption interface base object """ + + def __init__(self, filename): + self.logger = logging.getLogger(self.__class__.__name__) + self.filename = filename + self.data = open(self.filename).read() + self.pname, self.passphrase = self._get_passphrase() + + def _get_passphrase(self): + """ get the passphrase for the current file """ + if not Bcfg2.Options.setup.passphrases: + raise PassphraseError("No passphrases available in %s" % + Bcfg2.Options.setup.config) + + pname = None + if Bcfg2.Options.setup.passphrase: + pname = Bcfg2.Options.setup.passphrase + + if pname: + try: + passphrase = Bcfg2.Options.setup.passphrases[pname] + self.logger.debug("Using passphrase %s specified on command " + "line" % pname) + return (pname, passphrase) + except KeyError: + raise PassphraseError("Could not find passphrase %s in %s" % + (pname, Bcfg2.Options.setup.config)) + else: + if len(Bcfg2.Options.setup.passphrases) == 1: + pname, passphrase = Bcfg2.Options.setup.passphrases.items()[0] + self.logger.info("Using passphrase %s" % pname) + return (pname, passphrase) + elif len(Bcfg2.Options.setup.passphrases) > 1: + return (None, None) + raise PassphraseError("No passphrase could be determined") + + def get_destination_filename(self, original_filename): + """ Get the filename where data should be written """ + return original_filename + + def write(self, data): + """ write data to disk """ + new_fname = self.get_destination_filename(self.filename) + try: + self._write(new_fname, data) + self.logger.info("Wrote data to %s" % new_fname) + return True + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error writing data from %s to %s: %s" % + (self.filename, new_fname, err)) + return False + + def _write(self, filename, data): + """ Perform the actual write of data. This is separate from + :func:`CryptoTool.write` so it can be easily + overridden. """ + open(filename, "wb").write(data) + + +class Decryptor(CryptoTool): + """ Decryptor interface """ + def decrypt(self): + """ decrypt the file, returning the encrypted data """ + raise NotImplementedError + + +class Encryptor(CryptoTool): + """ encryptor interface """ + def encrypt(self): + """ encrypt the file, returning the encrypted data """ + raise NotImplementedError + + +class CfgEncryptor(Encryptor): + """ encryptor class for Cfg files """ + + def __init__(self, filename): + Encryptor.__init__(self, filename) + if self.passphrase is None: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + Bcfg2.Options.setup.config) + + def encrypt(self): + if is_encrypted(self.data): + raise EncryptError("Data is alraedy encrypted") + return ssl_encrypt(self.data, self.passphrase) + + def get_destination_filename(self, original_filename): + return original_filename + ".crypt" + + +class CfgDecryptor(Decryptor): + """ Decrypt Cfg files """ + + def decrypt(self): + """ decrypt the given file, returning the plaintext data """ + if self.passphrase: + try: + return ssl_decrypt(self.data, self.passphrase) + except EVPError: + raise DecryptError("Could not decrypt %s with the " + "specified passphrase" % self.filename) + except: + raise DecryptError("Error decrypting %s: %s" % + (self.filename, sys.exc_info()[1])) + else: # no passphrase given, brute force + try: + return bruteforce_decrypt(self.data) + except EVPError: + raise DecryptError("Could not decrypt %s with any passphrase" % + self.filename) + + def get_destination_filename(self, original_filename): + if original_filename.endswith(".crypt"): + return original_filename[:-6] + else: + return Decryptor.get_destination_filename(self, original_filename) + + +class PropertiesCryptoMixin(object): + """ Mixin to provide some common methods for Properties crypto """ + default_xpath = '//*[@encrypted]' + + def _get_elements(self, xdata): + """ Get the list of elements to encrypt or decrypt """ + if Bcfg2.Options.setup.xpath: + elements = xdata.xpath(Bcfg2.Options.setup.xpath) + if not elements: + self.logger.warning("XPath expression %s matched no elements" % + Bcfg2.Options.setup.xpath) + else: + elements = xdata.xpath(self.default_xpath) + if not elements: + elements = list(xdata.getiterator(tag=lxml.etree.Element)) + + # filter out elements without text data + for el in elements[:]: + if not el.text: + elements.remove(el) + + if Bcfg2.Options.setup.interactive: + for element in elements[:]: + if len(element): + elt = copy.copy(element) + for child in elt.iterchildren(): + elt.remove(child) + else: + elt = element + print(lxml.etree.tostring( + elt, + xml_declaration=False).decode("UTF-8").strip()) + ans = safe_input("Encrypt this element? [y/N] ") + if not ans.lower().startswith("y"): + elements.remove(element) + return elements + + def _get_element_passphrase(self, element): + """ Get the passphrase to use to encrypt or decrypt a given + element """ + pname = element.get("encrypted") + if pname in Bcfg2.Options.setup.passphrases: + passphrase = Bcfg2.Options.setup.passphrases[pname] + else: + if pname: + self.logger.warning("Passphrase %s not found in %s, " + "using passphrase given on command line" % + (pname, Bcfg2.Options.setup.config)) + if self.passphrase: + passphrase = self.passphrase + pname = self.pname + else: + self.logger.warning("No passphrase specified for %s element" % + element.tag) + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with " + "-p" % Bcfg2.Options.setup.config) + return (pname, passphrase) + + def _write(self, filename, data): + """ Write the data """ + data.getroottree().write(filename, + xml_declaration=False, + pretty_print=True) + + +class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): + """ encryptor class for Properties files """ + + def encrypt(self): + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + if is_encrypted(elt.text): + raise EncryptError("Element is already encrypted: %s" % + print_xml(elt)) + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + raise EncryptError(str(sys.exc_info()[1])) + self.logger.debug("Encrypting %s" % print_xml(elt)) + elt.text = ssl_encrypt(elt.text, passphrase).strip() + elt.set("encrypted", pname) + return xdata + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) + + +class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): + """ decryptor class for Properties files """ + + def decrypt(self): + decrypted_any = False + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + raise DecryptError(str(sys.exc_info()[1])) + self.logger.debug("Decrypting %s" % print_xml(elt)) + try: + decrypted = ssl_decrypt(elt.text, passphrase).strip() + elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') + elt.set("encrypted", pname) + decrypted_any = True + except (EVPError, TypeError): + self.logger.error("Could not decrypt %s, skipping" % + print_xml(elt)) + except UnicodeDecodeError: + # we managed to decrypt the value, but it contains + # content that can't even be encoded into xml + # entities. what probably happened here is that we + # coincidentally could decrypt a value encrypted with + # a different key, and wound up with gibberish. + self.logger.warning("Decrypted %s to gibberish, skipping" % + elt.tag) + if decrypted_any: + return xdata + else: + raise DecryptError("Failed to decrypt any data in %s" % + self.filename) + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) + + +class CLI(object): + """ The bcfg2-crypt CLI """ + + options = [ + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption( + "--encrypt", help='Encrypt the specified file'), + Bcfg2.Options.BooleanOption( + "--decrypt", help='Decrypt the specified file')), + Bcfg2.Options.BooleanOption( + "--stdout", + help='Decrypt or encrypt the specified file to stdout'), + Bcfg2.Options.Option( + "-p", "--passphrase", metavar="NAME", + help='Encryption passphrase name'), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption( + "--properties", + help='Encrypt the specified file as a Properties file'), + Bcfg2.Options.BooleanOption( + "--cfg", help='Encrypt the specified file as a Cfg file')), + Bcfg2.Options.OptionGroup( + Bcfg2.Options.Common.interactive, + Bcfg2.Options.Option( + "--xpath", + help='XPath expression to select elements to encrypt'), + title="Options for handling Properties files"), + Bcfg2.Options.OptionGroup( + Bcfg2.Options.BooleanOption( + "--remove", help='Remove the plaintext file after encrypting'), + title="Options for handling Cfg files"), + Bcfg2.Options.PathOption( + "files", help="File(s) to encrypt or decrypt", nargs='+')] + + def __init__(self, argv=None): + parser = Bcfg2.Options.get_parser( + description="Encrypt and decrypt Bcfg2 data", + components=[self, _OptionContainer]) + parser.parse(argv=argv) + self.logger = logging.getLogger(parser.prog) + + if Bcfg2.Options.setup.decrypt: + if Bcfg2.Options.setup.remove: + self.logger.error("--remove cannot be used with --decrypt, " + "ignoring --remove") + Bcfg2.Options.setup.remove = False + elif Bcfg2.Options.setup.interactive: + self.logger.error("Cannot decrypt interactively") + Bcfg2.Options.setup.interactive = False + + def _is_properties(self, filename): + """ Determine if a given file is a Properties file or not """ + if Bcfg2.Options.setup.properties: + return True + elif Bcfg2.Options.setup.cfg: + return False + elif filename.endswith(".xml"): + try: + xroot = lxml.etree.parse(filename).getroot() + return xroot.tag == "Properties" + except lxml.etree.XMLSyntaxError: + return False + else: + return False + + def run(self): # pylint: disable=R0912,R0915 + """ Run bcfg2-crypt """ + for fname in Bcfg2.Options.setup.files: + if not os.path.exists(fname): + self.logger.error("%s does not exist, skipping" % fname) + continue + + # figure out if we need to encrypt this as a Properties file + # or as a Cfg file + try: + props = self._is_properties(fname) + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + + if props: + ftype = "Properties" + if Bcfg2.Options.setup.remove: + self.logger.info("Cannot use --remove with Properties " + "file %s, ignoring for this file" % fname) + tools = (PropertiesEncryptor, PropertiesDecryptor) + else: + ftype = "Cfg" + if Bcfg2.Options.setup.xpath: + self.logger.error("Specifying --xpath with --cfg is " + "nonsensical, ignoring --xpath") + Bcfg2.Options.setup.xpath = None + if Bcfg2.Options.setup.interactive: + self.logger.error("Cannot use interactive mode with " + "--cfg, ignoring --interactive") + Bcfg2.Options.setup.interactive = False + tools = (CfgEncryptor, CfgDecryptor) + + data = None + mode = None + if Bcfg2.Options.setup.encrypt: + try: + tool = tools[0](fname) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + continue + except IOError: + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + mode = "encrypt" + self.logger.debug("Encrypting %s file %s" % (ftype, fname)) + elif Bcfg2.Options.setup.decrypt: + try: + tool = tools[1](fname) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + continue + except IOError: + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + mode = "decrypt" + self.logger.debug("Decrypting %s file %s" % (ftype, fname)) + else: + self.logger.info("Neither --encrypt nor --decrypt specified, " + "determining mode") + try: + tool = tools[1](fname) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + continue + except IOError: + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + try: + self.logger.debug("Trying to decrypt %s file %s" % (ftype, + fname)) + data = tool.decrypt() + mode = "decrypt" + self.logger.debug("Decrypted %s file %s" % (ftype, fname)) + except DecryptError: + self.logger.info("Failed to decrypt %s, trying encryption" + % fname) + try: + tool = tools[0](fname) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + continue + except IOError: + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + mode = "encrypt" + self.logger.debug("Encrypting %s file %s" % (ftype, fname)) + + if data is None: + try: + data = getattr(tool, mode)() + except (EncryptError, DecryptError): + self.logger.error("Failed to %s %s, skipping: %s" % + (mode, fname, sys.exc_info()[1])) + continue + if Bcfg2.Options.setup.stdout: + if len(Bcfg2.Options.setup.files) > 1: + print("----- %s -----" % fname) + print(data) + if len(Bcfg2.Options.setup.files) > 1: + print("") + else: + tool.write(data) + + if (Bcfg2.Options.setup.remove and + tool.get_destination_filename(fname) != fname): + try: + os.unlink(fname) + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error removing %s: %s" % (fname, err)) + continue diff --git a/src/lib/Bcfg2/Server/FileMonitor/Fam.py b/src/lib/Bcfg2/Server/FileMonitor/Fam.py deleted file mode 100644 index 09d41038e..000000000 --- a/src/lib/Bcfg2/Server/FileMonitor/Fam.py +++ /dev/null @@ -1,105 +0,0 @@ -""" File monitor backend with support for the `File Alteration Monitor -<http://oss.sgi.com/projects/fam/>`_. The FAM backend is deprecated. """ - -import os -import _fam -import stat -import logging -from time import time -from Bcfg2.Server.FileMonitor import FileMonitor - -LOGGER = logging.getLogger(__name__) - - -class Fam(FileMonitor): - """ **Deprecated** file monitor backend with support for the `File - Alteration Monitor <http://oss.sgi.com/projects/fam/>`_ (also - abbreviated "FAM").""" - - #: FAM is the worst actual monitor backend, so give it a low - #: priority. - __priority__ = 10 - - def __init__(self, ignore=None, debug=False): - FileMonitor.__init__(self, ignore=ignore, debug=debug) - self.filemonitor = _fam.open() - self.users = {} - LOGGER.warning("The Fam file monitor backend is deprecated. Please " - "switch to a supported file monitor.") - __init__.__doc__ = FileMonitor.__init__.__doc__ - - def fileno(self): - return self.filemonitor.fileno() - fileno.__doc__ = FileMonitor.fileno.__doc__ - - def handle_event_set(self, _=None): - self.Service() - handle_event_set.__doc__ = FileMonitor.handle_event_set.__doc__ - - def handle_events_in_interval(self, interval): - now = time() - while (time() - now) < interval: - if self.Service(): - now = time() - handle_events_in_interval.__doc__ = \ - FileMonitor.handle_events_in_interval.__doc__ - - def AddMonitor(self, path, obj, _=None): - mode = os.stat(path)[stat.ST_MODE] - if stat.S_ISDIR(mode): - handle = self.filemonitor.monitorDirectory(path, None) - else: - handle = self.filemonitor.monitorFile(path, None) - self.handles[handle.requestID()] = handle - if obj is not None: - self.users[handle.requestID()] = obj - return handle.requestID() - AddMonitor.__doc__ = FileMonitor.AddMonitor.__doc__ - - def Service(self, interval=0.50): - """ Handle events for the specified period of time (in - seconds). This call will block for ``interval`` seconds. - - :param interval: The interval, in seconds, during which events - should be handled. Any events that are - already pending when :func:`Service` is - called will also be handled. - :type interval: int - :returns: None - """ - count = 0 - collapsed = 0 - rawevents = [] - start = time() - now = time() - while (time() - now) < interval: - if self.filemonitor.pending(): - while self.filemonitor.pending(): - count += 1 - rawevents.append(self.filemonitor.nextEvent()) - now = time() - unique = [] - bookkeeping = [] - for event in rawevents: - if self.should_ignore(event): - continue - if event.code2str() != 'changed': - # process all non-change events - unique.append(event) - else: - if (event.filename, event.requestID) not in bookkeeping: - bookkeeping.append((event.filename, event.requestID)) - unique.append(event) - else: - collapsed += 1 - for event in unique: - if event.requestID in self.users: - try: - self.users[event.requestID].HandleEvent(event) - except: # pylint: disable=W0702 - LOGGER.error("Handling event for file %s" % event.filename, - exc_info=1) - end = time() - LOGGER.info("Processed %s fam events in %03.03f seconds. " - "%s coalesced" % (count, (end - start), collapsed)) - return count diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 9134758b8..b349d20fd 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -27,14 +27,14 @@ class GaminEvent(Event): class Gamin(FileMonitor): """ File monitor backend with `Gamin - <http://people.gnome.org/~veillard/gamin/>`_ support. """ + <http://people.gnome.org/~veillard/gamin/>`_ support. **Deprecated.** """ - #: The Gamin backend is fairly decent, particularly newer - #: releases, so it has a fairly high priority. - __priority__ = 90 + #: The Gamin backend is deprecated, but better than pseudo, so it + #: has a medium priority. + __priority__ = 50 - def __init__(self, ignore=None, debug=False): - FileMonitor.__init__(self, ignore=ignore, debug=debug) + def __init__(self): + FileMonitor.__init__(self) #: The :class:`Gamin.WatchMonitor` object for this monitor. self.mon = None @@ -46,6 +46,9 @@ class Gamin(FileMonitor): #: The queue used to record monitors that are added before #: :func:`start` has been called and :attr:`mon` is created. self.add_q = [] + + self.logger.warning("The Gamin file monitor backend is deprecated. " + "Please switch to a supported file monitor.") __init__.__doc__ = FileMonitor.__init__.__doc__ def start(self): diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index bce7ce7c2..c4b34a469 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -34,8 +34,8 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): #: listed in :attr:`action_map` mask = reduce(lambda x, y: x | y, action_map.keys()) - def __init__(self, ignore=None, debug=False): - Pseudo.__init__(self, ignore=ignore, debug=debug) + def __init__(self): + Pseudo.__init__(self) pyinotify.ProcessEvent.__init__(self) #: inotify can't set useful monitors directly on files, only @@ -149,7 +149,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): evt = Event(handleID, path, action) if (ievent.wd not in self.event_filter or - ievent.pathname in self.event_filter[ievent.wd]): + ievent.pathname in self.event_filter[ievent.wd]): self.events.append(evt) def AddMonitor(self, path, obj, handleID=None): diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 7a5d901fd..8e0dd2efe 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -48,11 +48,9 @@ Base Classes import os import sys import fnmatch -import logging +import Bcfg2.Options from time import sleep, time -from Bcfg2.Server.Plugin import Debuggable - -LOGGER = logging.getLogger(__name__) +from Bcfg2.Logger import Debuggable class Event(object): @@ -112,6 +110,14 @@ class FileMonitor(Debuggable): monitor objects to :attr:`handles` and received events to :attr:`events`; the basic interface will handle the rest. """ + options = [ + Bcfg2.Options.Option( + cf=('server', 'ignore_files'), + help='File globs to ignore', + type=Bcfg2.Options.Types.comma_list, + default=['*~', '*#', '.#*', '*.swp', '*.swpx', '.*.swx', + 'SCCS', '.svn', '4913', '.gitignore'])] + #: The relative priority of this FAM backend. Better backends #: should have higher priorities. __priority__ = -1 @@ -119,7 +125,7 @@ class FileMonitor(Debuggable): #: List of names of methods to be exposed as XML-RPC functions __rmi__ = Debuggable.__rmi__ + ["list_event_handlers"] - def __init__(self, ignore=None, debug=False): + def __init__(self): """ :param ignore: A list of filename globs describing events that should be ignored (i.e., not processed by any @@ -133,7 +139,6 @@ class FileMonitor(Debuggable): .. autoattribute:: __priority__ """ Debuggable.__init__(self) - self.debug_flag = debug #: A dict that records which objects handle which events. #: Keys are monitor handle IDs and values are objects whose @@ -143,12 +148,10 @@ class FileMonitor(Debuggable): #: Queue of events to handle self.events = [] - if ignore is None: - ignore = [] #: List of filename globs to ignore events for. For events #: that include the full path, both the full path and the bare #: filename will be checked against ``ignore``. - self.ignore = ignore + self.ignore = Bcfg2.Options.setup.ignore_files #: Whether or not the FAM has been started. See :func:`start`. self.started = False @@ -185,7 +188,8 @@ class FileMonitor(Debuggable): """ for pattern in self.ignore: if (fnmatch.fnmatch(event.filename, pattern) or - fnmatch.fnmatch(os.path.split(event.filename)[-1], pattern)): + fnmatch.fnmatch(os.path.split(event.filename)[-1], + pattern)): self.debug_log("Ignoring %s" % event) return True return False @@ -226,8 +230,8 @@ class FileMonitor(Debuggable): if self.should_ignore(event): return if event.requestID not in self.handles: - LOGGER.info("Got event for unexpected id %s, file %s" % - (event.requestID, event.filename)) + self.logger.info("Got event for unexpected id %s, file %s" % + (event.requestID, event.filename)) return self.debug_log("Dispatching event %s %s to obj %s" % (event.code2str(), event.filename, @@ -238,9 +242,8 @@ class FileMonitor(Debuggable): raise except: # pylint: disable=W0702 err = sys.exc_info()[1] - LOGGER.error("Error in handling of event %s for %s: %s" % - (event.code2str(), event.filename, err), - exc_info=1) + self.logger.error("Error in handling of event %s for %s: %s" % + (event.code2str(), event.filename, err)) def handle_event_set(self, lock=None): """ Handle all pending events. @@ -266,7 +269,8 @@ class FileMonitor(Debuggable): lock.release() end = time() if count > 0: - LOGGER.info("Handled %d events in %.03fs" % (count, (end - start))) + self.logger.info("Handled %d events in %.03fs" % (count, + (end - start))) def handle_events_in_interval(self, interval): """ Handle events for the specified period of time (in @@ -328,6 +332,25 @@ class FileMonitor(Debuggable): return rv +#: A module-level FAM object that all plugins, etc., can use. This +#: should not be used directly, but retrieved via :func:`get_fam`. +_FAM = None + + +def get_fam(): + """ Get a + :class:`Bcfg2.Server.FileMonitor.FileMonitor` object. If + :attr:`_FAM` has not been populated, then a new default + FileMonitor will be created. + + :returns: :class:`Bcfg2.Server.FileMonitor.FileMonitor` + """ + global _FAM # pylint: disable=W0603 + if _FAM is None: + _FAM = Bcfg2.Options.setup.filemonitor() + return _FAM + + #: A dict of all available FAM backends. Keys are the human-readable #: names of the backends, which are used in bcfg2.conf to select a #: backend; values are the backend classes. In addition, the @@ -340,12 +363,6 @@ from Bcfg2.Server.FileMonitor.Pseudo import Pseudo available['pseudo'] = Pseudo try: - from Bcfg2.Server.FileMonitor.Fam import Fam - available['fam'] = Fam -except ImportError: - pass - -try: from Bcfg2.Server.FileMonitor.Gamin import Gamin available['gamin'] = Gamin except ImportError: diff --git a/src/lib/Bcfg2/Server/Hostbase/.gitignore b/src/lib/Bcfg2/Server/Hostbase/.gitignore deleted file mode 100644 index 8e15b5395..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.pyc -dev.db -bcfg2.conf diff --git a/src/lib/Bcfg2/Server/Hostbase/__init__.py b/src/lib/Bcfg2/Server/Hostbase/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/__init__.py +++ /dev/null diff --git a/src/lib/Bcfg2/Server/Hostbase/backends.py b/src/lib/Bcfg2/Server/Hostbase/backends.py deleted file mode 100644 index cfa9e1e16..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/backends.py +++ /dev/null @@ -1,63 +0,0 @@ -from django.contrib.auth.models import User -#from ldapauth import * -from nisauth import * - -## class LDAPBackend(object): - -## def authenticate(self,username=None,password=None): -## try: - -## l = ldapauth(username,password) -## temp_pass = User.objects.make_random_password(100) -## ldap_user = dict(username=l.sAMAccountName, -## ) -## user_session_obj = dict( -## email=l.email, -## first_name=l.name_f, -## last_name=l.name_l, -## uid=l.badge_no -## ) -## #fixme: need to add this user session obj to session -## user,created = User.objects.get_or_create(username=username) -## return user - -## except LDAPAUTHError,e: -## return None - -## def get_user(self,user_id): -## try: -## return User.objects.get(pk=user_id) -## except User.DoesNotExist, e: -## return None - - -class NISBackend(object): - - def authenticate(self, username=None, password=None): - try: - n = nisauth(username, password) - temp_pass = User.objects.make_random_password(100) - nis_user = dict(username=username, - ) - - user_session_obj = dict( - email = username + "@mcs.anl.gov", - first_name = None, - last_name = None, - uid = n.uid - ) - user, created = User.objects.get_or_create(username=username) - - return user - - except NISAUTHError: - e = sys.exc_info()[1] - return None - - - def get_user(self, user_id): - try: - return User.objects.get(pk=user_id) - except User.DoesNotExist: - e = sys.exc_info()[1] - return None diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/__init__.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/__init__.py +++ /dev/null diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/admin.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/admin.py deleted file mode 100644 index 70a2233cc..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/admin.py +++ /dev/null @@ -1,15 +0,0 @@ -from django.contrib import admin - -from models import Host, Interface, IP, MX, Name, CName, Nameserver, ZoneAddress, Zone, Log, ZoneLog - -admin.site.register(Host) -admin.site.register(Interface) -admin.site.register(IP) -admin.site.register(MX) -admin.site.register(Name) -admin.site.register(CName) -admin.site.register(Nameserver) -admin.site.register(ZoneAddress) -admin.site.register(Zone) -admin.site.register(Log) -admin.site.register(ZoneLog) diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/models.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/models.py deleted file mode 100644 index 3f08a09a0..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/models.py +++ /dev/null @@ -1,210 +0,0 @@ -from django.db import models - -# Create your models here. -class Host(models.Model): - NETGROUP_CHOICES = ( - ('none', 'none'),('cave', 'cave'),('ccst', 'ccst'),('mcs', 'mcs'), - ('mmlab', 'mmlab'),('sp', 'sp'),('red', 'red'),('virtual', 'virtual'), - ('win', 'win'),('xterm', 'xterm'),('lcrc', 'lcrc'),('anlext', 'anlext'), - ('teragrid', 'teragrid') - ) - STATUS_CHOICES = ( - ('active','active'),('dormant','dormant') - ) - SUPPORT_CHOICES = ( - ('green','green'),('yellow','yellow'),('red','red') - ) - CLASS_CHOICES = ( - ('scientific','scientific'), - ('operations','operations'),('guest','guest'), - ('confidential','confidential'),('public','public') - ) - WHATAMI_CHOICES = ( - ('aix-3', 'aix-3'), ('aix-4', 'aix-4'), - ('aix-5', 'aix-5'), ('baytech', 'baytech'), - ('decserver', 'decserver'), ('dialup', 'dialup'), - ('dos', 'dos'), ('freebsd', 'freebsd'), - ('hpux', 'hpux'), ('irix-5', 'irix-5'), - ('irix-6', 'irix-6'), ('linux', 'linux'), - ('linux-2', 'linux-2'), ('linux-rh73', 'linux-rh73'), - ('linux-rh8', 'linux-rh8'), ('linux-sles8', 'linux-sles8'), - ('linux-sles8-64', 'linux-sles8-64'), ('linux-sles8-ia32', 'linux-sles8-ia32'), - ('linux-sles8-ia64', 'linux-sles8-ia64'), ('mac', 'mac'), - ('network', 'network'), ('next', 'next'), - ('none', 'none'), ('osf', 'osf'), ('printer', 'printer'), - ('robot', 'robot'), ('solaris-2', 'solaris-2'), - ('sun4', 'sun4'), ('unknown', 'unknown'), ('virtual', 'virtual'), - ('win31', 'win31'), ('win95', 'win95'), - ('winNTs', 'winNTs'), ('winNTw', 'winNTw'), - ('win2k', 'win2k'), ('winXP', 'winXP'), ('xterm', 'xterm') - ) - hostname = models.CharField(max_length=64) - whatami = models.CharField(max_length=16) - netgroup = models.CharField(max_length=32, choices=NETGROUP_CHOICES) - security_class = models.CharField('class', max_length=16) - support = models.CharField(max_length=8, choices=SUPPORT_CHOICES) - csi = models.CharField(max_length=32, blank=True) - printq = models.CharField(max_length=32, blank=True) - outbound_smtp = models.BooleanField() - primary_user = models.EmailField() - administrator = models.EmailField(blank=True) - location = models.CharField(max_length=16) - comments = models.TextField(blank=True) - expiration_date = models.DateField(null=True, blank=True) - last = models.DateField(auto_now=True, auto_now_add=True) - status = models.CharField(max_length=7, choices=STATUS_CHOICES) - dirty = models.BooleanField() - - class Admin: - list_display = ('hostname', 'last') - search_fields = ['hostname'] - - def __str__(self): - return self.hostname - - def get_logs(self): - """ - Get host's log. - """ - return Log.objects.filter(hostname=self.hostname) - -class Interface(models.Model): - TYPE_CHOICES = ( - ('eth', 'ethernet'), ('wl', 'wireless'), ('virtual', 'virtual'), ('myr', 'myr'), - ('mgmt', 'mgmt'), ('tape', 'tape'), ('fe', 'fe'), ('ge', 'ge'), - ) - # FIXME: The new admin interface has change a lot. - #host = models.ForeignKey(Host, edit_inline=models.TABULAR, num_in_admin=2) - host = models.ForeignKey(Host) - # FIXME: The new admin interface has change a lot. - #mac_addr = models.CharField(max_length=32, core=True) - mac_addr = models.CharField(max_length=32) - hdwr_type = models.CharField('type', max_length=16, choices=TYPE_CHOICES, blank=True) - # FIXME: The new admin interface has change a lot. - # radio_admin=True, blank=True) - dhcp = models.BooleanField() - - def __str__(self): - return self.mac_addr - - class Admin: - list_display = ('mac_addr', 'host') - search_fields = ['mac_addr'] - -class IP(models.Model): - interface = models.ForeignKey(Interface) - # FIXME: The new admin interface has change a lot. - # edit_inline=models.TABULAR, num_in_admin=1) - #ip_addr = models.IPAddressField(core=True) - ip_addr = models.IPAddressField() - - def __str__(self): - return self.ip_addr - - class Admin: - pass - - class Meta: - ordering = ('ip_addr', ) - -class MX(models.Model): - priority = models.IntegerField(blank=True) - # FIXME: The new admin interface has change a lot. - #mx = models.CharField(max_length=64, blank=True, core=True) - mx = models.CharField(max_length=64, blank=True) - - def __str__(self): - return (" ".join([str(self.priority), self.mx])) - - class Admin: - pass - -class Name(models.Model): - DNS_CHOICES = ( - ('global','global'),('internal','ANL internal'), - ('private','private') - ) - # FIXME: The new admin interface has change a lot. - #ip = models.ForeignKey(IP, edit_inline=models.TABULAR, num_in_admin=1) - ip = models.ForeignKey(IP) - # FIXME: The new admin interface has change a lot. - #name = models.CharField(max_length=64, core=True) - name = models.CharField(max_length=64) - dns_view = models.CharField(max_length=16, choices=DNS_CHOICES) - only = models.BooleanField(blank=True) - mxs = models.ManyToManyField(MX) - - def __str__(self): - return self.name - - class Admin: - pass - -class CName(models.Model): - # FIXME: The new admin interface has change a lot. - #name = models.ForeignKey(Name, edit_inline=models.TABULAR, num_in_admin=1) - name = models.ForeignKey(Name) - # FIXME: The new admin interface has change a lot. - #cname = models.CharField(max_length=64, core=True) - cname = models.CharField(max_length=64) - - def __str__(self): - return self.cname - - class Admin: - pass - -class Nameserver(models.Model): - name = models.CharField(max_length=64, blank=True) - - def __str__(self): - return self.name - - class Admin: - pass - -class ZoneAddress(models.Model): - ip_addr = models.IPAddressField(blank=True) - - def __str__(self): - return self.ip_addr - - class Admin: - pass - -class Zone(models.Model): - zone = models.CharField(max_length=64) - serial = models.IntegerField() - admin = models.CharField(max_length=64) - primary_master = models.CharField(max_length=64) - expire = models.IntegerField() - retry = models.IntegerField() - refresh = models.IntegerField() - ttl = models.IntegerField() - nameservers = models.ManyToManyField(Nameserver, blank=True) - mxs = models.ManyToManyField(MX, blank=True) - addresses = models.ManyToManyField(ZoneAddress, blank=True) - aux = models.TextField(blank=True) - - def __str__(self): - return self.zone - - class Admin: - pass - -class Log(models.Model): - # FIXME: Proposal hostname = models.ForeignKey(Host) - hostname = models.CharField(max_length=64) - date = models.DateTimeField(auto_now=True, auto_now_add=True) - log = models.TextField() - - def __str__(self): - return self.hostname - -class ZoneLog(models.Model): - zone = models.CharField(max_length=64) - date = models.DateTimeField(auto_now=True, auto_now_add=True) - log = models.TextField() - - def __str__(self): - return self.zone diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/sql/zone.sql b/src/lib/Bcfg2/Server/Hostbase/hostbase/sql/zone.sql deleted file mode 100644 index b78187ab2..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/sql/zone.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO hostbase_zone (zone, serial, admin, primary_master, expire, retry, refresh, ttl, aux) -VALUES ('.rev', 0, '', '', 1209600, 1800, 7200, 7200, '');
\ No newline at end of file diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py deleted file mode 100644 index a03d2c919..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -from Bcfg2.Reporting.Compat.django_urls import * -from django.contrib.auth.decorators import login_required -from django.core.urlresolvers import reverse -from django.views.generic.create_update import create_object, update_object, delete_object -from django.views.generic.list_detail import object_detail, object_list - -from models import Host, Zone, Log - -host_detail_dict = { - 'queryset':Host.objects.all(), - 'template_name':'host.html', - 'template_object_name':'host', -} - -host_delete_dict = { - 'model':Host, - 'post_delete_redirect':'/', -} - -host_log_detail_dict = host_detail_dict.copy() -host_log_detail_dict['template_name'] = 'logviewer.html' - -host_dns_detail_dict = host_detail_dict.copy() -host_dns_detail_dict['template_name'] = 'dns.html' - -zone_new_dict = { - 'model':Zone, - 'template_name':'zonenew.html', - 'post_save_redirect':'../%(id)s', -} - -zones_list_dict = { - 'queryset':Zone.objects.all(), - 'template_name':'zones.html', - 'template_object_name':'zone', -} - -zone_detail_dict = { - 'queryset':Zone.objects.all(), - 'template_name':'zoneview.html', - 'template_object_name':'zone', -} - -urlpatterns = patterns('', - (r'^(?P<object_id>\d+)/$', object_detail, host_detail_dict, 'host_detail'), - (r'^zones/new/$', login_required(create_object), zone_new_dict, 'zone_new'), - (r'^zones/(?P<object_id>\d+)/edit', login_required(update_object), zone_new_dict, 'zone_edit'), - (r'^zones/$', object_list, zones_list_dict, 'zone_list'), - (r'^zones/(?P<object_id>\d+)/$', object_detail, zone_detail_dict, 'zone_detail'), - (r'^zones/(?P<object_id>\d+)/$', object_detail, zone_detail_dict, 'zone_detail'), - (r'^\d+/logs/(?P<object_id>\d+)/', object_detail, { 'queryset':Log.objects.all() }, 'log_detail'), - (r'^(?P<object_id>\d+)/logs/', object_detail, host_log_detail_dict, 'host_log_list'), - (r'^(?P<object_id>\d+)/dns', object_detail, host_dns_detail_dict, 'host_dns_list'), - (r'^(?P<object_id>\d+)/remove', login_required(delete_object), host_delete_dict, 'host_delete'), -) - -urlpatterns += patterns('Bcfg2.Server.Hostbase.hostbase.views', - (r'^$', 'search'), - (r'^(?P<host_id>\d+)/edit', 'edit'), - (r'^(?P<host_id>\d+)/(?P<item>\D+)/(?P<item_id>\d+)/confirm', 'confirm'), - (r'^(?P<host_id>\d+)/(?P<item>\D+)/(?P<item_id>\d+)/(?P<name_id>\d+)/confirm', 'confirm'), - (r'^(?P<host_id>\d+)/dns/edit', 'dnsedit'), - (r'^new', 'new'), - (r'^(?P<host_id>\d+)/copy', 'copy'), -# (r'^hostinfo', 'hostinfo'), - (r'^zones/(?P<zone_id>\d+)/(?P<item>\D+)/(?P<item_id>\d+)/confirm', 'confirm'), -) diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/views.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/views.py deleted file mode 100644 index 57ef5eff8..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/views.py +++ /dev/null @@ -1,970 +0,0 @@ -"""Views.py -Contains all the views associated with the hostbase app -Also has does form validation -""" -from django.http import HttpResponse, HttpResponseRedirect - -from django.contrib.auth.decorators import login_required -from django.contrib.auth import logout -from django.template import RequestContext -from Bcfg2.Server.Hostbase.hostbase.models import * -from datetime import date -from django.db import connection -from django.shortcuts import render_to_response -from django import forms -from Bcfg2.Server.Hostbase import settings, regex -import re, copy - -attribs = ['hostname', 'whatami', 'netgroup', 'security_class', 'support', - 'csi', 'printq', 'primary_user', 'administrator', 'location', - 'status', 'comments'] - -zoneattribs = ['zone', 'admin', 'primary_master', 'expire', 'retry', - 'refresh', 'ttl', 'aux'] - -dispatch = {'mac_addr':'i.mac_addr LIKE \'%%%%%s%%%%\'', - 'ip_addr':'p.ip_addr LIKE \'%%%%%s%%%%\'', - 'name':'n.name LIKE \'%%%%%s%%%%\'', -## 'hostname':'n.name LIKE \'%%%%%s%%%%\'', -## 'cname':'n.name LIKE \'%%%%%s%%%%\'', - 'mx':'m.mx LIKE \'%%%%%s%%%%\'', - 'dns_view':'n.dns_view = \'%s\'', - 'hdwr_type':'i.hdwr_type = \'%s\'', - 'dhcp':'i.dhcp = \'%s\''} - -def search(request): - """Search for hosts in the database - If more than one field is entered, logical AND is used - """ - if 'sub' in request.GET: - querystring = """SELECT DISTINCT h.hostname, h.id, h.status - FROM (((((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - INNER JOIN hostbase_name_mxs x ON n.id = x.name_id) - INNER JOIN hostbase_mx m ON m.id = x.mx_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE """ - - _and = False - for field in request.POST: - if request.POST[field] and field == 'hostname': - if _and: - querystring += ' AND ' - querystring += 'n.name LIKE \'%%%%%s%%%%\' or c.cname LIKE \'%%%%%s%%%%\'' % (request.POST[field], request.POST[field]) - _and = True - elif request.POST[field] and field in dispatch: - if _and: - querystring += ' AND ' - querystring += dispatch[field] % request.POST[field] - _and = True - elif request.POST[field]: - if _and: - querystring += ' AND ' - querystring += "h.%s LIKE \'%%%%%s%%%%\'" % (field, request.POST[field]) - _and = True - - if not _and: - cursor = connection.cursor() - cursor.execute("""SELECT hostname, id, status - FROM hostbase_host ORDER BY hostname""") - results = cursor.fetchall() - else: - querystring += " ORDER BY h.hostname" - cursor = connection.cursor() - cursor.execute(querystring) - results = cursor.fetchall() - - return render_to_response('results.html', - {'hosts': results, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - else: - return render_to_response('search.html', - {'TYPE_CHOICES': Interface.TYPE_CHOICES, - 'DNS_CHOICES': Name.DNS_CHOICES, - 'yesno': [(1, 'yes'), (0, 'no')], - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - - -def gethostdata(host_id, dnsdata=False): - """Grabs the necessary data about a host - Replaces a lot of repeated code""" - hostdata = {} - hostdata['ips'] = {} - hostdata['names'] = {} - hostdata['cnames'] = {} - hostdata['mxs'] = {} - hostdata['host'] = Host.objects.get(id=host_id) - hostdata['interfaces'] = hostdata['host'].interface_set.all() - for interface in hostdata['interfaces']: - hostdata['ips'][interface.id] = interface.ip_set.all() - if dnsdata: - for ip in hostdata['ips'][interface.id]: - hostdata['names'][ip.id] = ip.name_set.all() - for name in hostdata['names'][ip.id]: - hostdata['cnames'][name.id] = name.cname_set.all() - hostdata['mxs'][name.id] = name.mxs.all() - return hostdata - -def fill(template, hostdata, dnsdata=False): - """Fills a generic template - Replaces a lot of repeated code""" - if dnsdata: - template.names = hostdata['names'] - template.cnames = hostdata['cnames'] - template.mxs = hostdata['mxs'] - template.host = hostdata['host'] - template.interfaces = hostdata['interfaces'] - template.ips = hostdata['ips'] - return template - -def edit(request, host_id): - """edit general host information""" - manipulator = Host.ChangeManipulator(host_id) - changename = False - if request.method == 'POST': - host = Host.objects.get(id=host_id) - before = host.__dict__.copy() - if request.POST['hostname'] != host.hostname: - oldhostname = host.hostname.split(".")[0] - changename = True - interfaces = host.interface_set.all() - old_interfaces = [interface.__dict__.copy() for interface in interfaces] - - new_data = request.POST.copy() - - errors = manipulator.get_validation_errors(new_data) - if not errors: - - # somehow keep track of multiple interface change manipulators - # as well as multiple ip chnage manipulators??? (add manipulators???) - # change to many-to-many?????? - - # dynamically look up mx records? - text = '' - - for attrib in attribs: - if host.__dict__[attrib] != request.POST[attrib]: - text = do_log(text, attrib, host.__dict__[attrib], request.POST[attrib]) - host.__dict__[attrib] = request.POST[attrib] - - if 'expiration_date' in request.POST: - ymd = request.POST['expiration_date'].split("-") - if date(int(ymd[0]), int(ymd[1]), int(ymd[2])) != host.__dict__['expiration_date']: - text = do_log(text, 'expiration_date', host.__dict__['expiration_date'], - request.POST['expiration_date']) - host.__dict__['expiration_date'] = date(int(ymd[0]), int(ymd[1]), int(ymd[2])) - - for inter in interfaces: - changetype = False - ips = IP.objects.filter(interface=inter.id) - if inter.mac_addr != request.POST['mac_addr%d' % inter.id]: - text = do_log(text, 'mac_addr', inter.mac_addr, request.POST['mac_addr%d' % inter.id]) - inter.mac_addr = request.POST['mac_addr%d' % inter.id].lower().replace('-',':') - if inter.hdwr_type != request.POST['hdwr_type%d' % inter.id]: - oldtype = inter.hdwr_type - text = do_log(text, 'hdwr_type', oldtype, request.POST['hdwr_type%d' % inter.id]) - inter.hdwr_type = request.POST['hdwr_type%d' % inter.id] - changetype = True - if (('dhcp%d' % inter.id) in request.POST and not inter.dhcp or - not ('dhcp%d' % inter.id) in request.POST and inter.dhcp): - text = do_log(text, 'dhcp', inter.dhcp, int(not inter.dhcp)) - inter.dhcp = not inter.dhcp - for ip in ips: - names = ip.name_set.all() - if not ip.ip_addr == request.POST['ip_addr%d' % ip.id]: - oldip = ip.ip_addr - oldsubnet = oldip.split(".")[2] - ip.ip_addr = request.POST['ip_addr%d' % ip.id] - ip.save() - text = do_log(text, 'ip_addr', oldip, ip.ip_addr) - for name in names: - if name.name.split(".")[0].endswith('-%s' % oldsubnet): - name.name = name.name.replace('-%s' % oldsubnet, '-%s' % ip.ip_addr.split(".")[2]) - name.save() - if changetype: - for name in names: - if name.name.split(".")[0].endswith('-%s' % oldtype): - name.name = name.name.replace('-%s' % oldtype, '-%s' % inter.hdwr_type) - name.save() - if changename: - for name in names: - if name.name.startswith(oldhostname): - name.name = name.name.replace(oldhostname, host.hostname.split(".")[0]) - name.save() - if request.POST['%dip_addr' % inter.id]: - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_ip = IP(interface=inter, ip_addr=request.POST['%dip_addr' % inter.id]) - new_ip.save() - text = do_log(text, '*new*', 'ip_addr', new_ip.ip_addr) - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - inter.save() - if request.POST['mac_addr_new']: - new_inter = Interface(host=host, - mac_addr=request.POST['mac_addr_new'].lower().replace('-',':'), - hdwr_type=request.POST['hdwr_type_new'], - dhcp=request.POST['dhcp_new']) - text = do_log(text, '*new*', 'mac_addr', new_inter.mac_addr) - new_inter.save() - if request.POST['mac_addr_new'] and request.POST['ip_addr_new']: - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) - new_ip.save() - text = do_log(text, '*new*', 'ip_addr', new_ip.ip_addr) - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['ip_addr_new'] and not request.POST['mac_addr_new']: - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_inter = Interface(host=host, mac_addr="", - hdwr_type=request.POST['hdwr_type_new'], - dhcp=False) - new_inter.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) - new_ip.save() - text = do_log(text, '*new*', 'ip_addr', new_ip.ip_addr) - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if text: - log = Log(hostname=host.hostname, log=text) - log.save() - host.save() - return HttpResponseRedirect('/hostbase/%s/' % host.id) - else: - return render_to_response('errors.html', - {'failures': errors, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - else: - host = Host.objects.get(id=host_id) - interfaces = [] - for interface in host.interface_set.all(): - interfaces.append([interface, interface.ip_set.all()]) - return render_to_response('edit.html', - {'host': host, - 'interfaces': interfaces, - 'TYPE_CHOICES': Interface.TYPE_CHOICES, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - -def confirm(request, item, item_id, host_id=None, name_id=None, zone_id=None): - """Asks if the user is sure he/she wants to remove an item""" - if 'sub' in request.GET: - if item == 'interface': - for ip in Interface.objects.get(id=item_id).ip_set.all(): - for name in ip.name_set.all(): - name.cname_set.all().delete() - ip.name_set.all().delete() - Interface.objects.get(id=item_id).ip_set.all().delete() - Interface.objects.get(id=item_id).delete() - elif item=='ip': - for name in IP.objects.get(id=item_id).name_set.all(): - name.cname_set.all().delete() - IP.objects.get(id=item_id).name_set.all().delete() - IP.objects.get(id=item_id).delete() - elif item=='cname': - CName.objects.get(id=item_id).delete() - elif item=='mx': - mx = MX.objects.get(id=item_id) - Name.objects.get(id=name_id).mxs.remove(mx) - elif item=='name': - Name.objects.get(id=item_id).cname_set.all().delete() - Name.objects.get(id=item_id).delete() - elif item=='nameserver': - nameserver = Nameserver.objects.get(id=item_id) - Zone.objects.get(id=zone_id).nameservers.remove(nameserver) - elif item=='zonemx': - mx = MX.objects.get(id=item_id) - Zone.objects.get(id=zone_id).mxs.remove(mx) - elif item=='address': - address = ZoneAddress.objects.get(id=item_id) - Zone.objects.get(id=zone_id).addresses.remove(address) - if item == 'cname' or item == 'mx' or item == 'name': - return HttpResponseRedirect('/hostbase/%s/dns/edit' % host_id) - elif item == 'nameserver' or item == 'zonemx' or item == 'address': - return HttpResponseRedirect('/hostbase/zones/%s/edit' % zone_id) - else: - return HttpResponseRedirect('/hostbase/%s/edit' % host_id) - else: - interface = None - ips = [] - names = [] - cnames = [] - mxs = [] - zonemx = None - nameserver = None - address = None - if item == 'interface': - interface = Interface.objects.get(id=item_id) - ips = interface.ip_set.all() - for ip in ips: - for name in ip.name_set.all(): - names.append((ip.id, name)) - for cname in name.cname_set.all(): - cnames.append((name.id, cname)) - for mx in name.mxs.all(): - mxs.append((name.id, mx)) - elif item=='ip': - ips = [IP.objects.get(id=item_id)] - for name in ips[0].name_set.all(): - names.append((ips[0].id, name)) - for cname in name.cname_set.all(): - cnames.append((name.id, cname)) - for mx in name.mxs.all(): - mxs.append((name.id, mx)) - elif item=='name': - names = [Name.objects.get(id=item_id)] - for cname in names[0].cname_set.all(): - cnames.append((names[0].id, cname)) - for mx in names[0].mxs.all(): - mxs.append((names[0].id, mx)) - elif item=='cname': - cnames = [CName.objects.get(id=item_id)] - elif item=='mx': - mxs = [MX.objects.get(id=item_id)] - elif item=='zonemx': - zonemx = MX.objects.get(id=item_id) - elif item=='nameserver': - nameserver = Nameserver.objects.get(id=item_id) - elif item=='address': - address = ZoneAddress.objects.get(id=item_id) - return render_to_response('confirm.html', - {'interface': interface, - 'ips': ips, - 'names': names, - 'cnames': cnames, - 'id': item_id, - 'type': item, - 'host_id': host_id, - 'mxs': mxs, - 'zonemx': zonemx, - 'nameserver': nameserver, - 'address': address, - 'zone_id': zone_id, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - -def dnsedit(request, host_id): - """Edits specific DNS information - Data is validated before committed to the database""" - text = '' - if 'sub' in request.GET: - hostdata = gethostdata(host_id, True) - for ip in hostdata['names']: - ipaddr = IP.objects.get(id=ip) - ipaddrstr = ipaddr.__str__() - for name in hostdata['cnames']: - for cname in hostdata['cnames'][name]: - if regex.host.match(request.POST['cname%d' % cname.id]): - text = do_log(text, 'cname', cname.cname, request.POST['cname%d' % cname.id]) - cname.cname = request.POST['cname%d' % cname.id] - cname.save() - for name in hostdata['mxs']: - for mx in hostdata['mxs'][name]: - if (mx.priority != request.POST['priority%d' % mx.id] and mx.mx != request.POST['mx%d' % mx.id]): - text = do_log(text, 'mx', ' '.join([str(mx.priority), str(mx.mx)]), - ' '.join([request.POST['priority%d' % mx.id], request.POST['mx%d' % mx.id]])) - nameobject = Name.objects.get(id=name) - nameobject.mxs.remove(mx) - newmx, created = MX.objects.get_or_create(priority=request.POST['priority%d' % mx.id], mx=request.POST['mx%d' % mx.id]) - if created: - newmx.save() - nameobject.mxs.add(newmx) - nameobject.save() - for name in hostdata['names'][ip]: - name.name = request.POST['name%d' % name.id] - name.dns_view = request.POST['dns_view%d' % name.id] - if (request.POST['%dcname' % name.id] and - regex.host.match(request.POST['%dcname' % name.id])): - cname = CName(name=name, - cname=request.POST['%dcname' % name.id]) - text = do_log(text, '*new*', 'cname', cname.cname) - cname.save() - if (request.POST['%dpriority' % name.id] and - request.POST['%dmx' % name.id]): - mx, created = MX.objects.get_or_create(priority=request.POST['%dpriority' % name.id], - mx=request.POST['%dmx' % name.id]) - if created: - mx.save() - text = do_log(text, '*new*', 'mx', - ' '.join([request.POST['%dpriority' % name.id], - request.POST['%dmx' % name.id]])) - name.mxs.add(mx) - name.save() - if request.POST['%sname' % ipaddrstr]: - name = Name(ip=ipaddr, - dns_view=request.POST['%sdns_view' % ipaddrstr], - name=request.POST['%sname' % ipaddrstr], only=False) - text = do_log(text, '*new*', 'name', name.name) - name.save() - if (request.POST['%scname' % ipaddrstr] and - regex.host.match(request.POST['%scname' % ipaddrstr])): - cname = CName(name=name, - cname=request.POST['%scname' % ipaddrstr]) - text = do_log(text, '*new*', 'cname', cname.cname) - cname.save() - if (request.POST['%smx' % ipaddrstr] and - request.POST['%spriority' % ipaddrstr]): - mx, created = MX.objects.get_or_create(priority=request.POST['%spriority' % ipaddrstr], - mx=request.POST['%smx' % ipaddrstr]) - if created: - mx.save() - text = do_log(text, '*new*', 'mx', - ' '.join([request.POST['%spriority' % ipaddrstr], request.POST['%smx' % ipaddrstr]])) - name.mxs.add(mx) - if text: - log = Log(hostname=hostdata['host'].hostname, log=text) - log.save() - return HttpResponseRedirect('/hostbase/%s/dns' % host_id) - else: - host = Host.objects.get(id=host_id) - ips = [] - info = [] - cnames = [] - mxs = [] - interfaces = host.interface_set.all() - for interface in host.interface_set.all(): - ips.extend(interface.ip_set.all()) - for ip in ips: - info.append([ip, ip.name_set.all()]) - for name in ip.name_set.all(): - cnames.extend(name.cname_set.all()) - mxs.append((name.id, name.mxs.all())) - return render_to_response('dnsedit.html', - {'host': host, - 'info': info, - 'cnames': cnames, - 'mxs': mxs, - 'request': request, - 'interfaces': interfaces, - 'DNS_CHOICES': Name.DNS_CHOICES, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - -def new(request): - """Function for creating a new host in hostbase - Data is validated before committed to the database""" - if 'sub' in request.GET: - try: - Host.objects.get(hostname=request.POST['hostname'].lower()) - return render_to_response('errors.html', - {'failures': ['%s already exists in hostbase' % request.POST['hostname']], - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - except: - pass - if not validate(request, True): - if not request.POST['ip_addr_new'] and not request.POST['ip_addr_new2']: - return render_to_response('errors.html', - {'failures': ['ip_addr: You must enter an ip address'], - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - host = Host() - # this is the stuff that validate() should take care of - # examine the check boxes for any changes - host.outbound_smtp = 'outbound_smtp' in request.POST - for attrib in attribs: - if attrib in request.POST: - host.__dict__[attrib] = request.POST[attrib].lower() - if 'comments' in request.POST: - host.comments = request.POST['comments'] - if 'expiration_date' in request.POST: -# ymd = request.POST['expiration_date'].split("-") -# host.__dict__['expiration_date'] = date(int(ymd[0]), int(ymd[1]), int(ymd[2])) - host.__dict__['expiration_date'] = date(2000, 1, 1) - host.status = 'active' - host.save() - else: - return render_to_response('errors.html', - {'failures': validate(request, True), - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - - if request.POST['mac_addr_new']: - new_inter = Interface(host=host, - mac_addr = request.POST['mac_addr_new'].lower().replace('-',':'), - hdwr_type = request.POST['hdwr_type_new'], - dhcp = 'dhcp_new' in request.POST) - new_inter.save() - if request.POST['mac_addr_new'] and request.POST['ip_addr_new']: - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) -# Change all this things. Use a "post_save" signal handler for model Host to create all sociate models -# and use a generi view. - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['ip_addr_new'] and not request.POST['mac_addr_new']: - new_inter = Interface(host=host, - mac_addr="", - hdwr_type=request.POST['hdwr_type_new'], - dhcp=False) - new_inter.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['mac_addr_new2']: - new_inter = Interface(host=host, - mac_addr = request.POST['mac_addr_new2'].lower().replace('-',':'), - hdwr_type = request.POST['hdwr_type_new2'], - dhcp = 'dhcp_new2' in request.POST) - new_inter.save() - if request.POST['mac_addr_new2'] and request.POST['ip_addr_new2']: - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new2']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['ip_addr_new2'] and not request.POST['mac_addr_new2']: - new_inter = Interface(host=host, - mac_addr="", - hdwr_type=request.POST['hdwr_type_new2'], - dhcp=False) - new_inter.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new2']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - host.save() - return HttpResponseRedirect('/hostbase/%s/' % host.id) - else: - return render_to_response('new.html', - {'TYPE_CHOICES': Interface.TYPE_CHOICES, - 'NETGROUP_CHOICES': Host.NETGROUP_CHOICES, - 'CLASS_CHOICES': Host.CLASS_CHOICES, - 'SUPPORT_CHOICES': Host.SUPPORT_CHOICES, - 'WHATAMI_CHOICES': Host.WHATAMI_CHOICES, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - -def copy(request, host_id): - """Function for creating a new host in hostbase - Data is validated before committed to the database""" - if 'sub' in request.GET: - try: - Host.objects.get(hostname=request.POST['hostname'].lower()) - return render_to_response('errors.html', - {'failures': ['%s already exists in hostbase' % request.POST['hostname']], - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - except: - pass - if not validate(request, True): - if not request.POST['ip_addr_new'] and not request.POST['ip_addr_new2']: - return render_to_response('errors.html', - {'failures': ['ip_addr: You must enter an ip address'], - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - host = Host() - # this is the stuff that validate() should take care of - # examine the check boxes for any changes - host.outbound_smtp = 'outbound_smtp' in request.POST - for attrib in attribs: - if attrib in request.POST: - host.__dict__[attrib] = request.POST[attrib].lower() - if 'comments' in request.POST: - host.comments = request.POST['comments'] - if 'expiration_date' in request.POST: -# ymd = request.POST['expiration_date'].split("-") -# host.__dict__['expiration_date'] = date(int(ymd[0]), int(ymd[1]), int(ymd[2])) - host.__dict__['expiration_date'] = date(2000, 1, 1) - host.status = 'active' - host.save() - else: - return render_to_response('errors.html', - {'failures': validate(request, True), - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - - if request.POST['mac_addr_new']: - new_inter = Interface(host=host, - mac_addr = request.POST['mac_addr_new'].lower().replace('-',':'), - hdwr_type = request.POST['hdwr_type_new'], - dhcp = 'dhcp_new' in request.POST) - new_inter.save() - if request.POST['mac_addr_new'] and request.POST['ip_addr_new']: - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['ip_addr_new'] and not request.POST['mac_addr_new']: - new_inter = Interface(host=host, - mac_addr="", - hdwr_type=request.POST['hdwr_type_new'], - dhcp=False) - new_inter.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['mac_addr_new2']: - new_inter = Interface(host=host, - mac_addr = request.POST['mac_addr_new2'].lower().replace('-',':'), - hdwr_type = request.POST['hdwr_type_new2'], - dhcp = 'dhcp_new2' in request.POST) - new_inter.save() - if request.POST['mac_addr_new2'] and request.POST['ip_addr_new2']: - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new2']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['ip_addr_new2'] and not request.POST['mac_addr_new2']: - new_inter = Interface(host=host, - mac_addr="", - hdwr_type=request.POST['hdwr_type_new2'], - dhcp=False) - new_inter.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new2']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - host.save() - return HttpResponseRedirect('/hostbase/%s/' % host.id) - else: - host = Host.objects.get(id=host_id) - return render_to_response('copy.html', - {'host': host, - 'TYPE_CHOICES': Interface.TYPE_CHOICES, - 'NETGROUP_CHOICES': Host.NETGROUP_CHOICES, - 'CLASS_CHOICES': Host.CLASS_CHOICES, - 'SUPPORT_CHOICES': Host.SUPPORT_CHOICES, - 'WHATAMI_CHOICES': Host.WHATAMI_CHOICES, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - -# FIXME: delete all this things in a signal handler "pre_delete" -#def remove(request, host_id): -# host = Host.objects.get(id=host_id) -# if 'sub' in request: -# for interface in host.interface_set.all(): -# for ip in interface.ip_set.all(): -# for name in ip.name_set.all(): -# name.cname_set.all().delete() -# ip.name_set.all().delete() -# interface.ip_set.all().delete() -# interface.delete() -# host.delete() - -def validate(request, new=False, host_id=None): - """Function for checking form data""" - failures = [] - if (request.POST['expiration_date'] - and regex.date.match(request.POST['expiration_date'])): - try: - (year, month, day) = request.POST['expiration_date'].split("-") - date(int(year), int(month), int(day)) - except (ValueError): - failures.append('expiration_date') - elif request.POST['expiration_date']: - failures.append('expiration_date') - - if not (request.POST['hostname'] - and regex.host.match(request.POST['hostname'])): - failures.append('hostname') - -## if not regex.printq.match(request.POST['printq']) and request.POST['printq']: -## failures.append('printq') - -## if not regex.user.match(request.POST['primary_user']): -## failures.append('primary_user') - -## if (not regex.user.match(request.POST['administrator']) -## and request.POST['administrator']): -## failures.append('administrator') - -## if not (request.POST['location'] -## and regex.location.match(request.POST['location'])): -## failures.append('location') - - if new: - if (not regex.macaddr.match(request.POST['mac_addr_new']) - and request.POST['mac_addr_new']): - failures.append('mac_addr (#1)') - if ((request.POST['mac_addr_new'] or request.POST['ip_addr_new']) and - not 'hdwr_type_new' in request.REQUEST): - failures.append('hdwr_type (#1)') - if ((request.POST['mac_addr_new2'] or request.POST['ip_addr_new2']) and - not 'hdwr_type_new2' in request.REQUEST): - failures.append('hdwr_type (#2)') - - if (not regex.macaddr.match(request.POST['mac_addr_new2']) - and request.POST['mac_addr_new2']): - failures.append('mac_addr (#2)') - - if (not regex.ipaddr.match(request.POST['ip_addr_new']) - and request.POST['ip_addr_new']): - failures.append('ip_addr (#1)') - if (not regex. ipaddr.match(request.POST['ip_addr_new2']) - and request.POST['ip_addr_new2']): - failures.append('ip_addr (#2)') - - [failures.append('ip_addr (#1)') for number in - request.POST['ip_addr_new'].split(".") - if number.isdigit() and int(number) > 255 - and 'ip_addr (#1)' not in failures] - [failures.append('ip_addr (#2)') for number in - request.POST['ip_addr_new2'].split(".") - if number.isdigit() and int(number) > 255 - and 'ip_addr (#2)' not in failures] - - elif host_id: - interfaces = Interface.objects.filter(host=host_id) - for interface in interfaces: - if (not regex.macaddr.match(request.POST['mac_addr%d' % interface.id]) - and request.POST['mac_addr%d' % interface.id]): - failures.append('mac_addr (%s)' % request.POST['mac_addr%d' % interface.id]) - for ip in interface.ip_set.all(): - if not regex.ipaddr.match(request.POST['ip_addr%d' % ip.id]): - failures.append('ip_addr (%s)' % request.POST['ip_addr%d' % ip.id]) - [failures.append('ip_addr (%s)' % request.POST['ip_addr%d' % ip.id]) - for number in request.POST['ip_addr%d' % ip.id].split(".") - if (number.isdigit() and int(number) > 255 and - 'ip_addr (%s)' % request.POST['ip_addr%d' % ip.id] not in failures)] - if (request.POST['%dip_addr' % interface.id] - and not regex.ipaddr.match(request.POST['%dip_addr' % interface.id])): - failures.append('ip_addr (%s)' % request.POST['%dip_addr' % interface.id]) - if (request.POST['mac_addr_new'] - and not regex.macaddr.match(request.POST['mac_addr_new'])): - failures.append('mac_addr (%s)' % request.POST['mac_addr_new']) - if (request.POST['ip_addr_new'] - and not regex.ipaddr.match(request.POST['ip_addr_new'])): - failures.append('ip_addr (%s)' % request.POST['ip_addr_new']) - - if not failures: - return 0 - return failures - -def do_log(text, attribute, previous, new): - if previous != new: - text += "%-20s%-20s -> %s\n" % (attribute, previous, new) - return text - -## login required stuff -## uncomment the views below that you would like to restrict access to - -## uncomment the lines below this point to restrict access to pages that modify the database -## anonymous users can still view data in Hostbase - -edit = login_required(edit) -confirm = login_required(confirm) -dnsedit = login_required(dnsedit) -new = login_required(new) -copy = login_required(copy) -#remove = login_required(remove) -#zoneedit = login_required(zoneedit) -#zonenew = login_required(zonenew) - -## uncomment the lines below this point to restrict access to all of hostbase - -## search = login_required(search) -## look = login_required(look) -## dns = login_required(dns) -## zones = login_required(zones) -## zoneview = login_required(zoneview) - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/base.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/base.html deleted file mode 100644 index 1d7c5565b..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/base.html +++ /dev/null @@ -1,34 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> -<head> - <title>{% block title %}BCFG2 - Hostbase{% endblock %}</title> - <link rel="stylesheet" type="text/css" href="{{ MEDIA_URL }}/boxypastel.css" /> - <link rel="stylesheet" type="text/css" href="{{ MEDIA_URL }}/base.css" /> - <!--<script type="text/javascript" src="http://hostbase.mcs.anl.gov/site_media/main.js"> --> - {% block extra_header_info %}{% endblock %} -</head> - -<body> - <div id="header"> - <div id="branding"> - <h1>BCFG2</h1> - </div> - <div id="user-tools">...Change is Coming...</div> - </div> - <div id="sidebar"> - {% block sidebar %} - <ul class="sidebar"> - </ul> - {% endblock %} - </div> - - <div id="content-main"> - <div id="container"> - {% block pagebanner %}{% endblock %} - {% block content %}{% endblock %} - - </div> - </div> -</body> -</html> diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/confirm.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/confirm.html deleted file mode 100644 index ca8b0cc07..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/confirm.html +++ /dev/null @@ -1,117 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Confirm Removal</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -<form name="input" action="confirm.html?sub=true" method="post"> -Are you sure you want to remove these items? - -{% if interface %} -<ul> -<li> interface: {{ interface.mac_addr }} </li> -{% endif %} - - -{% if ips %} -<ul> -{% for ip in ips %} -<li> ip: {{ ip.ip_addr }} </li> -<ul> -{% for name in names %} -{% ifequal name.0 ip.id %} -<li> name: {{ name.1.name }} </li> -<ul> -{% endifequal %} -{% for cname in cnames %} -{% ifequal cname.0 name.1.id %} -<li> cname: {{ cname.1.name }} </li> -{% endifequal %} -{% endfor %} -</ul> -<ul> -{% for mx in mxs %} -{% ifequal mx.0 name.1.id %} -<li> mx: {{ mx.1.priority }} {{ mx.1.mx }} </li> -{% endifequal %} -{% endfor %} -</ul> -{% endfor %} -</ul> -{% endfor %} -</ul> -{% endif %} - -{% if names and not ips %} -<ul> -{% for name in names %} -<li> name: {{ name.name }} </li> -<ul> -{% for cname in cnames %} -{% ifequal cname.0 name.id %} -<li> cname: {{ cname.1.cname }} </li> -{% endifequal %} -{% endfor %} -</ul> -<ul> -{% for mx in mxs %} -{% ifequal mx.0 name.id %} -<li> mx: {{ mx.1.priority }} {{ mx.1.mx }} </li> -{% endifequal %} -{% endfor %} -</ul> -{% endfor %} -</ul> -{% endif %} - -{% if cnames and not names %} -<ul> -{% for cname in cnames %} -<li> cname: {{ cname.cname }} </li> -{% endfor %} -</ul> -{% endif %} - -{% if mxs and not names %} -<ul> -{% for mx in mxs %} -<li> mx: {{ mx.priority }} {{ mx.mx }} </li> -{% endfor %} -</ul> -{% endif %} - -{% if interface %} -</ul> -{% endif %} - -{% if zone_id %} -<ul> -{% ifequal type 'zonemx' %} -<li> mx: {{ zonemx.priority }} {{ zonemx.mx }} </li> -{% endifequal %} - -{% ifequal type 'nameserver' %} -<li> nameserver: {{ nameserver.name }} </li> -{% endifequal %} - -{% ifequal type 'address' %} -<li> address: {{ address.ip_addr }} </li> -{% endifequal %} -</ul> -{% endif %} - -<input type="submit" value="confirm"> -<input type="reset" value="cancel" onclick="history.back()"> -</form> - -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/copy.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/copy.html deleted file mode 100644 index 400ef58f2..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/copy.html +++ /dev/null @@ -1,122 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>new host information</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -<a href="/hostbase/" class="sidebar">search hostbase</a> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -<form name="hostdata" action="?sub=true" method="post"> -<input type="hidden" name="host"> -<table border="0" width="100%"> - <colgroup> - <col width="150"> - <col width="*"> - <tr> <td> <b>hostname</b></td> - <td> <input name="hostname" type="text" value="{{ host.hostname }}" ></td></tr> - <tr> <td> <b>whatami</b></td> - <td> - <select name="whatami"> - {% for choice in WHATAMI_CHOICES %} - {% ifequal host.whatami choice.0 %} - <option value="{{ choice.0 }}" selected="selected" >{{ choice.1 }} - {% else %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endifequal %} - {% endfor %} - </select> - </td></tr> - <tr> <td> <b>netgroup</b></td> - <td> - <select name="netgroup"> - {% for choice in NETGROUP_CHOICES %} - {% ifequal host.netgroup choice.0 %} - <option value="{{ choice.0 }}" selected="selected" >{{ choice.1 }} - {% else %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endifequal %} - {% endfor %} - </select> - </td></tr> - <tr> <td> <b>class</b></td> - <td> - <select name="security_class"> - {% for choice in CLASS_CHOICES %} - {% ifequal host.security_class choice.0 %} - <option value="{{ choice.0 }}" selected="selected" >{{ choice.1 }} - {% else %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endifequal %} - {% endfor %} - </select></td></tr> - <tr> <td> <b>support</b></td> - <td> - <select name="support"> - {% for choice in SUPPORT_CHOICES %} - {% ifequal host.support choice.0 %} - <option value="{{ choice.0 }}" selected="selected" >{{ choice.1 }} - {% else %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endifequal %} - {% endfor %} - </select></td></tr> - <tr> <td> <b>csi</b></td> - <td> <input name="csi" type="text" value="{{ host.csi }}" ></td></tr> - <tr> <td> <b>printq</b></td> - <td> <input name="printq" type="text" value="{{ host.printq }}" ></td></tr> - <tr> <td> <b>outbound_smtp</b></td> - <td> - {% if host.outbound_smtp %} - <input type="checkbox" name="outbound_smtp" checked="checked" ></td></tr> - {% else %} - <input type="checkbox" name="outbound_smtp" ></td></tr> - {% endif %} - <tr> <td> <b>primary_user</b></td> - <td> <input name="primary_user" type="text" size="32" value="{{ host.primary_user }}"> (email address)</td></tr> - <tr> <td> <b>administrator</b></td> - <td> <input name="administrator" type="text" size="32" value="{{ host.administrator }}"> (email address)</td></tr> - <tr> <td> <b>location</b></td> - <td> <input name="location" type="text" value="{{ host.location }}"></td></tr> - <tr> <td> <b>expiration_date</b></td> - <td> <input name="expiration_date" type="text" size="10" value="{{ host.expiration_date }}">YYYY-MM-DD</td></tr> - <tr> <td><br><b>Interface</b></td><td><br> - {% for choice in TYPE_CHOICES %} - <input type="radio" name="hdwr_type_new" value="{{ choice.0 }}" >{{ choice.1 }} - {% endfor %} - </td></tr> - <tr> <td> <b>dhcp</b></td> - <td> - <input type="checkbox" name="dhcp_new"></td></tr> - <tr> <td> <b>mac_addr</b></td> - <td> <input name="mac_addr_new" type="text"></td></tr> - <tr> <td> <b>ip_addr</b></td> - <td> <input name="ip_addr_new" type="text"></td></tr> - <tr> <td><br><b>Interface</b></td><td><br> - {% for choice in TYPE_CHOICES %} - <input type="radio" name="hdwr_type_new2" value="{{ choice.0 }}" >{{ choice.1 }} - {% endfor %} - </td></tr> - <tr> <td> <b>dhcp</b></td> - <td> - <input type="checkbox" name="dhcp_new2"></td></tr> - <tr> <td> <b>mac_addr</b></td> - <td> <input name="mac_addr_new2" type="text"></td></tr> - <tr> <td> <b>ip_addr</b></td> - <td> <input name="ip_addr_new2" type="text"></td></tr> - <tr> <td> <b>comments</b></td> - <td> <textarea rows="10" cols="50" name="comments"></textarea></td></tr> -</table> -<br> -<p><input type="submit" value="Submit"> -</form> - -{% endblock %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dns.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dns.html deleted file mode 100644 index da179e5a1..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dns.html +++ /dev/null @@ -1,40 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>dns info for {{ host.hostname }}</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -<ul class="sidebar"> - <li><a href="/hostbase/{{ host.id }}/" class="sidebar">host info</a></li> - <li><a href="/hostbase/{{ host.id }}/edit/" class="sidebar">edit host info</a></li> - <li><a href="edit/" class="sidebar">edit dns info</a></li> - <li><a href="/hostbase/{{ host.id }}/logs/" class="sidebar">change logs</a></li> -</ul> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -{% for interface in host.interface_set.all %} - {% for ip in interface.ip_set.all %} - <ul><li> <b>ip_addr:</b> {{ ip.ip_addr }}</li> - {% for name in ip.name_set.all %} - <ul> <li><b>name:</b> {{ name.name }}</li> <ul> - {% for cname in name.cname_set.all %} - <li> <b>cname:</b> {{ cname.cname }}</li> - {% endfor %} - {% for mx in name.mxs.all %} - <li> <b>mx:</b> {{ mx.priority }} {{ mx.mx }}</li> - {% endfor %} - </ul></ul> - {% endfor %} - </ul> - {% endfor %} -{% endfor %} -{% endblock %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dnsedit.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dnsedit.html deleted file mode 100644 index b1b71ab67..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dnsedit.html +++ /dev/null @@ -1,98 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>dns info for {{ host.hostname }}</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -<ul class="sidebar"> - <li><a href="/hostbase/{{ host.id }}/" class="sidebar">host info</a></li> - <li><a href="/hostbase/{{ host.id }}/edit/" class="sidebar">edit host info</a></li> - <li><a href="/hostbase/{{ host.id }}/dns/" class="sidebar">see dns info</a></li> - <li><a href="/hostbase/{{ host.id }}/logs/" class="sidebar">change logs</a></li> -</ul> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -<form name="dns" action="?sub=true" method="post"> -<input type="hidden" name="host" value="{{ host.id }}"> -<table border="0" width="100%"> - <colgroup> - <col width="150"> - <col width="*"> - {% for interface in interfaces %} - <tr><td><br></td></tr> - <tr> <td> <b>interface type</b> </td> - <td> {{ interface.hdwr_type }} </td></tr> - <tr> <td> <b>mac_addr</b> </td> - <td> {{ interface.mac_addr }} </td></tr> - <tr><td><hr></td><td><hr></td></tr> - {% for ip in info %} - {% ifequal ip.0.interface interface %} - <tr> <td> <b>ip_addr</b></td> - <td>{{ ip.0.ip_addr }}</td></tr> - {% for name in ip.1 %} - <tr> <td><b>name(dns)</b></td> - <td> <input name="name{{ name.id }}" type="text" value="{{ name.name }}"> - <select name="dns_view{{ name.id }}"> - {% for choice in DNS_CHOICES %} - {% ifequal name.dns_view choice.0 %} - <option value="{{ choice.0 }}" selected="selected">{{ choice.1 }} - {% else %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endifequal %} - {% endfor %} - </select> - <a style="font-size:75%" href="/hostbase/{{ host.id }}/name/{{ name.id }}/confirm">remove</a></td></tr> - {% for cname in cnames %} - {% ifequal name cname.name %} - <tr> <td> <b>cname</b></td> - <td> <input name="cname{{ cname.id }}" type="text" value="{{ cname.cname }}"> - <a style="font-size:75%" href="/hostbase/{{ host.id }}/cname/{{ cname.id }}/confirm">remove</a></td></tr> - {% endifequal %} - {% endfor %} - <tr> <td> <b>cname</b></td> - <td> <input name="{{ name.id }}cname" type="text"></td></tr> - {% for mx in mxs %} - {% ifequal mx.0 name.id %} - {% for record in mx.1 %} - <tr> <td> <b>mx</b></td> - <td> <input name="priority{{ record.id }}" type="text" size="6" value="{{ record.priority }}"> - <input name="mx{{ record.id }}" type="text" value="{{ record.mx }}"> - <a style="font-size:75%" href="/hostbase/{{ host.id }}/mx/{{ record.id }}/{{ name.id }}/confirm">remove</a></td></tr> - {% endfor %} - {% endifequal %} - {% endfor %} - <tr> <td> <b>mx</b></td> - <td> <input name="{{ name.id }}priority" type="text" size="6"> - <input name="{{ name.id }}mx" type="text"></td></tr> - {% endfor %} - <tr> <td> <b>name</b></td> - <td> <input name="{{ ip.0.ip_addr }}name" type="text"> - <select name="{{ ip.0.ip_addr }}dns_view"> - {% for choice in DNS_CHOICES %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endfor %} - </select></td></tr> - <tr> <td> <b>cname</b></td> - <td> <input name="{{ ip.0.ip_addr }}cname" type="text"></td></tr> - <tr> <td> <b>mx</b></td> - <td> <input name="{{ ip.0.ip_addr }}priority" type="text" size="6"> - <input name="{{ ip.0.ip_addr }}mx" type="text"></td></tr> - <tr><td></td></tr> - <tr><td><hr></td><td><hr></td></tr> - {% endifequal %} - {% endfor %} - {% endfor %} - </table> - -<p><input type="submit" value="Submit"> -</form> - -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/edit.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/edit.html deleted file mode 100644 index 961c9d143..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/edit.html +++ /dev/null @@ -1,191 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>{{ host.hostname }}</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -<ul class="sidebar"> -<li><a href="/hostbase/{{ host.id }}/" class="sidebar">host info</a></li> -<li><a href="/hostbase/{{ host.id }}/dns/" class="sidebar">detailed dns info</a></li> -<li><a href="/hostbase/{{ host.id }}/dns/edit/" class="sidebar">edit dns info</a></li> -<li><a href="/hostbase/{{ host.id }}/logs/" class="sidebar">change logs</a></li> -</ul> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -<script language="JavaScript" type="text/Javascript"> -function toggleAddr(interface_id){ - if(document.getElementById){ - var style = document.getElementById('ipaddr'+interface_id).style; - style.display = style.display? "":"block"; - } -} -function toggleInter(){ - if(document.getElementById){ - var style = document.getElementById('interface').style; - style.display = style.display? "":"block"; - } -} -</script> - -<style type=text/css> -{% for interface in interfaces %} -div#ipaddr{{ interface.0.id }}{ - display: none; -} -{% endfor %} -div#interface{ - display: none; -} -</style> - -<form name="hostdata" action="" method="post"> -<fieldset class="module aligned ()"> -<input type="hidden" name="host" value="{{ host.id }}"> - <label for="id_hostname">hostname:</label> - <input name="hostname" value="{{ host.hostname }}"><br> - <label for="id_whatami">whatami:</label> - <select name="whatami"> - {% for choice in host.WHATAMI_CHOICES %} - {% ifequal host.whatami choice.0 %} - <option value="{{ choice.0 }}" selected="selected">{{ choice.1 }} - {% else %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endifequal %} - {% endfor %} - </select><br> - <label for="id_netgroup">netgroup:</label> - <select name="netgroup"> - {% for choice in host.NETGROUP_CHOICES %} - {% ifequal host.netgroup choice.0 %} - <option value="{{ choice.0 }}" selected="selected">{{ choice.1 }} - {% else %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endifequal %} - {% endfor %} - </select><br> - <label for="id_security_class">class:</label> - <select name="security_class"> - {% for choice in host.CLASS_CHOICES %} - {% ifequal host.security_class choice.0 %} - <option value="{{ choice.0 }}" selected="selected">{{ choice.1 }} - {% else %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endifequal %} - {% endfor %} - </select><br> - <label for="id_support">support:</label> - <select name="support"> - {% for choice in host.SUPPORT_CHOICES %} - {% ifequal host.support choice.0 %} - <option value="{{ choice.0 }}" selected="selected">{{ choice.1 }} - {% else %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endifequal %} - {% endfor %} - </select><br> - <label for="id_csi">csi:</label> - <input name="csi" type="text" value="{{ host.csi }}"><br> - <label for="id_printq">printq:</label> - <input name="printq" type="text" value="{{ host.printq }}"><br> - <label for="id_outbound_smtp">outbound_smtp:</label> - {% if host.outbound_smtp %} - <input type="checkbox" checked="checked" name="outbound_smtp"> - {% else %} - <input type="checkbox" name="outbound_smtp"> - {% endif %}<br> - <label for="id_primary_user">primary_user:</label> - <input name="primary_user" type="text" size="32" value="{{ host.primary_user }}"><br> - <label for="id_administrator">administrator:</label> - <input name="administrator" type="text" size="32" value="{{ host.administrator }}"><br> - <label for="id_location">location:</label> - <input name="location" type="text" value="{{ host.location }}"><br> - <label for="id_expiration_date">expiration_date:</label> - <input name="expiration_date" type="text" value="{{ host.expiration_date }}"> YYYY-MM-DD<br> - {% for interface in interfaces %} - <label for="id_interface">Interface:</label> - <select name="hdwr_type{{ interface.0.id }}"> - {% for choice in interface.0.TYPE_CHOICES %} - {% ifequal interface.0.hdwr_type choice.0 %} - <option value="{{ choice.0 }}" selected="selected">{{ choice.1 }} - {% else %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endifequal %} - {% endfor %} - </select><br> - <label for="id_dhcp">dhcp:</label> - {% if interface.0.dhcp %} - <input type="checkbox" checked="checked" name="dhcp{{ interface.0.id }}"> - {% else %} - <input type="checkbox" name="dhcp{{ interface.0.id }}"> - {% endif %}<br> - <label for="id_mac_addr">mac_addr:</label> - <input name="mac_addr{{ interface.0.id }}" type="text" value="{{ interface.0.mac_addr }}"> - <a style="font-size:75%" href="/hostbase/{{ host.id }}/interface/{{ interface.0.id }}/confirm">remove</a><br> - {% for ip in interface.1 %} - <label for="id_ip_addr">ip_addr:</label> - <input name="ip_addr{{ ip.id }}" type="text" value="{{ ip.ip_addr }}"> - <a style="font-size:75%" href="/hostbase/{{ host.id }}/ip/{{ ip.id }}/confirm">remove</a><br> - {% endfor %} - -<!-- Section for adding a new IP address to an existing interface --> -<!-- By default, section is hidden --> - <div id=ipaddr{{ interface.0.id }}> - <label for="id_ip_addr">ip_addr:</label> - <input name="{{ interface.0.id }}ip_addr" type="text"><br> - </div> - <a style="font-size:75%" href=# onclick="toggleAddr({{ interface.0.id }})">Add a New IP Address</a><br> - {% endfor %} -<!-- End section for new IP address --> - -<!-- Section for add an entirely new interface to a host --> -<!-- By default, section is hidden --> - <div id=interface> - <label for="id_interface">Interface:</label> - <select name="hdwr_type_new"> - {% for choice in TYPE_CHOICES %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endfor %} - </select><br> - <label for="id_dhcp">dhcp:</label> - {% if host.dhcp %} - <input type="checkbox" checked="checked" name="dhcp_new"> - {% else %} - <input type="checkbox" name="dhcp_new"> - {% endif %}<br> - <label for="id_mac_addr">mac_addr:</label> - <td> <input name="mac_addr_new" type="text"><br> - <label for="id_ip_addr">ip_addr:</label> - <td> <input name="ip_addr_new" type="text"><br> -</div> -<a style="font-size:75%" href=# onclick="toggleInter()">Add a New Interface</a><br> -<!-- End new interface section --> - - -<label for="id_comments">comments:</label> -<textarea rows="10" cols="50" name="comments">{{ host.comments }}</textarea><br> -<a style="font-size:75%" href="/hostbase/{{ host.id }}/dns/edit">edit detailed DNS information for this host</a> -<br> -this host is -<select name="status"> -{% for choice in host.STATUS_CHOICES %} -{% ifequal host.status choice.0 %} -<option value="{{ choice.0 }}" selected="selected">{{ choice.1 }} -{% else %} -<option value="{{ choice.0 }}">{{ choice.1 }} -{% endifequal %} -{% endfor %} -</select><br> -last update on {{ host.last }}<br> -<input type="submit" value="submit"> -<input type="reset" value="cancel" onclick="history.back()"> -</form> - -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/errors.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/errors.html deleted file mode 100644 index e5429b86c..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/errors.html +++ /dev/null @@ -1,31 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Search Results</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -{% if failures %} -There were errors in the following fields<br><br> -{% for failure in failures %} - -<font color="#FF0000">{{ failure }}</font><br> -{% comment %} -{{ failure.1|join:", " }} -{% endcomment %} - -{% endfor %} -{% endif %} -<br> -Press the back button on your browser and edit those field(s) - -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/host.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/host.html deleted file mode 100644 index d6b8873bc..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/host.html +++ /dev/null @@ -1,80 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>{{ host.hostname }}</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -<ul class="sidebar"> - <li><a href="dns/" class="sidebar">detailed dns info</a></li> - <li><a href="edit/" class="sidebar">edit host info</a></li> - <li><a href="dns/edit/" class="sidebar">edit dns info</a></li> - <li><a href="logs/" class="sidebar">change logs</a></li> -</ul> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -<table border="0" width="100%"> - <colgroup> - <col width="150"> - <col width="*"> - <tr> <td> <b>hostname</b></td> - <td> {{ host.hostname }}</td></tr> - <tr> <td> <b>whatami</b></td> - <td> {{ host.whatami }}</td></tr> - <tr> <td> <b>netgroup</b></td> - <td> {{ host.netgroup }}</td></tr> - <tr> <td> <b>class</b></td> - <td> {{ host.security_class }}</td></tr> - <tr> <td> <b>support</b></td> - <td> {{ host.support }}</td></tr> - <tr> <td> <b>csi</b></td> - <td> {{ host.csi }}</td></tr> - <tr> <td> <b>printq</b></td> - <td> {{ host.printq }}</td></tr> - <tr> <td> <b>outbound_smtp</b></td> - {% if host.outbound_smtp %} - <td> y </td></tr> - {% else %} - <td> n </td></tr> - {% endif %} - <tr> <td> <b>primary_user</b></td> - <td> {{ host.primary_user }}</td></tr> - <tr> <td> <b>administrator</b></td> - <td> {{ host.administrator }}</td></tr> - <tr> <td> <b>location</b></td> - <td> {{ host.location }}</td></tr> - <tr> <td> <b>expiration_date</b></td> - <td> {{ host.expiration_date }}</td></tr> - {% for interface in host.inserface_set.all %} - <tr> <td><br><b>Interface</b></td> - {% ifnotequal interface.0.hdwr_type 'no' %} - <td><br>{{ interface.0.hdwr_type }}</td></tr> - {% endifnotequal %} - {% if interface.0.dhcp %} - <tr> <td> <b>mac_addr</b></td> - <td> {{ interface.0.mac_addr }}</b></td></tr> - {% endif %} - {% for ip in interface.1 %} - <tr> <td> <b>ip_addr</b></td> - <td> {{ ip.ip_addr }}</td></tr> - {% endfor %} - {% endfor %} - <tr> <td valign="top"> <b>comments</b></td> - <td> - {{ host.comments|linebreaksbr }}<br> - </td></tr> - -</table> -<a style="font-size:75%" href="/hostbase/{{ host.id }}/dns/">see detailed DNS information for this host</a> -<br><br> -this host is {{ host.status }}<br> -last update on {{ host.last }}<br> - -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html deleted file mode 100644 index b5d794b50..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html +++ /dev/null @@ -1,89 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Are you sure you want to remove {{ object.hostname }}?</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -<ul class="sidebar"> - <li><a href="dns/" class="sidebar">detailed dns info</a></li> - <li><a href="edit/" class="sidebar">edit host info</a></li> - <li><a href="dns/edit/" class="sidebar">edit dns info</a></li> -</ul> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -<table border="0" width="100%"> - <colgroup> - <col width="150"> - <col width="*"> - <tr> <td> <b>hostname</b></td> - <td> {{ object.hostname }}</td></tr> - <tr> <td> <b>whatami</b></td> - <td> {{ object.whatami }}</td></tr> - <tr> <td> <b>netgroup</b></td> - <td> {{ object.netgroup }}</td></tr> - <tr> <td> <b>class</b></td> - <td> {{ object.security_class }}</td></tr> - <tr> <td> <b>support</b></td> - <td> {{ object.support }}</td></tr> - <tr> <td> <b>csi</b></td> - <td> {{ object.csi }}</td></tr> - <tr> <td> <b>printq</b></td> - <td> {{ object.printq }}</td></tr> - <tr> <td> <b>dhcp</b></td> - {% if host.dhcp %} - <td> y </td></tr> - {% else %} - <td> n </td></tr> - {% endif %} - <tr> <td> <b>outbound_smtp</b></td> - {% if host.outbound_smtp %} - <td> y </td></tr> - {% else %} - <td> n </td></tr> - {% endif %} - <tr> <td> <b>primary_user</b></td> - <td> {{ object.primary_user }}</td></tr> - <tr> <td> <b>administrator</b></td> - <td> {{ object.administrator }}</td></tr> - <tr> <td> <b>location</b></td> - <td> {{ object.location }}</td></tr> - <tr> <td> <b>expiration_date</b></td> - <td> {{ object.expiration_date }}</td></tr> - {% for interface in interfaces %} - <tr> <td><br><b>Interface</b></td> - {% ifnotequal interface.0.hdwr_type 'no' %} - <td><br>{{ interface.0.hdwr_type }}</td></tr> - {% endifnotequal %} - <tr> <td> <b>mac_addr</b></td> - <td> {{ interface.0.mac_addr }}</b></td></tr> - {% for ip in interface.1 %} - <tr> <td> <b>ip_addr</b></td> - <td> {{ ip.ip_addr }}</td></tr> - {% endfor %} - {% endfor %} - <tr> <td valign="top"> <b>comments</b></td> - <td> - {{ object.comments|linebreaksbr }}<br> - </td></tr> - -</table> -<a style="font-size:75%" href="/hostbase/{{ object.id }}/dns/">see detailed DNS information for this host</a> -<br><br> -this host is {{ object.status }}<br> -last update on {{ object.last }}<br> - -<form name="input" action="remove.html?sub=true" method="post"> -<input type="submit" value="remove"> -<input type="reset" value="cancel" onclick="history.back()"> -</form> - -{% endblock %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html deleted file mode 100644 index aa9679cbd..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Change Logs for {{ object.hostname }}</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -<ul> -<li><b>Hostname:</b>{{ object.hostname }}</li> -<li><b>Date:</b>{{ object.date }}</li> -<li><b>Log:</b>{{ object.log }}</li> -</ul> - -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/index.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/index.html deleted file mode 100644 index 92258b648..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/index.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends "base.html" %} -{% block pagebanner %} - <div class="header"> - <h2>Welcome to Hostbase!</h2> - <p>Hostbase is a web based management tools for Bcfg2 Hosts</p> - </div> - <br/> -{% endblock %} -{% block sidebar %} -<a href="/login/" class="sidebar">login to hostbase</a><br> -<a href="/hostbase/" class="sidebar">search for hosts</a><br> -<a href="hostbase/zones/" class="sidebar">zone file information</a> -{% endblock %} -{% block content %} -{% endblock %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/login.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/login.html deleted file mode 100644 index ec24a0fc0..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/login.html +++ /dev/null @@ -1,37 +0,0 @@ -{% extends "base.html" %} -{% block pagebanner %} - <div class="header"> - <h2>Login to Hostbase!</h2> - <p>You must login to manage hosts</p> - </div> - <br/> -{% endblock %} -{% block sidebar %} -<a href="/hostbase/" class="sidebar">search for hosts</a><br> -<a href="/hostbase/new" class="sidebar">add a new host</a><br> -<a href="hostbase/zones/" class="sidebar">zone file information</a> -{% endblock %} -{% block content %} - {% if form.has_errors %} - {{ form.username.errors|join:", " }} - <p>Login Failed.</p> - {% endif %} - {% if user.is_authenticated %} - <p>Welcome, {{ user.username }}. Thanks for logging in.</p> - {% else %} - <p>Welcome, user. Please log in.</p> - <form name="input" action="." method="post"> - <input name="username" type="text"> - <br /> - <input name="password" type="password"> - <br /> - <input type="submit" value="Login"> - {% if next %} - <input type="hidden" name="next" value="{{ next }}" /> - {% else %} - <input type="hidden" name="next" value="/hostbase/" /> - {% endif %} - - </form> - {% endif %} -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.html deleted file mode 100644 index 994f631a8..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.html +++ /dev/null @@ -1,13 +0,0 @@ -{% extends "base.html" %} -{% block pagebanner %} - <div class="header"> - <h2>You are logged out of Hostbase!</h2> - </div> - <br/> -{% endblock %} -{% block sidebar %} -<a href="/login/" class="sidebar">Login to Hostbase</a> -{% endblock %} -{% block content %} -{% endblock %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.tmpl b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.tmpl deleted file mode 100644 index e71e90e76..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.tmpl +++ /dev/null @@ -1,6 +0,0 @@ -<p> -{% if logged_in %} -<a href="/logout/" class="sidebar">logout</a> -{% else %} -<a href="/login/" class="sidebar">login</a> -{% endif %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logviewer.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logviewer.html deleted file mode 100644 index 806ccd63d..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logviewer.html +++ /dev/null @@ -1,27 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Change Logs for {{ hostname }}</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -{% if host.get_logs %} -<ul> -{% for log in host.get_logs %} -<li><a href="{{ log.id }}/">{{ log.date }}</li> -{% endfor %} -</ul> -{% else %} -There are no logs for this host<br> -{% endif %} - -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/navbar.tmpl b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/navbar.tmpl deleted file mode 100644 index 877d427d0..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/navbar.tmpl +++ /dev/null @@ -1,5 +0,0 @@ -<a href="/hostbase/" class="sidebar">host search</a><br> -<a href="/hostbase/new" class="sidebar">add a new host</a><br> -<a href="/hostbase/zones" class="sidebar">zone file information</a><br> -<a href="/hostbase/zones/new" class="sidebar">add a zone</a><br> - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/new.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/new.html deleted file mode 100644 index 2dcd6271f..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/new.html +++ /dev/null @@ -1,102 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>new host information</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -<a href="/hostbase/" class="sidebar">search hostbase</a> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -<form name="hostdata" action="?sub=true" method="post"> -<input type="hidden" name="host"> -<table border="0" width="100%"> - <colgroup> - <col width="150"> - <col width="*"> - <tr> <td> <b>hostname</b></td> - <td> <input name="hostname" type="text" ></td></tr> - <tr> <td> <b>whatami</b></td> - <td> - <select name="whatami"> - {% for choice in WHATAMI_CHOICES %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endfor %} - </select> - </td></tr> - <tr> <td> <b>netgroup</b></td> - <td> - <select name="netgroup"> - {% for choice in NETGROUP_CHOICES %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endfor %} - </select> - </td></tr> - <tr> <td> <b>class</b></td> - <td> - <select name="security_class"> - {% for choice in CLASS_CHOICES %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endfor %} - </select></td></tr> - <tr> <td> <b>support</b></td> - <td> - <select name="support"> - {% for choice in SUPPORT_CHOICES %} - <option value="{{ choice.0 }}">{{ choice.1 }} - {% endfor %} - </select></td></tr> - <tr> <td> <b>csi</b></td> - <td> <input name="csi" type="text" ></td></tr> - <tr> <td> <b>printq</b></td> - <td> <input name="printq" type="text" ></td></tr> - <tr> <td> <b>outbound_smtp</b></td> - <td> - <input type="checkbox" name="outbound_smtp"></td></tr> - <tr> <td> <b>primary_user</b></td> - <td> <input name="primary_user" type="text" size="32" > (email address)</td></tr> - <tr> <td> <b>administrator</b></td> - <td> <input name="administrator" type="text" size="32" > (email address)</td></tr> - <tr> <td> <b>location</b></td> - <td> <input name="location" type="text" ></td></tr> - <tr> <td> <b>expiration_date</b></td> - <td> <input name="expiration_date" type="text" size="10" >YYYY-MM-DD</td></tr> - <tr> <td><br><b>Interface</b></td><td><br> - {% for choice in TYPE_CHOICES %} - <input type="radio" name="hdwr_type_new" value="{{ choice.0 }}" >{{ choice.1 }} - {% endfor %} - </td></tr> - <tr> <td> <b>dhcp</b></td> - <td> - <input type="checkbox" name="dhcp_new"></td></tr> - <tr> <td> <b>mac_addr</b></td> - <td> <input name="mac_addr_new" type="text"></td></tr> - <tr> <td> <b>ip_addr</b></td> - <td> <input name="ip_addr_new" type="text"></td></tr> - <tr> <td><br><b>Interface</b></td><td><br> - {% for choice in TYPE_CHOICES %} - <input type="radio" name="hdwr_type_new2" value="{{ choice.0 }}" >{{ choice.1 }} - {% endfor %} - </td></tr> - <tr> <td> <b>dhcp</b></td> - <td> - <input type="checkbox" name="dhcp_new2"></td></tr> - <tr> <td> <b>mac_addr</b></td> - <td> <input name="mac_addr_new2" type="text"></td></tr> - <tr> <td> <b>ip_addr</b></td> - <td> <input name="ip_addr_new2" type="text"></td></tr> - <tr> <td> <b>comments</b></td> - <td> <textarea rows="10" cols="50" name="comments"></textarea></td></tr> -</table> -<br> -<p><input type="submit" value="Submit"> -</form> - -{% endblock %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/remove.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/remove.html deleted file mode 100644 index 4329200dd..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/remove.html +++ /dev/null @@ -1,89 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Are you sure you want to remove {{ host.hostname }}?</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -<ul class="sidebar"> - <li><a href="dns/" class="sidebar">detailed dns info</a></li> - <li><a href="edit/" class="sidebar">edit host info</a></li> - <li><a href="dns/edit/" class="sidebar">edit dns info</a></li> -</ul> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -<table border="0" width="100%"> - <colgroup> - <col width="150"> - <col width="*"> - <tr> <td> <b>hostname</b></td> - <td> {{ host.hostname }}</td></tr> - <tr> <td> <b>whatami</b></td> - <td> {{ host.whatami }}</td></tr> - <tr> <td> <b>netgroup</b></td> - <td> {{ host.netgroup }}</td></tr> - <tr> <td> <b>class</b></td> - <td> {{ host.security_class }}</td></tr> - <tr> <td> <b>support</b></td> - <td> {{ host.support }}</td></tr> - <tr> <td> <b>csi</b></td> - <td> {{ host.csi }}</td></tr> - <tr> <td> <b>printq</b></td> - <td> {{ host.printq }}</td></tr> - <tr> <td> <b>dhcp</b></td> - {% if host.dhcp %} - <td> y </td></tr> - {% else %} - <td> n </td></tr> - {% endif %} - <tr> <td> <b>outbound_smtp</b></td> - {% if host.outbound_smtp %} - <td> y </td></tr> - {% else %} - <td> n </td></tr> - {% endif %} - <tr> <td> <b>primary_user</b></td> - <td> {{ host.primary_user }}</td></tr> - <tr> <td> <b>administrator</b></td> - <td> {{ host.administrator }}</td></tr> - <tr> <td> <b>location</b></td> - <td> {{ host.location }}</td></tr> - <tr> <td> <b>expiration_date</b></td> - <td> {{ host.expiration_date }}</td></tr> - {% for interface in interfaces %} - <tr> <td><br><b>Interface</b></td> - {% ifnotequal interface.0.hdwr_type 'no' %} - <td><br>{{ interface.0.hdwr_type }}</td></tr> - {% endifnotequal %} - <tr> <td> <b>mac_addr</b></td> - <td> {{ interface.0.mac_addr }}</b></td></tr> - {% for ip in interface.1 %} - <tr> <td> <b>ip_addr</b></td> - <td> {{ ip.ip_addr }}</td></tr> - {% endfor %} - {% endfor %} - <tr> <td valign="top"> <b>comments</b></td> - <td> - {{ host.comments|linebreaksbr }}<br> - </td></tr> - -</table> -<a style="font-size:75%" href="/hostbase/{{ host.id }}/dns/">see detailed DNS information for this host</a> -<br><br> -this host is {{ host.status }}<br> -last update on {{ host.last }}<br> - -<form name="input" action="remove.html?sub=true" method="post"> -<input type="submit" value="remove"> -<input type="reset" value="cancel" onclick="history.back()"> -</form> - -{% endblock %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/results.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/results.html deleted file mode 100644 index 45b22058d..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/results.html +++ /dev/null @@ -1,45 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Search Results</h2> - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -{% if hosts %} -<table border="0" width="100%"> - <colgroup> - <col width="200"> - <col width="75"> - <col width="50"> - <col width="50"> - <col width="50"> - <col width="*"> - <tr> <td><b>hostname</b></td> - <td> <b>status</b> </td> - </tr> - {% for host in hosts %} - <tr> <td>{{ host.0 }}</td> - <td> {{ host.2 }} </td> - <td> <a href="{{ host.1 }}">view</a> </td> - <td> <a href="{{ host.1 }}/edit">edit</a> </td> - <td> <a href="{{ host.1 }}/copy">copy</a> </td> - <td> <a href="{{ host.1 }}/logs">logs</a> </td> -<!-- <td> <a href="{{ host.1 }}/remove">remove</a> </td> --> - </tr> - {% endfor %} -</table> -{% else %} -No hosts matched your query<br> -Click the back button on your browser to edit your search -{% endif %} - -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/search.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/search.html deleted file mode 100644 index 409d418fe..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/search.html +++ /dev/null @@ -1,57 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Welcome to Hostbase!</h2> - <p>search for hosts using one or more of the fields below - </div> - <br/> -{% endblock %} - -{% block sidebar %} -<a href="/hostbase/new" class="sidebar">add a new host</a><br> -<a href="/hostbase/zones" class="sidebar">zone file information</a><br> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} -{% comment %} - ...or go to <a href="hostinfo">this</a> - page to enter hostinfo-like queries<br><br> -{% endcomment %} - -<form name="input" action="?sub=true" method="post"> - <fieldset class="module aligned ()"> - <label for="hostname">hostname:</label><input name="hostname" type="text" ><br> - <label for="netgroup">netgroup:</label><input name="netgroup" type="text" ><br> - <label for="security_class">class:</label><input name="security_class" type="text" ><br> - <label for="support">support:</label><input name="support" type="text" ><br> - <label for="csi">csi:</label><input name="csi" type="text" ><br> - <label for="printq">printq:</label><input name="printq" type="text" ><br> - <label for="outbound_smtp">outbound_smtp:</label> - {% for choice in yesno %} - <input type="radio" name="outbound_smtp" value="{{ choice.0 }}" >{{ choice.1 }} - {% endfor %}<br> - <label for="primary_user">primary_user:</label><input name="primary_user" type="text" ><br> - <label for="administrator">administrator:</label><input name="administrator" type="text" ><br> - <label for="location">location:</label><input name="location" type="text" ><br> - <label for="expiration_date">expiration_date:</label><input name="expiration_date" type="text" ><br> - <br><label for="Interface">Interface:</label> - {% for choice in TYPE_CHOICES %} - <input type="radio" name="hdwr_type" value="{{ choice.0 }}" >{{ choice.1 }} - {% endfor %}<br> - <label for="dhcp">dhcp:</label> - {% for choice in yesno %} - <input type="radio" name="dhcp" value="{{ choice.0 }}" >{{ choice.1 }} - {% endfor %}<br> - <label for="mac_addr">mac_addr:</label><input name="mac_addr" type="text" ><br> - <label for="ip_addr">ip_addr:</label><input name="ip_addr" type="text" ><br> - <label for="dns_view">dns_viewer:</label> - {% for choice in DNS_CHOICES %} - <input type="radio" name="dns_view" value="{{ choice.0 }}" >{{ choice.1 }} - {% endfor %}<br> - <label for="mx">mx:</label><input name="mx" type="text" ><br> -<p> -<input type="submit" value="Search"> -</form> -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneedit.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneedit.html deleted file mode 100644 index ee355ee87..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneedit.html +++ /dev/null @@ -1,81 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Zones</h2> - <p>Edit information for {{ zone }} - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -<ul> -<li><a href="/hostbase/zones/{{ zone_id }}/" class="sidebar">view zone</a><br> -</li> -</ul> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -<script language="JavaScript" type="text/Javascript"> -function toggleField(fieldname){ - if(document.getElementById){ - var style = document.getElementById(fieldname).style; - style.display = style.display? "":"block"; - } -} -</script> - -<style type=text/css> -div#nameserver{ - display: none; -} -div#mx{ - display: none; -} -div#address{ - display: none; -} -</style> - -<form name="zonedata" action="" method="post"> - <fieldset class="module aligned ()"> -<label for="id_zone">zone:</label></td> <td>{{ form.zone }}<br> -<label for="id_admin">admin:</label></td> <td>{{ form.admin }}<br> -<label for="id_primary_master">primary_master:</label></td> <td>{{ form.primary_master }}<br> -<label for="id_expire">expire:</label></td> <td>{{ form.expire }}<br> -<label for="id_retry">retry:</label></td> <td>{{ form.retry }}<br> -<label for="id_refresh">refresh:</label></td> <td>{{ form.refresh }}<br> -<label for="id_ttl">ttl:</label></td> <td>{{ form.ttl }}<br> -{% for ns in nsforms %} -<label for="id_name">nameserver:</label></td> <td>{{ ns.name }}<br> -{% endfor %} -</table> -<div id=nameserver> - <label for="id_name">nameserver:</label></td> <td>{{ nsadd.name }}<br> - <label for="id_name">nameserver:</label></td> <td>{{ nsadd.name }}<br> -</div> -<a style="font-size:75%" href=# onclick="toggleField('nameserver')">Add NS records</a><br> -{% for mx in mxforms %} -<label for="id_mx">mx:</label></td> <td>{{ mx.priority }} {{ mx.mx }}<br> -{% endfor %} -<div id=mx> - <label for="id_mx">mx:</label></td> <td>{{ mxadd.priority }} {{ mxadd.mx }}<br> - <label for="id_mx">mx:</label></td> <td>{{ mxadd.priority }} {{ mxadd.mx }}<br> -</div> -<a style="font-size:75%" href=# onclick="toggleField('mx')">Add MX records</a><br> -{% for a in aforms %} -<label for="id_address">ip address:</label></td> <td>{{ a.ip_addr }}<br> -{% endfor %} -<div id=address> - <label for="id_address">ip address:</label></td> <td>{{ addadd.ip_addr }}<br> - <label for="id_address">ip address:</label></td> <td>{{ addadd.ip_addr }}<br> -</div> -<a style="font-size:75%" href=# onclick="toggleField('address')">Add A records</a><br> -<label for="id_aux">aux:</label></td> <td>{{ form.aux }}<br> -<p><input type="submit" value="Submit"> -</form> - -{% endblock %} diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zonenew.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zonenew.html deleted file mode 100644 index b59fa9e3c..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zonenew.html +++ /dev/null @@ -1,43 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Zones</h2> - <p>Enter information for a new zone to be generated by Hostbase - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} -<form name="zonedata" action="" method="post"> - <fieldset class="module aligned ()"> - {{ form.as_p}} -<!-- - <label for="id_zone">zone:</label>{{ form.zone }}<br> - <label for="id_admin">admin:</label>{{ form.admin }}<br> - <label for="id_primary_master">primary_master:</label>{{ form.primary_master }}<br> - <label for="id_expire">expire:</label>{{ form.expire }}<br> - <label for="id_retry">retry:</label>{{ form.retry }}<br> - <label for="id_refresh">refresh:</label>{{ form.refresh }}<br> - <label for="id_ttl">ttl:</label>{{ form.ttl }}<br> - <label for="id_name">nameserver:</label>{{ nsform.name }}<br> - <label for="id_name">nameserver:</label>{{ nsform.name }}<br> - <label for="id_name">nameserver:</label>{{ nsform.name }}<br> - <label for="id_name">nameserver:</label>{{ nsform.name }}<br> - <label for="id_mx">mx:</label>{{ mxform.priority }} {{ mxform.mx }}<br> - <label for="id_mx">mx:</label>{{ mxform.priority }} {{ mxform.mx }}<br> - <label for="id_mx">ip address:</label>{{ aform.ip_addr }}<br> - <label for="id_mx">ip address:</label>{{ aform.ip_addr }}<br> - <label for="id_aux">aux: -(information not generated from Hostbase)</label>{{ form.aux }}<br> ---!> - <p><input type="submit" value="Submit"> - </fieldset> -</form> -{% endblock %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zones.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zones.html deleted file mode 100644 index c773e7922..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zones.html +++ /dev/null @@ -1,37 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Zones</h2> - <p>Hostbase generates DNS zone files for the following zones. - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} -{% if zone_list %} -<table border="0" width="100%"> - <colgroup> - <col width="200"> - <col width="75"> - <col width="50"> - <col width="*"> - <tr> <td><b>zone</b></td> - </tr> - {% for zone in zone_list|dictsort:"zone" %} - <tr> <td> {{ zone.zone }}</td> - <td> <a href="{{ zone.id }}">view</a> </td> - <td> <a href="{{ zone.id }}/edit">edit</a> </td> - </tr> - {% endfor %} -</table> -{% else %} -There is no zone data currently in the database<br> -{% endif %} -{% endblock %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneview.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneview.html deleted file mode 100644 index fa12e3ec5..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneview.html +++ /dev/null @@ -1,71 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} - <div class="header"> - <h2>Zones</h2> - <p>Hostbase generates DNS zone files for the following zones. - </div> - <br/> -{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -<ul class="sidebar"> -<li><a href="/hostbase/zones/{{ zone.id }}/edit/" class="sidebar">edit zone</a><br> -</li> -</ul> -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} -<table border="0" width="100%"> - <colgroup> - <col width="200"> - <col width="*"> - <tr> <td> <b>zone</b></td> - <td> {{ zone.zone }}</td></tr> - <tr> <td> <b>serial</b></td> - <td> {{ zone.serial }}</td></tr> - <tr> <td> <b>admin</b></td> - <td> {{ zone.admin }}</td></tr> - <tr> <td> <b>primary_master</b></td> - <td> {{ zone.primary_master }}</td></tr> - <tr> <td> <b>expire</b></td> - <td> {{ zone.expire }}</td></tr> - <tr> <td> <b>retry</b></td> - <td> {{ zone.retry }}</td></tr> - <tr> <td> <b>refresh</b></td> - <td> {{ zone.refresh }}</td></tr> - <tr> <td> <b>ttl</b></td> - <td> {{ zone.ttl }}</td></tr> - - <tr><td valign="top"> <b>nameservers</b></td> - <td> - {% for nameserver in zone.nameservers.all %} - {{ nameserver.name }}<br> - {% endfor %} - </td></tr> - <tr><td valign="top"> <b>mxs</b></td> - <td> - {% for mx in zone.mxs.all %} - {{ mx.priority }} {{ mx.mx }}<br> - {% endfor %} - </td></tr> - {% if addresses %} - <tr><td valign="top"> <b>A records</b></td> - <td> - {% for address in sof.addresses.all %} - {{ address.ip_addr }}<br> - {% endfor %} - </td></tr> - {% endif %} - - <tr> <td valign="top"> <b>aux</b></td> - <td> - {{ zone.aux|linebreaksbr }} - </td></tr> - -</table> -<br><br> -{% endblock %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/ldapauth.py b/src/lib/Bcfg2/Server/Hostbase/ldapauth.py deleted file mode 100644 index fc2ca1bf1..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/ldapauth.py +++ /dev/null @@ -1,179 +0,0 @@ -""" -Checks with LDAP (ActiveDirectory) to see if the current user is an LDAP(AD) -user, and returns a subset of the user's profile that is needed by Argonne/CIS -to set user level privleges in Django -""" - -import os -import ldap - - -class LDAPAUTHError(Exception): - """LDAPAUTHError is raised when somehting goes boom.""" - pass - - -class ldapauth(object): - group_test = False - check_member_of = os.environ['LDAP_CHECK_MBR_OF_GRP'] - securitylevel = 0 - distinguishedName = None - sAMAccountName = None - telephoneNumber = None - title = None - memberOf = None - department = None # this will be a list - mail = None - extensionAttribute1 = None # badgenumber - badge_no = None - - def __init__(self, login, passwd): - """get username (if using ldap as auth the - apache env var REMOTE_USER should be used) - from username get user profile from AD/LDAP - """ - #p = self.user_profile(login,passwd) - d = self.user_dn(login) # success, distname - print(d[1]) - if d[0] == 'success': - pass - p = self.user_bind(d[1], passwd) - if p[0] == 'success': - #parse results - parsed = self.parse_results(p[2]) - print(self.department) - self.group_test = self.member_of() - securitylevel = self.security_level() - print("ACCESS LEVEL: " + str(securitylevel)) - else: - raise LDAPAUTHError(p[2]) - else: - raise LDAPAUTHError(p[2]) - - def user_profile(self, login, passwd=None): - """NOT USED RIGHT NOW""" - ldap_login = "CN=%s" % login - svc_acct = os.environ['LDAP_SVC_ACCT_NAME'] - svc_pass = os.environ['LDAP_SVC_ACCT_PASS'] - #svc_acct = 'CN=%s,DC=anl,DC=gov' % login - #svc_pass = passwd - - search_pth = os.environ['LDAP_SEARCH_PTH'] - - try: - conn = ldap.initialize(os.environ['LDAP_URI']) - conn.bind(svc_acct, svc_pass, ldap.AUTH_SIMPLE) - result_id = conn.search(search_pth, - ldap.SCOPE_SUBTREE, - ldap_login, - None) - result_type, result_data = conn.result(result_id, 0) - return ('success', 'User profile found', result_data,) - except ldap.LDAPError: - e = sys.exc_info()[1] - #connection failed - return ('error', 'LDAP connect failed', e,) - - def user_bind(self, distinguishedName, passwd): - """Binds to LDAP Server""" - search_pth = os.environ['LDAP_SEARCH_PTH'] - try: - conn = ldap.initialize(os.environ['LDAP_URI']) - conn.bind(distinguishedName, passwd, ldap.AUTH_SIMPLE) - cn = distinguishedName.split(",") - result_id = conn.search(search_pth, - ldap.SCOPE_SUBTREE, - cn[0], - None) - result_type, result_data = conn.result(result_id, 0) - return ('success', 'User profile found', result_data,) - except ldap.LDAPError: - e = sys.exc_info()[1] - #connection failed - return ('error', 'LDAP connect failed', e,) - - def user_dn(self, cn): - """Uses Service Account to get distinguishedName""" - ldap_login = "CN=%s" % cn - svc_acct = os.environ['LDAP_SVC_ACCT_NAME'] - svc_pass = os.environ['LDAP_SVC_ACCT_PASS'] - search_pth = os.environ['LDAP_SEARCH_PTH'] - - try: - conn = ldap.initialize(os.environ['LDAP_URI']) - conn.bind(svc_acct, svc_pass, ldap.AUTH_SIMPLE) - result_id = conn.search(search_pth, - ldap.SCOPE_SUBTREE, - ldap_login, - None) - result_type, result_data = conn.result(result_id, 0) - raw_obj = result_data[0][1] - distinguishedName = raw_obj['distinguishedName'] - return ('success', distinguishedName[0],) - except ldap.LDAPError: - e = sys.exc_info()[1] - #connection failed - return ('error', 'LDAP connect failed', e,) - - def parse_results(self, user_obj): - """Clean up the huge ugly object handed to us in the LDAP query""" - #user_obj is a list formatted like this: - #[('LDAP_DN',{user_dict},),] - try: - raw_obj = user_obj[0][1] - self.memberOf = raw_obj['memberOf'] - self.sAMAccountName = raw_obj['sAMAccountName'][0] - self.distinguishedName = raw_obj['distinguishedName'][0] - self.telephoneNumber = raw_obj['telephoneNumber'][0] - self.title = raw_obj['title'][0] - self.department = raw_obj['department'][0] - self.mail = raw_obj['mail'][0] - self.badge_no = raw_obj['extensionAttribute1'][0] - self.email = raw_obj['extensionAttribute2'][0] - display_name = raw_obj['displayName'][0].split(",") - self.name_f = raw_obj['givenName'][0] - self.name_l = display_name[0] - self.is_staff = False - self.is_superuser = False - - return - except KeyError: - e = sys.exc_info()[1] - raise LDAPAUTHError("Portions of the LDAP User profile not present") - - def member_of(self): - """See if this user is in our group that is allowed to login""" - m = [g for g in self.memberOf if g == self.check_member_of] - if len(m) == 1: - return True - else: - return False - - def security_level(self): - level = self.securitylevel - - user = os.environ['LDAP_GROUP_USER'] - m = [g for g in self.memberOf if g == user] - if len(m) == 1: - if level < 1: - level = 1 - - cspr = os.environ['LDAP_GROUP_SECURITY_LOW'] - m = [g for g in self.memberOf if g == cspr] - if len(m) == 1: - if level < 2: - level = 2 - - cspo = os.environ['LDAP_GROUP_SECURITY_HIGH'] - m = [g for g in self.memberOf if g == cspo] - if len(m) == 1: - if level < 3: - level = 3 - - admin = os.environ['LDAP_GROUP_ADMIN'] - m = [g for g in self.memberOf if g == admin] - if len(m) == 1: - if level < 4: - level = 4 - - return level diff --git a/src/lib/Bcfg2/Server/Hostbase/manage.py b/src/lib/Bcfg2/Server/Hostbase/manage.py deleted file mode 100755 index 5e78ea979..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/manage.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python -from django.core.management import execute_manager -try: - import settings # Assumed to be in the same directory. -except ImportError: - import sys - sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) - sys.exit(1) - -if __name__ == "__main__": - execute_manager(settings) diff --git a/src/lib/Bcfg2/Server/Hostbase/media/base.css b/src/lib/Bcfg2/Server/Hostbase/media/base.css deleted file mode 100644 index ddbf02165..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/media/base.css +++ /dev/null @@ -1,5 +0,0 @@ - -/* Import other styles */ -@import url('global.css'); -@import url('layout.css'); -@import url('boxypastel.css'); diff --git a/src/lib/Bcfg2/Server/Hostbase/media/boxypastel.css b/src/lib/Bcfg2/Server/Hostbase/media/boxypastel.css deleted file mode 100644 index 7ae0684ef..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/media/boxypastel.css +++ /dev/null @@ -1,179 +0,0 @@ -body { - background-color: #fff; - color: #000; - font: 12px 'Lucida Grande', Arial, Helvetica, sans-serif; - margin-left:0px; - margin-right:100px; -} -/* links */ -a:link { - color: #00f; - text-decoration: none; -} -a:visited { - color: #00a; - text-decoration: none; -} -a:hover { - color: #00a; - text-decoration: underline; -} -a:active { - color: #00a; - text-decoration: underline; -} -/* divs*/ -div.bad { - border: 1px solid #660000; - background: #FF6A6A; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.modified { - border: 1px solid #CC9900; - background: #FFEC8B; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.clean { - border: 1px solid #006600; - background: #9AFF9A; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.extra { - border: 1px solid #006600; - background: #6699CC; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.warning { - border: 1px - solid #CC3300; - background: #FF9933; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.all-warning { - border: 1px solid #DD5544; - background: #FFD9A2; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.down { - border: 1px - solid #999; - background-color: #DDD; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.items{ - display: none; -} -div.nodebox { - border: 1px solid #c7cfd5; - background: #f1f5f9; - margin: 20px 0; - padding: 8px 8px 16px 8px; - text-align: left; - position:relative; -} -div.header { - background-color: #DDD; - padding: 8px; - text-indent:50px; - position:relative; -} - -/*Spans*/ -.nodename { - font-style: italic; -} -.nodelisttitle { - font-size: 14px; -} - -h2{ - font-size: 16px; - color: #000; -} - -ul.plain { - list-style-type:none; - text-align: left; -} - -.notebox { - position: absolute; - top: 0px; - right: 0px; - padding: 1px; - text-indent:0px; - border: 1px solid #FFF; - background: #999; - color: #FFF; -} - -.configbox { - position: absolute; - bottom: 0px; - right: 0px; - padding: 1px; - text-indent:0px; - border: 1px solid #999; - background: #FFF; - color: #999; -} - -p.indented{ - text-indent: 50px -} - -/* - Sortable tables */ -table.sortable a.sortheader { - background-color:#dfd; - font-weight: bold; - text-decoration: none; - display: block; -} -table.sortable { - padding: 2px 4px 2px 4px; - border: 1px solid #000000; - border-spacing: 0px -} -td.sortable{ - padding: 2px 8px 2px 8px; -} - -th.sortable{ - background-color:#F3DD91; - border: 1px solid #FFFFFF; -} -tr.tablelist { - background-color:#EDF3FE; -} -tr.tablelist-alt{ - background-color:#FFFFFF; -} diff --git a/src/lib/Bcfg2/Server/Hostbase/media/global.css b/src/lib/Bcfg2/Server/Hostbase/media/global.css deleted file mode 100644 index 73451e1bc..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/media/global.css +++ /dev/null @@ -1,8 +0,0 @@ -body { - margin:0; - padding:0; - font-size:12px; - font-family:"Lucida Grande","Bitstream Vera Sans",Verdana,Arial,sans-serif; - color:#000; - background:#fff; - } diff --git a/src/lib/Bcfg2/Server/Hostbase/media/layout.css b/src/lib/Bcfg2/Server/Hostbase/media/layout.css deleted file mode 100644 index 9085cc220..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/media/layout.css +++ /dev/null @@ -1,62 +0,0 @@ -/* Page Structure */ -#container { position:absolute; top: 3em; margin-left:1em; margin-right:2em; padding:0; margin-top:1.5em; min-width: - 650px; } -#header { width:100%; } -#content-main { float:left; } - -/* HEADER */ -#header { -background:#000; -color:#ffc; -position:absolute; -} -#header a:link, #header a:visited { color:white; } -#header a:hover { text-decoration:underline; } -#branding h1 { padding:0 10px; font-size:18px; margin:8px 0; font-weight:normal; color:#f4f379; } -#branding h2 { padding:0 10px; font-size:14px; margin:-8px 0 8px 0; font-weight:normal; color:#ffc; } -#user-tools { position:absolute; top:0; right:0; padding:1.2em 10px; font-size:11px; text-align:right; } - -/*SIDEBAR*/ -#sidebar { - float:left; - position: relative; - width: auto; - height: 100%; - margin-top: 3em; - padding-right: 1.5em; - padding-left: 1.5em; - padding-top: 1em; - padding-bottom:3em; - background: #000; - color:ffc; -} - -a.sidebar:link {color: #fff;} -a.sidebar:active {color: #fff;} -a.sidebar:visited {color: #fff;} -a.sidebar:hover {color: #fff;} - -ul.sidebar { - color: #ffc; - text-decoration: none; - list-style-type: none; - text-indent: -1em; -} -ul.sidebar-level2 { - text-indent: -2em; - list-style-type: none; - font-size: 11px; -} - -/* ALIGNED FIELDSETS */ -.aligned label { display:block; padding:0 1em 3px 0; float:left; width:8em; } -.aligned label.inline { display:inline; float:none; } -.colMS .aligned .vLargeTextField, .colMS .aligned .vXMLLargeTextField { width:350px; } -form .aligned p, form .aligned ul { margin-left:7em; padding-left:30px; } -form .aligned table p { margin-left:0; padding-left:0; } -form .aligned p.help { padding-left:38px; } -.aligned .vCheckboxLabel { float:none !important; display:inline; padding-left:4px; } -.colM .aligned .vLargeTextField, colM .aligned .vXMLLargeTextField { width:610px; } -.checkbox-row p.help { margin-left:0; padding-left:0 !important; } - - diff --git a/src/lib/Bcfg2/Server/Hostbase/nisauth.py b/src/lib/Bcfg2/Server/Hostbase/nisauth.py deleted file mode 100644 index ae4c6c021..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/nisauth.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Checks with NIS to see if the current user is in the support group""" -import os -import crypt, nis -from Bcfg2.Server.Hostbase.settings import AUTHORIZED_GROUP - - -class NISAUTHError(Exception): - """NISAUTHError is raised when somehting goes boom.""" - pass - -class nisauth(object): - group_test = False -# check_member_of = os.environ['LDAP_CHECK_MBR_OF_GRP'] - samAcctName = None - distinguishedName = None - sAMAccountName = None - telephoneNumber = None - title = None - memberOf = None - department = None #this will be a list - mail = None - extensionAttribute1 = None #badgenumber - badge_no = None - uid = None - - def __init__(self,login,passwd=None): - """get user profile from NIS""" - try: - p = nis.match(login, 'passwd.byname').split(":") - except: - raise NISAUTHError('username') - # check user password using crypt and 2 character salt from passwd file - if p[1] == crypt.crypt(passwd, p[1][:2]): - # check to see if user is in valid support groups - # will have to include these groups in a settings file eventually - if not login in nis.match(AUTHORIZED_GROUP, 'group.byname').split(':')[-1].split(',') and p[3] != nis.match(AUTHORIZED_GROUP, 'group.byname').split(':')[2]: - raise NISAUTHError('group') - self.uid = p[2] - else: - raise NISAUTHError('password') diff --git a/src/lib/Bcfg2/Server/Hostbase/regex.py b/src/lib/Bcfg2/Server/Hostbase/regex.py deleted file mode 100644 index 41cc0f6f0..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/regex.py +++ /dev/null @@ -1,6 +0,0 @@ -import re - -date = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}$') -host = re.compile('^[a-z0-9-_]+(\.[a-z0-9-_]+)+$') -macaddr = re.compile('^[0-9abcdefABCDEF]{2}(:[0-9abcdefABCDEF]{2}){5}$|virtual') -ipaddr = re.compile('^[0-9]{1,3}(\.[0-9]{1,3}){3}$') diff --git a/src/lib/Bcfg2/Server/Hostbase/settings.py b/src/lib/Bcfg2/Server/Hostbase/settings.py deleted file mode 100644 index 7660e1bdc..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/settings.py +++ /dev/null @@ -1,143 +0,0 @@ -import os.path -# Compatibility import -from Bcfg2.Compat import ConfigParser - -PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) - -c = ConfigParser.ConfigParser() -#This needs to be configurable one day somehow -c.read(['./bcfg2.conf']) - -defaults = {'database_engine':'sqlite3', - 'database_name':'./dev.db', - 'database_user':'', - 'database_password':'', - 'database_host':'', - 'database_port':3306, - 'default_mx':'localhost', - 'priority':10, - 'authorized_group':'admins', - } - -if c.has_section('hostbase'): - options = dict(c.items('hostbase')) -else: - options = defaults - -# Django settings for Hostbase project. -DEBUG = True -TEMPLATE_DEBUG = DEBUG -ADMINS = ( - ('Root', 'root'), -) -MANAGERS = ADMINS - -# 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'. -DATABASE_ENGINE = options['database_engine'] -# Or path to database file if using sqlite3. -DATABASE_NAME = options['database_name'] -# Not used with sqlite3. -DATABASE_USER = options['database_user'] -# Not used with sqlite3. -DATABASE_PASSWORD = options['database_password'] -# Set to empty string for localhost. Not used with sqlite3. -DATABASE_HOST = options['database_host'] -# Set to empty string for default. Not used with sqlite3. -DATABASE_PORT = int(options['database_port']) -# Local time zone for this installation. All choices can be found here: -# http://docs.djangoproject.com/en/dev/ref/settings/#time-zone -try: - TIME_ZONE = c.get('statistics', 'time_zone') -except: - TIME_ZONE = None - -# enter the defauly MX record machines will get in Hostbase -# this setting may move elsewhere eventually -DEFAULT_MX = options['default_mx'] -PRIORITY = int(options['priority']) - -SESSION_EXPIRE_AT_BROWSER_CLOSE = True - -# Uncomment a backend below if you would like to use it for authentication -AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend', - 'Bcfg2.Server.Hostbase.backends.NISBackend', - #'Bcfg2.Server.Hostbase.backends.LDAPBacken', - ) -# enter an NIS group name you'd like to give access to edit hostbase records -AUTHORIZED_GROUP = options['authorized_group'] - -#create login url area: -import django.contrib.auth -django.contrib.auth.LOGIN_URL = '/login' -# Absolute path to the directory that holds media. -# Example: "/home/media/media.lawrence.com/" -MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media') -# Just for development -SERVE_MEDIA = DEBUG - -# Language code for this installation. All choices can be found here: -# http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes -# http://blogs.law.harvard.edu/tech/stories/storyReader$15 -LANGUAGE_CODE = 'en-us' -SITE_ID = 1 -# URL that handles the media served from MEDIA_ROOT. -# Example: "http://media.lawrence.com" -MEDIA_URL = '/site_media/' -# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a -# trailing slash. -# Examples: "http://foo.com/media/", "/media/". -ADMIN_MEDIA_PREFIX = '/media/' -# Make this unique, and don't share it with anybody. -SECRET_KEY = '*%=fv=yh9zur&gvt4&*d#84o(cy^-*$ox-v1e9%32pzf2*qu#s' -# List of callables that know how to import templates from various sources. -TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.load_template_source', - 'django.template.loaders.app_directories.load_template_source', -# 'django.template.loaders.eggs.load_template_source', -) - -TEMPLATE_CONTEXT_PROCESSORS = ( - "django.core.context_processors.auth", - "django.core.context_processors.debug", - "django.core.context_processors.i18n", - "django.core.context_processors.request", - "django.core.context_processors.media", -# Django development version. -# "django.core.context_processors.csrf", -) - - -MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.locale.LocaleMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.middleware.doc.XViewMiddleware', -) - -ROOT_URLCONF = 'Bcfg2.Server.Hostbase.urls' - -TEMPLATE_DIRS = ( - # Put strings here, like "/home/html/django_templates". - # Always use forward slashes, even on Windows. - '/usr/lib/python2.3/site-packages/Bcfg2/Server/Hostbase/hostbase/webtemplates', - '/usr/lib/python2.4/site-packages/Bcfg2/Server/Hostbase/hostbase/webtemplates', - '/usr/lib/python2.3/site-packages/Bcfg2/Server/Hostbase/templates', - '/usr/lib/python2.4/site-packages/Bcfg2/Server/Hostbase/templates', - '/usr/share/bcfg2/Hostbase/templates', - os.path.join(PROJECT_ROOT, 'templates'), - os.path.join(PROJECT_ROOT, 'hostbase/webtemplates'), -) - -INSTALLED_APPS = ( - 'django.contrib.admin', - 'django.contrib.admindocs', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.humanize', - 'Bcfg2.Server.Hostbase.hostbase', -) - -LOGIN_URL = '/login/' diff --git a/src/lib/Bcfg2/Server/Hostbase/templates/batchadd.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/batchadd.tmpl deleted file mode 100644 index 74ea3c047..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/templates/batchadd.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -#mx -> -#priority -> - -hostname -> -whatami -> -netgroup -> -security_class -> -support -> -csi -> -printq -> -dhcp -> -outbound_smtp -> -primary_user -> -administrator -> -location -> -expiration_date -> YYYY-MM-DD -comments -> - -mac_addr -> -hdwr_type -> -ip_addr -> -#ip_addr -> -cname -> -#cname -> - -#mac_addr -> -#hdwr_type -> -#ip_addr -> -#cname -> diff --git a/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.conf.head b/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.conf.head deleted file mode 100644 index a3d19547e..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.conf.head +++ /dev/null @@ -1,5 +0,0 @@ -# -# dhcpd.conf -# -# Configuration file for ISC dhcpd -# diff --git a/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.tmpl deleted file mode 100644 index 757b263cd..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -# -# This file is automatically generated. -# DO NOT EDIT IT BY HAND! -# -# This file contains {{ numips }} IP addresses -# Generated on: {% now "r" %} -# - -{% include "dhcpd.conf.head" %} - -# Hosts which require special configuration options can be listed in -# host statements. If no address is specified, the address will be -# allocated dynamically (if possible), but the host-specific information -# will still come from the host declaration. - -{% for host in hosts %}host {{ host.0 }} {hardware ethernet {{ host.1 }};fixed-address {{ host.2 }};} -{% endfor %} diff --git a/src/lib/Bcfg2/Server/Hostbase/templates/hosts.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/hosts.tmpl deleted file mode 100644 index 251cb5a79..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/templates/hosts.tmpl +++ /dev/null @@ -1,26 +0,0 @@ -############################################################################## -# MCS hosts file -# -# This file is generated automatically - DO NOT EDIT IT. -# -# Generated on: {% now "r" %} -# - -127.0.0.1 localhost.mcs.anl.gov localhost - -# This file lists hosts in these domains: -{% for domain in domain_data %}# {{ domain.0 }}: {{ domain.1 }} -{% endfor %} -# -# This file lists hosts on these networks: -# -# Network Hosts -# --------------------------------------------------------------------- -{% for octet in two_octets_data %}# {{ octet.0 }} {{octet.1 }} -{% endfor %} -# -{% for octet in three_octets_data %}# {{ octet.0 }} {{ octet.1 }} -{% endfor %} -# -# Total host interfaces (ip addresses) in this file: {{ num_ips }} - diff --git a/src/lib/Bcfg2/Server/Hostbase/templates/hostsappend.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/hostsappend.tmpl deleted file mode 100644 index 00e0d5d04..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/templates/hostsappend.tmpl +++ /dev/null @@ -1,5 +0,0 @@ -########################################################################## -# Hosts on subnet: {{ subnet.0 }} -# total hosts: {{ subnet.1 }} -{% for ip in ips %}{{ ip.0 }} {{ ip.1 }}{% if ip.4 and not ip.3 %} # {{ ip.5 }}{% else %}{% for name in ip.2 %} {{ name }}{% endfor %}{% for cname in ip.3 %} {{ cname }}{% endfor %} # {{ ip.5 }}{% endif %} -{% endfor %} diff --git a/src/lib/Bcfg2/Server/Hostbase/templates/named.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/named.tmpl deleted file mode 100644 index 03e054198..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/templates/named.tmpl +++ /dev/null @@ -1,69 +0,0 @@ -// This is the primary configuration file for the BIND DNS server named. -// -// Please read /usr/share/doc/bind9/README.Debian.gz for information on the -// structure of BIND configuration files in Debian, *BEFORE* you customize -// this configuration file. -// - -include "/etc/bind/named.conf.options"; - -include "/etc/bind/rndc.key"; - -// prime the server with knowledge of the root servers -zone "." { - type hint; - file "/etc/bind/db.root"; -}; - -// be authoritative for the localhost forward and reverse zones, and for -// broadcast zones as per RFC 1912 -{% for zone in zones %} -zone "{{ zone.1 }}" { - type master; - file "/etc/bind/hostbase/{{ zone.1 }}"; - notify no; - also-notify { 140.221.9.6;140.221.8.10; }; -};{% endfor %} - -zone "localhost" { - type master; - file "/etc/bind/db.local"; -}; - -zone "127.in-addr.arpa" { - type master; - file "/etc/bind/db.127"; -}; - -zone "0.in-addr.arpa" { - type master; - file "/etc/bind/db.0"; -}; - -zone "255.in-addr.arpa" { - type master; - file "/etc/bind/db.255"; -}; -{% for reverse in reverses %} -zone "{{ reverse.0 }}.in-addr.arpa" { - type master; - file "/etc/bind/hostbase/{{ reverse.0 }}.rev"; - notify no; - also-notify { 140.221.9.6;140.221.8.10; }; -};{% endfor %} - -// zone "com" { type delegation-only; }; -// zone "net" { type delegation-only; }; - -// From the release notes: -// Because many of our users are uncomfortable receiving undelegated answers -// from root or top level domains, other than a few for whom that behaviour -// has been trusted and expected for quite some length of time, we have now -// introduced the "root-delegations-only" feature which applies delegation-only -// logic to all top level domains, and to the root domain. An exception list -// should be specified, including "MUSEUM" and "DE", and any other top level -// domains from whom undelegated responses are expected and trusted. -// root-delegation-only exclude { "DE"; "MUSEUM"; }; - -include "/etc/bind/named.conf.local"; -include "/etc/bind/named.conf.static"; diff --git a/src/lib/Bcfg2/Server/Hostbase/templates/namedviews.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/namedviews.tmpl deleted file mode 100644 index 52021620e..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/templates/namedviews.tmpl +++ /dev/null @@ -1,92 +0,0 @@ -// This is the primary configuration file for the BIND DNS server named. -// -// Please read /usr/share/doc/bind9/README.Debian.gz for information on the -// structure of BIND configuration files in Debian, *BEFORE* you customize -// this configuration file. -// - -include "/etc/bind/named.conf.options"; - -include "/etc/bind/rndc.key"; - -view "internal" { - match-clients { 140.221.9.6;140.221.8.10;140.221.8.88;140.221.8.15; }; - recursion yes; - // prime the server with knowledge of the root servers - zone "." { - type hint; - file "/etc/bind/db.root"; - }; - {% for zone in zones %} - zone "{{ zone.1 }}" { - type master; - file "/etc/bind/hostbase/{{ zone.1 }}"; - notify no; - also-notify { 140.221.9.6;140.221.8.10;140.221.8.88;140.221.8.15; }; - };{% endfor %} - // be authoritative for the localhost forward and reverse zones, and for - // broadcast zones as per RFC 1912 - - zone "localhost" { - type master; - file "/etc/bind/db.local"; - }; - - zone "127.in-addr.arpa" { - type master; - file "/etc/bind/db.127"; - }; - - zone "0.in-addr.arpa" { - type master; - file "/etc/bind/db.0"; - }; - - zone "255.in-addr.arpa" { - type master; - file "/etc/bind/db.255"; - }; - {% for reverse in reverses %} - zone "{{ reverse.0 }}.in-addr.arpa" { - type master; - file "/etc/bind/hostbase/{{ reverse.0 }}.rev"; - notify no; - also-notify { 140.221.9.6;140.221.8.10;140.221.8.88; }; - };{% endfor %} - include "/etc/bind/named.conf.static"; -}; - -view "external" { - match-clients { any; }; - recursion no; - {% for zone in zones %} - zone "{{ zone.1 }}" { - type master; - file "/etc/bind/hostbase/{{ zone.1 }}.external"; - notify no; - };{% endfor %} - - {% for reverse in reverses %} - zone "{{ reverse.0 }}.in-addr.arpa" { - type master; - file "/etc/bind/hostbase/{{ reverse.0 }}.rev.external"; - notify no; - };{% endfor %} - include "/etc/bind/named.conf.static"; -}; - - -// zone "com" { type delegation-only; }; -// zone "net" { type delegation-only; }; - -// From the release notes: -// Because many of our users are uncomfortable receiving undelegated answers -// from root or top level domains, other than a few for whom that behaviour -// has been trusted and expected for quite some length of time, we have now -// introduced the "root-delegations-only" feature which applies delegation-only -// logic to all top level domains, and to the root domain. An exception list -// should be specified, including "MUSEUM" and "DE", and any other top level -// domains from whom undelegated responses are expected and trusted. -// root-delegation-only exclude { "DE"; "MUSEUM"; }; - -include "/etc/bind/named.conf.local"; diff --git a/src/lib/Bcfg2/Server/Hostbase/templates/reverseappend.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/reverseappend.tmpl deleted file mode 100644 index 6ed520c98..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/templates/reverseappend.tmpl +++ /dev/null @@ -1,4 +0,0 @@ -{% if fileorigin %}$ORIGIN {{ fileorigin }}.in-addr.arpa.{% endif %} -$ORIGIN {{ inaddr }}.in-addr.arpa. -{% for host in hosts %}{{ host.0.3 }} PTR {{ host.1 }}. -{% endfor %} diff --git a/src/lib/Bcfg2/Server/Hostbase/templates/reversesoa.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/reversesoa.tmpl deleted file mode 100644 index d142eaf7f..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/templates/reversesoa.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -$ORIGIN . -$TTL {{ zone.8 }} -{{ inaddr }}.in-addr.arpa IN SOA {{ zone.4 }}. {{ zone.3 }} ( - {{ zone.2 }} ; serial - {{ zone.7 }} ; refresh interval - {{ zone.6 }} ; retry interval - {{ zone.5 }} ; expire interval - {{ zone.8 }} ; min ttl - ) - - {% for ns in nameservers %}NS {{ ns.0 }} - {% endfor %} - diff --git a/src/lib/Bcfg2/Server/Hostbase/templates/zone.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/zone.tmpl deleted file mode 100644 index aad48d179..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/templates/zone.tmpl +++ /dev/null @@ -1,18 +0,0 @@ -$ORIGIN . -$TTL {{ zone.8 }} -{{ zone.1 }}. IN SOA {{ zone.4 }}. {{ zone.3 }}. ( - {{ zone.2 }} ; serial - {{ zone.7 }} ; refresh interval - {{ zone.6 }} ; retry interval - {{ zone.5 }} ; expire interval - {{ zone.8 }} ; min ttl - ) - - {% for ns in nameservers %}NS {{ ns.0 }} - {% endfor %} - {% for a in addresses %}A {{ a.0 }} - {% endfor %} - {% for mx in mxs %}MX {{ mx.0 }} {{ mx.1 }} - {% endfor %} -$ORIGIN {{ zone.1 }}. -localhost A 127.0.0.1 diff --git a/src/lib/Bcfg2/Server/Hostbase/urls.py b/src/lib/Bcfg2/Server/Hostbase/urls.py deleted file mode 100644 index 4a0c33f98..000000000 --- a/src/lib/Bcfg2/Server/Hostbase/urls.py +++ /dev/null @@ -1,27 +0,0 @@ -from Bcfg2.Reporting.Compat.django_urls import * -from django.conf import settings -from django.views.generic.simple import direct_to_template -from django.contrib import admin - - -admin.autodiscover() - - -urlpatterns = patterns('', - # Uncomment the admin/doc line below and add 'django.contrib.admindocs' - # to INSTALLED_APPS to enable admin documentation: - (r'^admin/doc/', include('django.contrib.admindocs.urls')), - - # Uncomment the next line to enable the admin: - (r'^admin/', include(admin.site.urls)), - - (r'^$',direct_to_template, {'template':'index.html'}, 'index'), - (r'^hostbase/', include('hostbase.urls')), - (r'^login/$', 'django.contrib.auth.views.login', {'template_name': 'login.html'}), - (r'^logout/$', 'django.contrib.auth.views.logout', {'template_name': 'logout.html'}) -) - -if settings.SERVE_MEDIA: - urlpatterns += patterns('', - (r'^site_media/(?P<path>.*)$', 'django.views.static.serve', - dict(document_root=settings.MEDIA_ROOT)),) diff --git a/src/lib/Bcfg2/Server/Info.py b/src/lib/Bcfg2/Server/Info.py new file mode 100644 index 000000000..a5136f01d --- /dev/null +++ b/src/lib/Bcfg2/Server/Info.py @@ -0,0 +1,884 @@ +""" Subcommands and helpers for bcfg2-info """ +# -*- coding: utf-8 -*- + +import os +import sys +import cmd +import math +import time +import copy +import pipes +import fnmatch +import argparse +import operator +import lxml.etree +from code import InteractiveConsole +import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.Server.Core +import Bcfg2.Server.Plugin +import Bcfg2.Client.Tools.POSIX +from Bcfg2.Compat import any # pylint: disable=W0622 + +try: + try: + import cProfile as profile + except ImportError: + import profile + import pstats + HAS_PROFILE = True +except ImportError: + HAS_PROFILE = False + + +def print_tabular(rows): + """Print data in tabular format.""" + cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 + for index in range(len(rows[0]))]) + fstring = (" %%-%ss |" * len(cmax)) % cmax + fstring = ('|'.join([" %%-%ss "] * len(cmax))) % cmax + print(fstring % rows[0]) + print((sum(cmax) + (len(cmax) * 2) + (len(cmax) - 1)) * '=') + for row in rows[1:]: + print(fstring % row) + + +def display_trace(trace): + """ display statistics from a profile trace """ + stats = pstats.Stats(trace) + stats.sort_stats('cumulative', 'calls', 'time') + stats.print_stats(200) + + +def load_interpreters(): + """ Load a dict of available Python interpreters """ + interpreters = dict(python=lambda v: InteractiveConsole(v).interact()) + default = "python" + try: + import bpython.cli + interpreters["bpython"] = lambda v: bpython.cli.main(args=[], + locals_=v) + default = "bpython" + except ImportError: + pass + + try: + # whether ipython is actually better than bpython is + # up for debate, but this is the behavior that existed + # before --interpreter was added, so we call IPython + # better + import IPython + # pylint: disable=E1101 + if hasattr(IPython, "Shell"): + interpreters["ipython"] = lambda v: \ + IPython.Shell.IPShell(argv=[], user_ns=v).mainloop() + default = "ipython" + elif hasattr(IPython, "embed"): + interpreters["ipython"] = lambda v: IPython.embed(user_ns=v) + default = "ipython" + else: + print("Unknown IPython API version") + # pylint: enable=E1101 + except ImportError: + pass + + return (interpreters, default) + + +class InfoCmd(Bcfg2.Options.Subcommand): # pylint: disable=W0223 + """ Base class for bcfg2-info subcommands """ + + def _expand_globs(self, globs, candidates): + """ Given a list of globs, select the items from candidates + that match the globs """ + # special cases to speed things up: + if not globs or '*' in globs: + return candidates + has_wildcards = False + for glob in globs: + # check if any wildcard characters are in the string + if set('*?[]') & set(glob): + has_wildcards = True + break + if not has_wildcards: + return globs + + rv = set() + cset = set(candidates) + for glob in globs: + rv.update(c for c in cset if fnmatch.fnmatch(c, glob)) + cset.difference_update(rv) + return list(rv) + + def get_client_list(self, globs): + """ given a list of host globs, get a list of clients that + match them """ + return self._expand_globs(globs, self.core.metadata.clients) + + def get_group_list(self, globs): + """ given a list of group glob, get a list of groups that + match them""" + # special cases to speed things up: + return self._expand_globs(globs, + list(self.core.metadata.groups.keys())) + + +class Help(InfoCmd, Bcfg2.Options.HelpCommand): + """ Get help on a specific subcommand """ + def command_registry(self): + return self.core.commands + + def run(self, setup): + Bcfg2.Options.HelpCommand.run(self, setup) + + +class Debug(InfoCmd): + """ Shell out to a Python interpreter """ + interpreters, default_interpreter = load_interpreters() + options = [ + Bcfg2.Options.BooleanOption( + "-n", "--non-interactive", + help="Do not enter the interactive debugger"), + Bcfg2.Options.PathOption( + "-f", dest="cmd_list", type=argparse.FileType('r'), + help="File containing commands to run"), + Bcfg2.Options.Option( + "--interpreter", cf=("bcfg2-info", "interpreter"), + env="BCFG2_INFO_INTERPRETER", + choices=interpreters.keys(), default=default_interpreter)] + + def run(self, setup): + if setup.cmd_list: + console = InteractiveConsole(locals()) + for command in setup.cmd_list.readlines(): + command = command.strip() + if command: + console.push(command) + if not setup.non_interactive: + print("Dropping to interpreter; press ^D to resume") + self.interpreters[setup.interpreter](self.core.get_locals()) + + +class Build(InfoCmd): + """ Build config for hostname, writing to filename """ + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("filename", nargs='?', + default=sys.stdout, + type=argparse.FileType('w'))] + + def run(self, setup): + etree = lxml.etree.ElementTree( + self.core.BuildConfiguration(setup.hostname)) + try: + etree.write( + setup.filename, + encoding='UTF-8', xml_declaration=True, + pretty_print=True) + except IOError: + err = sys.exc_info()[1] + print("Failed to write %s: %s" % (setup.filename, err)) + + +class Builddir(InfoCmd): + """ Build config for hostname, writing separate files to directory + """ + + # don't try to isntall these types of entries + blacklisted_types = ["nonexistent", "permissions"] + + options = Bcfg2.Client.Tools.POSIX.POSIX.options + [ + Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PathOption("directory")] + + help = """Generates a config for client <hostname> and writes the +individual configuration files out separately in a tree under <output +dir>. This only handles file entries, and does not respect 'owner' or +'group' attributes unless run as root. """ + + def run(self, setup): + setup.paranoid = False + client_config = self.core.BuildConfiguration(setup.hostname) + if client_config.tag == 'error': + print("Building client configuration failed.") + return 1 + + entries = [] + for struct in client_config: + for entry in struct: + if (entry.tag == 'Path' and + entry.get("type") not in self.blacklisted_types): + failure = entry.get("failure") + if failure is not None: + print("Skipping entry %s:%s with bind failure: %s" % + (entry.tag, entry.get("name"), failure)) + continue + entry.set('name', + os.path.join(setup.directory, + entry.get('name').lstrip("/"))) + entries.append(entry) + + Bcfg2.Client.Tools.POSIX.POSIX(client_config).Install(entries) + + +class Buildfile(InfoCmd): + """ Build config file for hostname """ + + options = [ + Bcfg2.Options.Option("-f", "--outfile", metavar="<path>", + type=argparse.FileType('w'), default=sys.stdout), + Bcfg2.Options.PathOption("--altsrc"), + Bcfg2.Options.PathOption("filename"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + entry = lxml.etree.Element('Path', name=setup.filename) + if setup.altsrc: + entry.set("altsrc", setup.altsrc) + try: + self.core.Bind(entry, self.core.build_metadata(setup.hostname)) + except: # pylint: disable=W0702 + print("Failed to build entry %s for host %s" % (setup.filename, + setup.hostname)) + raise + try: + setup.outfile.write( + lxml.etree.tostring(entry, + xml_declaration=False).decode('UTF-8')) + setup.outfile.write("\n") + except IOError: + err = sys.exc_info()[1] + print("Failed to write %s: %s" % (setup.outfile.name, err)) + + +class BuildAllMixin(object): + """ InfoCmd mixin to make a version of an existing command that + applies to multiple hosts""" + + directory_arg = Bcfg2.Options.PathOption("directory") + hostname_arg = Bcfg2.Options.PositionalArgument("hostname", nargs='*', + default=[]) + options = [directory_arg, hostname_arg] + + @property + def _parent(self): + """ the parent command """ + for cls in self.__class__.__mro__: + if (cls != InfoCmd and cls != self.__class__ and + issubclass(cls, InfoCmd)): + return cls + + def run(self, setup): + """ Run the command """ + try: + os.makedirs(setup.directory) + except OSError: + err = sys.exc_info()[1] + if err.errno != 17: + print("Could not create %s: %s" % (setup.directory, err)) + return 1 + clients = self.get_client_list(setup.hostname) + for client in clients: + csetup = self._get_setup(client, copy.copy(setup)) + csetup.hostname = client + self._parent.run(self, csetup) # pylint: disable=E1101 + + def _get_setup(self, client, setup): + """ This can be overridden by children to populate individual + setup options on a per-client basis """ + raise NotImplementedError + + +class Buildallfile(Buildfile, BuildAllMixin): + """ Build config file for all clients in directory """ + + options = [BuildAllMixin.directory_arg, + Bcfg2.Options.PathOption("--altsrc"), + Bcfg2.Options.PathOption("filename"), + BuildAllMixin.hostname_arg] + + def run(self, setup): + BuildAllMixin.run(self, setup) + + def _get_setup(self, client, setup): + setup.outfile = open(os.path.join(setup.directory, client), 'w') + return setup + + +class Buildall(Build, BuildAllMixin): + """ Build configs for all clients in directory """ + + options = BuildAllMixin.options + + def run(self, setup): + BuildAllMixin.run(self, setup) + + def _get_setup(self, client, setup): + setup.filename = os.path.join(setup.directory, client + ".xml") + return setup + + +class Buildbundle(InfoCmd): + """ Render a templated bundle for hostname """ + + options = [Bcfg2.Options.PositionalArgument("bundle"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + bundler = self.core.plugins['Bundler'] + bundle = None + if setup.bundle in bundler.entries: + bundle = bundler.entries[setup.bundle] + elif not setup.bundle.endswith(".xml"): + fname = setup.bundle + ".xml" + if fname in bundler.entries: + bundle = bundler.entries[bundle] + if not bundle: + print("No such bundle %s" % setup.bundle) + return 1 + try: + metadata = self.core.build_metadata(setup.hostname) + print(lxml.etree.tostring(bundle.XMLMatch(metadata), + xml_declaration=False, + pretty_print=True).decode('UTF-8')) + except: # pylint: disable=W0702 + print("Failed to render bundle %s for host %s: %s" % + (setup.bundle, setup.hostname, sys.exc_info()[1])) + raise + + +class Automatch(InfoCmd): + """ Perform automatch on a Properties file """ + + options = [ + Bcfg2.Options.BooleanOption( + "-f", "--force", + help="Force automatch even if it's disabled"), + Bcfg2.Options.PositionalArgument("propertyfile"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + props = self.core.plugins['Properties'] + except KeyError: + print("Properties plugin not enabled") + return 1 + + pfile = props.entries[setup.propertyfile] + if (not Bcfg2.Options.setup.force and + not Bcfg2.Options.setup.automatch and + pfile.xdata.get("automatch", "false").lower() != "true"): + print("Automatch not enabled on %s" % setup.propertyfile) + else: + metadata = self.core.build_metadata(setup.hostname) + print(lxml.etree.tostring(pfile.XMLMatch(metadata), + xml_declaration=False, + pretty_print=True).decode('UTF-8')) + + +class ExpireCache(InfoCmd): + """ Expire the metadata cache """ + + options = [ + Bcfg2.Options.PositionalArgument( + "hostname", nargs="*", default=[], + help="Expire cache for the given host(s)")] + + def run(self, setup): + if setup.clients: + for client in self.get_client_list(setup.clients): + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) + else: + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + + +class Bundles(InfoCmd): + """ Print out group/bundle info """ + + options = [Bcfg2.Options.PositionalArgument("group", nargs='*')] + + def run(self, setup): + data = [('Group', 'Bundles')] + groups = self.get_group_list(setup.group) + groups.sort() + for group in groups: + data.append((group, + ','.join(self.core.metadata.groups[group][0]))) + print_tabular(data) + + +class Clients(InfoCmd): + """ Print out client/profile info """ + + options = [Bcfg2.Options.PositionalArgument("hostname", nargs='*', + default=[])] + + def run(self, setup): + data = [('Client', 'Profile')] + for client in sorted(self.get_client_list(setup.hostname)): + imd = self.core.metadata.get_initial_metadata(client) + data.append((client, imd.profile)) + print_tabular(data) + + +class Config(InfoCmd): + """ Print out the current configuration of Bcfg2""" + + options = [ + Bcfg2.Options.BooleanOption( + "--raw", + help="Produce more accurate but less readable raw output")] + + def run(self, setup): + parser = Bcfg2.Options.get_parser() + data = [('Description', 'Value')] + for option in parser.option_list: + if hasattr(setup, option.dest): + value = getattr(setup, option.dest) + if any(issubclass(a.__class__, + Bcfg2.Options.ComponentAction) + for a in option.actions.values()): + if not setup.raw: + try: + if option.action.islist: + value = [v.__name__ for v in value] + else: + value = value.__name__ + except AttributeError: + # just use the value as-is + pass + if setup.raw: + value = repr(value) + data.append((getattr(option, "help", option.dest), value)) + print_tabular(data) + + +class Probes(InfoCmd): + """ Get probes for the given host """ + + options = [ + Bcfg2.Options.BooleanOption("-p", "--pretty", + help="Human-readable output"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + if setup.pretty: + probes = [] + else: + probes = lxml.etree.Element('probes') + metadata = self.core.build_metadata(setup.hostname) + for plugin in self.core.plugins_by_type(Bcfg2.Server.Plugin.Probing): + for probe in plugin.GetProbes(metadata): + probes.append(probe) + if setup.pretty: + for probe in probes: + pname = probe.get("name") + print("=" * (len(pname) + 2)) + print(" %s" % pname) + print("=" * (len(pname) + 2)) + print("") + print(probe.text) + print("") + else: + print(lxml.etree.tostring(probes, xml_declaration=False, + pretty_print=True).decode('UTF-8')) + + +class Showentries(InfoCmd): + """ Show abstract configuration entries for a given host """ + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("type", nargs='?')] + + def run(self, setup): + try: + metadata = self.core.build_metadata(setup.hostname) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Unable to build metadata for %s: %s" % (setup.hostname, + sys.exc_info()[1])) + structures = self.core.GetStructures(metadata) + output = [('Entry Type', 'Name')] + etypes = None + if setup.type: + etypes = [setup.type, "Bound%s" % setup.type] + for item in structures: + output.extend((child.tag, child.get('name')) + for child in item.getchildren() + if not etypes or child.tag in etypes) + print_tabular(output) + + +class Groups(InfoCmd): + """ Print out group info """ + options = [Bcfg2.Options.PositionalArgument("group", nargs='*')] + + def _profile_flag(self, group): + """ Whether or not the group is a profile group """ + if self.core.metadata.groups[group].is_profile: + return 'yes' + else: + return 'no' + + def run(self, setup): + data = [("Groups", "Profile", "Category")] + groups = self.get_group_list(setup.group) + groups.sort() + for group in groups: + data.append((group, + self._profile_flag(group), + self.core.metadata.groups[group].category)) + print_tabular(data) + + +class Showclient(InfoCmd): + """ Show metadata for the given hosts """ + + options = [Bcfg2.Options.PositionalArgument("hostname", nargs='*')] + + def run(self, setup): + for client in self.get_client_list(setup.hostname): + try: + metadata = self.core.build_metadata(client) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Could not build metadata for %s: %s" % + (client, sys.exc_info()[1])) + continue + fmt = "%-10s %s" + print(fmt % ("Hostname:", metadata.hostname)) + print(fmt % ("Profile:", metadata.profile)) + + group_fmt = "%-10s %-30s %s" + header = False + for group in sorted(list(metadata.groups)): + category = "" + for cat, grp in metadata.categories.items(): + if grp == group: + category = "Category: %s" % cat + break + if not header: + print(group_fmt % ("Groups:", group, category)) + header = True + else: + print(group_fmt % ("", group, category)) + + if metadata.bundles: + sorted_bundles = sorted(list(metadata.bundles)) + print(fmt % ("Bundles:", sorted_bundles[0])) + for bnd in sorted_bundles[1:]: + print(fmt % ("", bnd)) + if metadata.connectors: + print("Connector data") + print("=" * 80) + for conn in metadata.connectors: + if getattr(metadata, conn): + print(fmt % (conn + ":", getattr(metadata, conn))) + print("=" * 80) + + +class Mappings(InfoCmd): + """ Print generator mappings for optional type and name """ + + options = [Bcfg2.Options.PositionalArgument("type", nargs='?'), + Bcfg2.Options.PositionalArgument("name", nargs='?')] + + def run(self, setup): + data = [('Plugin', 'Type', 'Name')] + for generator in self.core.plugins_by_type( + Bcfg2.Server.Plugin.Generator): + etypes = setup.type or list(generator.Entries.keys()) + if setup.name: + interested = [(etype, [setup.name]) for etype in etypes] + else: + interested = [(etype, generator.Entries[etype]) + for etype in etypes + if etype in generator.Entries] + for etype, names in interested: + data.extend((generator.name, etype, name) + for name in names + if name in generator.Entries.get(etype, {})) + print_tabular(data) + + +class PackageResolve(InfoCmd): + """ Resolve packages for the given host""" + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("package", nargs="*")] + + def run(self, setup): + try: + pkgs = self.core.plugins['Packages'] + except KeyError: + print("Packages plugin not enabled") + return 1 + + metadata = self.core.build_metadata(setup.hostname) + + indep = lxml.etree.Element("Independent", + name=self.__class__.__name__.lower()) + if setup.package: + structures = [lxml.etree.Element("Bundle", name="packages")] + for package in setup.package: + lxml.etree.SubElement(structures[0], "Package", name=package) + else: + structures = self.core.GetStructures(metadata) + + pkgs._build_packages(metadata, indep, # pylint: disable=W0212 + structures) + print("%d new packages added" % len(indep.getchildren())) + if len(indep.getchildren()): + print(" %s" % "\n ".join(lxml.etree.tostring(p) + for p in indep.getchildren())) + + +class Packagesources(InfoCmd): + """ Show package sources """ + + options = [Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + pkgs = self.core.plugins['Packages'] + except KeyError: + print("Packages plugin not enabled") + return 1 + try: + metadata = self.core.build_metadata(setup.hostname) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Unable to build metadata for %s: %s" % (setup.hostname, + sys.exc_info()[1])) + return 1 + print(pkgs.get_collection(metadata).sourcelist()) + + +class Query(InfoCmd): + """ Query clients """ + + options = [ + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.Option( + "-g", "--group", metavar="<group>", dest="querygroups", + type=Bcfg2.Options.Types.comma_list), + Bcfg2.Options.Option( + "-p", "--profile", metavar="<profile>", dest="queryprofiles", + type=Bcfg2.Options.Types.comma_list), + Bcfg2.Options.Option( + "-b", "--bundle", metavar="<bundle>", dest="querybundles", + type=Bcfg2.Options.Types.comma_list), + required=True)] + + def run(self, setup): + if setup.queryprofiles: + res = self.core.metadata.get_client_names_by_profiles( + setup.queryprofiles) + elif setup.querygroups: + res = self.core.metadata.get_client_names_by_groups( + setup.querygroups) + elif setup.querybundles: + res = self.core.metadata.get_client_names_by_bundles( + setup.querybundles) + print("\n".join(res)) + + +class Shell(InfoCmd): + """ Open an interactive shell to run multiple bcfg2-info commands """ + interactive = False + + def run(self, setup): + try: + self.core.cmdloop('Welcome to bcfg2-info\n' + 'Type "help" for more information') + except KeyboardInterrupt: + print("\nCtrl-C pressed, exiting...") + + +class ProfileTemplates(InfoCmd): + """ Benchmark template rendering times """ + + options = [ + Bcfg2.Options.Option( + "--clients", type=Bcfg2.Options.Types.comma_list, + help="Benchmark templates for the named clients"), + Bcfg2.Options.Option( + "--runs", help="Number of rendering passes per template", + default=5, type=int), + Bcfg2.Options.PositionalArgument( + "templates", nargs="*", default=[], + help="Profile the named templates instead of all templates")] + + def profile_entry(self, entry, metadata, runs=5): + """ Profile a single entry """ + times = [] + for i in range(runs): # pylint: disable=W0612 + start = time.time() + try: + self.core.Bind(entry, metadata) + times.append(time.time() - start) + except: # pylint: disable=W0702 + break + if times: + avg = sum(times) / len(times) + if avg: + self.logger.debug(" %s: %.02f sec" % + (metadata.hostname, avg)) + return times + + def profile_struct(self, struct, metadata, templates=None, runs=5): + """ Profile all entries in a given structure """ + times = dict() + entries = struct.xpath("//Path") + entry_count = 0 + for entry in entries: + entry_count += 1 + if templates is None or entry.get("name") in templates: + self.logger.info("Rendering Path:%s (%s/%s)..." % + (entry.get("name"), entry_count, + len(entries))) + times.setdefault(entry.get("name"), + self.profile_entry(entry, metadata, + runs=runs)) + return times + + def profile_client(self, metadata, templates=None, runs=5): + """ Profile all structures for a given client """ + structs = self.core.GetStructures(metadata) + struct_count = 0 + times = dict() + for struct in structs: + struct_count += 1 + self.logger.info("Rendering templates from structure %s:%s " + "(%s/%s)" % + (struct.tag, struct.get("name"), struct_count, + len(structs))) + times.update(self.profile_struct(struct, metadata, + templates=templates, runs=runs)) + return times + + def stdev(self, nums): + """ Calculate the standard deviation of a list of numbers """ + mean = float(sum(nums)) / len(nums) + return math.sqrt(sum((n - mean) ** 2 for n in nums) / float(len(nums))) + + def run(self, setup): + clients = self.get_client_list(setup.clients) + + times = dict() + client_count = 0 + for client in clients: + client_count += 1 + self.logger.info("Rendering templates for client %s (%s/%s)" % + (client, client_count, len(clients))) + times.update(self.profile_client(self.core.build_metadata(client), + templates=setup.templates, + runs=setup.runs)) + + # print out per-file results + tmpltimes = [] + for tmpl, ptimes in times.items(): + try: + mean = float(sum(ptimes)) / len(ptimes) + except ZeroDivisionError: + continue + ptimes.sort() + median = ptimes[len(ptimes) / 2] + std = self.stdev(ptimes) + if mean > 0.01 or median > 0.01 or std > 1 or setup.templates: + tmpltimes.append((tmpl, mean, median, std)) + print("%-50s %-9s %-11s %6s" % + ("Template", "Mean Time", "Median Time", "σ")) + for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): + print("%-50s %9.02f %11.02f %6.02f" % info) + + +if HAS_PROFILE: + class Profile(InfoCmd): + """ Profile a single bcfg2-info command """ + + options = [Bcfg2.Options.PositionalArgument("command"), + Bcfg2.Options.PositionalArgument("args", nargs="*")] + + def run(self, setup): + prof = profile.Profile() + cls = self.core.commands[setup.command] + prof.runcall(cls, " ".join(pipes.quote(a) for a in setup.args)) + display_trace(prof) + + +class InfoCore(cmd.Cmd, + Bcfg2.Server.Core.Core, + Bcfg2.Options.CommandRegistry): + """Main class for bcfg2-info.""" + + def __init__(self): + cmd.Cmd.__init__(self) + Bcfg2.Server.Core.Core.__init__(self) + Bcfg2.Options.CommandRegistry.__init__(self) + self.prompt = 'bcfg2-info> ' + + def get_locals(self): + """ Expose the local variables of the core to subcommands that + need to reference them (i.e., the interactive interpreter) """ + return locals() + + def do_quit(self, _): + """ quit|exit - Exit program """ + raise SystemExit(0) + + do_EOF = do_quit + do_exit = do_quit + + def do_eventdebug(self, _): + """ eventdebug - Enable debugging output for FAM events """ + self.fam.set_debug(True) + + do_event_debug = do_eventdebug + + def do_update(self, _): + """ update - Process pending filesystem events """ + self.fam.handle_events_in_interval(0.1) + + def run(self): + self.load_plugins() + self.block_for_fam_events(handle_events=True) + + def _run(self): + pass + + def _block(self): + pass + + def shutdown(self): + Bcfg2.Options.CommandRegistry.shutdown(self) + Bcfg2.Server.Core.Core.shutdown(self) + + +class CLI(object): + """ The bcfg2-info CLI """ + options = [Bcfg2.Options.BooleanOption("-p", "--profile", help="Profile")] + + def __init__(self): + Bcfg2.Options.register_commands(InfoCore, globals().values(), + parent=InfoCmd) + parser = Bcfg2.Options.get_parser( + description="Inspect a running Bcfg2 server", + components=[self, InfoCore]) + parser.parse() + + if Bcfg2.Options.setup.profile and HAS_PROFILE: + prof = profile.Profile() + self.core = prof.runcall(InfoCore) + display_trace(prof) + else: + if Bcfg2.Options.setup.profile: + print("Profiling functionality not available.") + self.core = InfoCore() + + for command in self.core.commands.values(): + command.core = self.core + + def run(self): + """ Run bcfg2-info """ + if Bcfg2.Options.setup.subcommand != 'help': + self.core.run() + return self.core.runcommand() diff --git a/src/lib/Bcfg2/Server/Lint/AWSTags.py b/src/lib/Bcfg2/Server/Lint/AWSTags.py new file mode 100644 index 000000000..c6d7a3a30 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/AWSTags.py @@ -0,0 +1,33 @@ +""" ``bcfg2-lint`` plugin to check all given :ref:`AWSTags +<server-plugins-connectors-awstags>` patterns for validity.""" + +import re +import sys +import Bcfg2.Server.Lint + + +class AWSTags(Bcfg2.Server.Lint.ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags + <server-plugins-connectors-awstags>` patterns for validity. """ + __serverplugin__ = 'AWSTags' + + def Run(self): + cfg = self.core.plugins['AWSTags'].config + for entry in cfg.xdata.xpath('//Tag'): + self.check(entry, "name") + if entry.get("value"): + self.check(entry, "value") + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, attr): + """ Check a single attribute (``name`` or ``value``) of a + single entry for validity. """ + try: + re.compile(entry.get(attr)) + except re.error: + self.LintError("pattern-fails-to-initialize", + "'%s' regex could not be compiled: %s\n %s" % + (attr, sys.exc_info()[1], entry.get("name"))) diff --git a/src/lib/Bcfg2/Server/Lint/Bundler.py b/src/lib/Bcfg2/Server/Lint/Bundler.py new file mode 100644 index 000000000..576e157ad --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Bundler.py @@ -0,0 +1,59 @@ +""" ``bcfg2-lint`` plugin for :ref:`Bundler +<server-plugins-structures-bundler>` """ + +from Bcfg2.Server.Lint import ServerPlugin + + +class Bundler(ServerPlugin): + """ Perform various :ref:`Bundler + <server-plugins-structures-bundler>` checks. """ + __serverplugin__ = 'Bundler' + + def Run(self): + self.missing_bundles() + for bundle in self.core.plugins['Bundler'].entries.values(): + if self.HandlesFile(bundle.name): + self.bundle_names(bundle) + + @classmethod + def Errors(cls): + return {"bundle-not-found": "error", + "unused-bundle": "warning", + "explicit-bundle-name": "error", + "genshi-extension-bundle": "error"} + + def missing_bundles(self): + """ Find bundles listed in Metadata but not implemented in + Bundler. """ + if self.files is None: + # when given a list of files on stdin, this check is + # useless, so skip it + groupdata = self.metadata.groups_xml.xdata + ref_bundles = set([b.get("name") + for b in groupdata.findall("//Bundle")]) + + allbundles = self.core.plugins['Bundler'].bundles.keys() + for bundle in ref_bundles: + if bundle not in allbundles: + self.LintError("bundle-not-found", + "Bundle %s referenced, but does not exist" % + bundle) + + for bundle in allbundles: + if bundle not in ref_bundles: + self.LintError("unused-bundle", + "Bundle %s defined, but is not referenced " + "in Metadata" % bundle) + + def bundle_names(self, bundle): + """ Verify that deprecated bundle .genshi bundles and explicit + bundle names aren't used """ + if bundle.xdata.get('name'): + self.LintError("explicit-bundle-name", + "Deprecated explicit bundle name in %s" % + bundle.name) + + if bundle.name.endswith(".genshi"): + self.LintError("genshi-extension-bundle", + "Bundle %s uses deprecated .genshi extension" % + bundle.name) diff --git a/src/lib/Bcfg2/Server/Lint/Cfg.py b/src/lib/Bcfg2/Server/Lint/Cfg.py new file mode 100644 index 000000000..13b04a6b8 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Cfg.py @@ -0,0 +1,118 @@ +""" ``bcfg2-lint`` plugin for :ref:`Cfg +<server-plugins-generators-cfg>` """ + +import os +import Bcfg2.Options +from fnmatch import fnmatch +from Bcfg2.Server.Lint import ServerPlugin +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + + +class Cfg(ServerPlugin): + """ warn about Cfg issues """ + __serverplugin__ = 'Cfg' + + def Run(self): + for basename, entry in list(self.core.plugins['Cfg'].entries.items()): + self.check_pubkey(basename, entry) + self.check_missing_files() + self.check_conflicting_handlers() + + @classmethod + def Errors(cls): + return {"no-pubkey-xml": "warning", + "unknown-cfg-files": "error", + "extra-cfg-files": "error", + "multiple-global-handlers": "error"} + + def check_conflicting_handlers(self): + """ Check that a single entryset doesn't have multiple + non-specific (i.e., 'all') handlers. """ + cfg = self.core.plugins['Cfg'] + for eset in cfg.entries.values(): + alls = [e for e in eset.entries.values() + if (e.specific.all and + issubclass(e.__class__, CfgGenerator))] + if len(alls) > 1: + self.LintError("multiple-global-handlers", + "%s has multiple global handlers: %s" % + (eset.path, ", ".join(os.path.basename(e.name) + for e in alls))) + + def check_pubkey(self, basename, entry): + """ check that privkey.xml files have corresponding pubkey.xml + files """ + if "privkey.xml" not in entry.entries: + return + privkey = entry.entries["privkey.xml"] + if not self.HandlesFile(privkey.name): + return + + pubkey = basename + ".pub" + if pubkey not in self.core.plugins['Cfg'].entries: + self.LintError("no-pubkey-xml", + "%s has no corresponding pubkey.xml at %s" % + (basename, pubkey)) + else: + pubset = self.core.plugins['Cfg'].entries[pubkey] + if "pubkey.xml" not in pubset.entries: + self.LintError("no-pubkey-xml", + "%s has no corresponding pubkey.xml at %s" % + (basename, pubkey)) + + def _list_path_components(self, path): + """ Get a list of all components of a path. E.g., + ``self._list_path_components("/foo/bar/foobaz")`` would return + ``["foo", "bar", "foo", "baz"]``. The list is not guaranteed + to be in order.""" + rv = [] + remaining, component = os.path.split(path) + while component != '': + rv.append(component) + remaining, component = os.path.split(remaining) + return rv + + def check_missing_files(self): + """ check that all files on the filesystem are known to Cfg """ + cfg = self.core.plugins['Cfg'] + + # first, collect ignore patterns from handlers + ignore = set() + for hdlr in Bcfg2.Options.setup.cfg_handlers: + ignore.update(hdlr.__ignore__) + + # next, get a list of all non-ignored files on the filesystem + all_files = set() + for root, _, files in os.walk(cfg.data): + for fname in files: + fpath = os.path.join(root, fname) + # check against the handler ignore patterns and the + # global FAM ignore list + if (not any(fname.endswith("." + i) for i in ignore) and + not any(fnmatch(fpath, p) + for p in Bcfg2.Options.setup.ignore_files) and + not any(fnmatch(c, p) + for p in Bcfg2.Options.setup.ignore_files + for c in self._list_path_components(fpath))): + all_files.add(fpath) + + # next, get a list of all files known to Cfg + cfg_files = set() + for root, eset in cfg.entries.items(): + cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) + for fname in eset.entries.keys()) + + # finally, compare the two + unknown_files = all_files - cfg_files + extra_files = cfg_files - all_files + if unknown_files: + self.LintError( + "unknown-cfg-files", + "Files on the filesystem could not be understood by Cfg: %s" % + "; ".join(unknown_files)) + if extra_files: + self.LintError( + "extra-cfg-files", + "Cfg has entries for files that do not exist on the " + "filesystem: %s\nThis is probably a bug." % + "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index f028e225e..fbe84de87 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -2,12 +2,14 @@ import os import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Lint from Bcfg2.Server import XI_NAMESPACE from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ import CfgPlaintextGenerator from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML @@ -16,6 +18,97 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): give information about the files. For instance, you can require SVN keywords in a comment, or require the name of the maintainer of a Genshi template, and so on. """ + + options = Bcfg2.Server.Lint.ServerPlugin.options + [ + Bcfg2.Options.Option( + cf=("Comments", "global_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for all file types"), + Bcfg2.Options.Option( + cf=("Comments", "global_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for all file types"), + Bcfg2.Options.Option( + cf=("Comments", "bundler_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for non-templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "bundler_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for non-templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "genshibundler_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "genshibundler_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "properties_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Properties files"), + Bcfg2.Options.Option( + cf=("Comments", "properties_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Properties files"), + Bcfg2.Options.Option( + cf=("Comments", "cfg_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for non-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "cfg_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for non-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "genshi_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Genshi-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "genshi_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Genshi-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "cheetah_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Cheetah-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "cheetah_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Cheetah-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "jinja2_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Jinja2-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "jinja2_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Jinja2-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "infoxml_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for info.xml files"), + Bcfg2.Options.Option( + cf=("Comments", "infoxml_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for info.xml files"), + Bcfg2.Options.Option( + cf=("Comments", "probes_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for probes"), + Bcfg2.Options.Option( + cf=("Comments", "probes_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for probes"), + Bcfg2.Options.Option( + cf=("Comments", "metadata_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for metadata files"), + Bcfg2.Options.Option( + cf=("Comments", "metadata_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for metadata files")] + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.config_cache = {} @@ -73,17 +166,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): if rtype not in self.config_cache[itype]: rv = [] - global_item = "global_%ss" % itype - if global_item in self.config: - rv.extend(self.config[global_item].split(",")) - - item = "%s_%ss" % (rtype.lower(), itype) - if item in self.config: - if self.config[item]: - rv.extend(self.config[item].split(",")) - else: - # config explicitly specifies nothing - rv = [] + rv.extend(getattr(Bcfg2.Options.setup, "global_%ss" % itype)) + local_reqs = getattr(Bcfg2.Options.setup, + "%s_%ss" % (rtype.lower(), itype)) + if local_reqs == ['']: + # explicitly specified as empty + rv = [] + else: + rv.extend(local_reqs) self.config_cache[itype][rtype] = rv return self.config_cache[itype][rtype] @@ -162,9 +252,11 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): rtype = "cfg" elif isinstance(entry, CfgCheetahGenerator): rtype = "cheetah" + elif isinstance(entry, CfgJinja2Generator): + rtype = "jinja2" elif isinstance(entry, CfgInfoXML): self.check_xml(entry.infoxml.name, - entry.infoxml.pnode.data, + entry.infoxml.xdata, "infoxml") continue if rtype: diff --git a/src/lib/Bcfg2/Server/Lint/Crypto.py b/src/lib/Bcfg2/Server/Lint/Crypto.py new file mode 100644 index 000000000..53a54031c --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Crypto.py @@ -0,0 +1,61 @@ +""" Check for data that claims to be encrypted, but is not. """ + +import os +import lxml.etree +import Bcfg2.Options +from Bcfg2.Server.Lint import ServerlessPlugin +from Bcfg2.Server.Encryption import is_encrypted + + +class Crypto(ServerlessPlugin): + """ Check for templated scripts or executables. """ + + def Run(self): + if os.path.exists(os.path.join(Bcfg2.Options.setup.repository, "Cfg")): + self.check_cfg() + if os.path.exists(os.path.join(Bcfg2.Options.setup.repository, + "Properties")): + self.check_properties() + # TODO: check all XML files + + @classmethod + def Errors(cls): + return {"unencrypted-cfg": "error", + "empty-encrypted-properties": "error", + "unencrypted-properties": "error"} + + def check_cfg(self): + """ Check for Cfg files that end in .crypt but aren't encrypted """ + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, "Cfg")): + for fname in files: + fpath = os.path.join(root, fname) + if self.HandlesFile(fpath) and fname.endswith(".crypt"): + if not is_encrypted(open(fpath).read()): + self.LintError( + "unencrypted-cfg", + "%s is a .crypt file, but it is not encrypted" % + fpath) + + def check_properties(self): + """ Check for Properties data that has an ``encrypted`` attribute but + aren't encrypted """ + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, "Properties")): + for fname in files: + fpath = os.path.join(root, fname) + if self.HandlesFile(fpath) and fname.endswith(".xml"): + xdata = lxml.etree.parse(fpath) + for elt in xdata.xpath('//*[@encrypted]'): + if not elt.text: + self.LintError( + "empty-encrypted-properties", + "Element in %s has an 'encrypted' attribute, " + "but no text content: %s" % + (fpath, self.RenderXML(elt))) + elif not is_encrypted(elt.text): + self.LintError( + "unencrypted-properties", + "Element in %s has an 'encrypted' attribute, " + "but is not encrypted: %s" % + (fpath, self.RenderXML(elt))) diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index 1ecb6da42..a2581e70b 100755..100644 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -4,7 +4,6 @@ import sys import Bcfg2.Server.Lint from genshi.template import TemplateLoader, NewTextTemplate, MarkupTemplate, \ TemplateSyntaxError -from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator @@ -14,60 +13,41 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): def Run(self): if 'Cfg' in self.core.plugins: self.check_cfg() - if 'TGenshi' in self.core.plugins: - self.check_tgenshi() if 'Bundler' in self.core.plugins: self.check_bundler() @classmethod def Errors(cls): - return {"genshi-syntax-error": "error"} + return {"genshi-syntax-error": "error", + "unknown-genshi-error": "error"} + + def check_template(self, loader, fname, cls=None): + """ Generic check for all genshi templates (XML and text) """ + try: + loader.load(fname, cls=cls) + except TemplateSyntaxError: + err = sys.exc_info()[1] + self.LintError("genshi-syntax-error", + "Genshi syntax error in %s: %s" % (fname, err)) + except: + err = sys.exc_info()[1] + self.LintError("unknown-genshi-error", + "Unknown Genshi error in %s: %s" % (fname, err)) def check_cfg(self): """ Check genshi templates in Cfg for syntax errors. """ for entryset in self.core.plugins['Cfg'].entries.values(): for entry in entryset.entries.values(): if (self.HandlesFile(entry.name) and - isinstance(entry, CfgGenshiGenerator) and - not entry.template): - try: - entry.loader.load(entry.name, - cls=NewTextTemplate) - except TemplateSyntaxError: - err = sys.exc_info()[1] - self.LintError("genshi-syntax-error", - "Genshi syntax error: %s" % err) - except: - etype, err = sys.exc_info()[:2] - self.LintError( - "genshi-syntax-error", - "Unexpected Genshi error on %s: %s: %s" % - (entry.name, etype.__name__, err)) - - def check_tgenshi(self): - """ Check templates in TGenshi for syntax errors. """ - loader = TemplateLoader() - - for eset in self.core.plugins['TGenshi'].entries.values(): - for fname, sdata in list(eset.entries.items()): - if self.HandlesFile(fname): - try: - loader.load(sdata.name, cls=NewTextTemplate) - except TemplateSyntaxError: - err = sys.exc_info()[1] - self.LintError("genshi-syntax-error", - "Genshi syntax error: %s" % err) + isinstance(entry, CfgGenshiGenerator) and + not entry.template): + self.check_template(entry.loader, entry.name, + cls=NewTextTemplate) def check_bundler(self): """ Check templates in Bundler for syntax errors. """ loader = TemplateLoader() - for entry in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(entry.name) and - isinstance(entry, BundleTemplateFile)): - try: - loader.load(entry.name, cls=MarkupTemplate) - except TemplateSyntaxError: - err = sys.exc_info()[1] - self.LintError("genshi-syntax-error", - "Genshi syntax error: %s" % err) + entry.template is not None): + self.check_template(loader, entry.name, cls=MarkupTemplate) diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py index b180083d5..e28080300 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupNames.py +++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py @@ -3,11 +3,6 @@ import os import re import Bcfg2.Server.Lint -try: - from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile - HAS_GENSHI = True -except ImportError: - HAS_GENSHI = False class GroupNames(Bcfg2.Server.Lint.ServerPlugin): @@ -44,14 +39,13 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): continue xdata = rules.pnode.data self.check_entries(xdata.xpath("//Group"), - os.path.join(self.config['repo'], rules.name)) + os.path.join(Bcfg2.Options.setup.repository, + rules.name)) def check_bundles(self): """ Check groups used in the Bundler plugin for validity. """ for bundle in self.core.plugins['Bundler'].entries.values(): - if (self.HandlesFile(bundle.name) and - (not HAS_GENSHI or - not isinstance(bundle, BundleTemplateFile))): + if self.HandlesFile(bundle.name) and bundle.template is None: self.check_entries(bundle.xdata.xpath("//Group"), bundle.name) @@ -59,7 +53,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): """ Check groups used or declared in the Metadata plugin for validity. """ self.check_entries(self.metadata.groups_xml.xdata.xpath("//Group"), - os.path.join(self.config['repo'], + os.path.join(Bcfg2.Options.setup.repository, self.metadata.groups_xml.name)) def check_grouppatterns(self): diff --git a/src/lib/Bcfg2/Server/Lint/GroupPatterns.py b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py new file mode 100644 index 000000000..8ddb9e796 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py @@ -0,0 +1,44 @@ +""" ``bcfg2-lint`` plugin for :ref:`GroupPatterns +<server-plugins-grouping-grouppatterns>` """ + +import sys +from Bcfg2.Server.Lint import ServerPlugin +from Bcfg2.Server.Plugins.GroupPatterns import PatternMap + + +class GroupPatterns(ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns + <server-plugins-grouping-grouppatterns>` patterns for validity. + This is simply done by trying to create a + :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for + each pattern, and catching exceptions and presenting them as + ``bcfg2-lint`` errors.""" + __serverplugin__ = 'GroupPatterns' + + def Run(self): + cfg = self.core.plugins['GroupPatterns'].config + for entry in cfg.xdata.xpath('//GroupPattern'): + groups = [g.text for g in entry.findall('Group')] + self.check(entry, groups, ptype='NamePattern') + self.check(entry, groups, ptype='NameRange') + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, groups, ptype="NamePattern"): + """ Check a single pattern for validity """ + if ptype == "NamePattern": + pmap = lambda p: PatternMap(p, None, groups) + else: + pmap = lambda p: PatternMap(None, p, groups) + + for el in entry.findall(ptype): + pat = el.text + try: + pmap(pat) + except: # pylint: disable=W0702 + err = sys.exc_info()[1] + self.LintError("pattern-fails-to-initialize", + "Failed to initialize %s %s for %s: %s" % + (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index 95657317e..950a86f01 100644 --- a/src/lib/Bcfg2/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -4,7 +4,6 @@ import os import Bcfg2.Options import Bcfg2.Server.Lint from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML -from Bcfg2.Server.Plugins.Cfg.CfgLegacyInfo import CfgLegacyInfo class InfoXML(Bcfg2.Server.Lint.ServerPlugin): @@ -16,6 +15,16 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): * Paranoid mode disabled in an ``info.xml`` file; * Required attributes missing from ``info.xml`` """ + __serverplugin__ = 'Cfg' + + options = Bcfg2.Server.Lint.ServerPlugin.options + [ + Bcfg2.Options.Common.default_paranoid, + Bcfg2.Options.Option( + cf=("InfoXML", "required_attrs"), + type=Bcfg2.Options.Types.comma_list, + default=["owner", "group", "mode"], + help="Attributes to require on <Info> tags")] + def Run(self): if 'Cfg' not in self.core.plugins: return @@ -27,25 +36,15 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): for entry in entryset.entries.values(): if isinstance(entry, CfgInfoXML): self.check_infoxml(infoxml_fname, - entry.infoxml.pnode.data) + entry.infoxml.xdata) found = True if not found: self.LintError("no-infoxml", "No info.xml found for %s" % filename) - for entry in entryset.entries.values(): - if isinstance(entry, CfgLegacyInfo): - if not self.HandlesFile(entry.path): - continue - self.LintError("deprecated-info-file", - "Deprecated %s file found at %s" % - (os.path.basename(entry.name), - entry.path)) - @classmethod def Errors(cls): return {"no-infoxml": "warning", - "deprecated-info-file": "warning", "paranoid-false": "warning", "required-infoxml-attrs-missing": "error"} @@ -53,8 +52,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): """ Verify that info.xml contains everything it should. """ for info in xdata.getroottree().findall("//Info"): required = [] - if "required_attrs" in self.config: - required = self.config["required_attrs"].split(",") + required = Bcfg2.Options.setup.required_attrs missing = [attr for attr in required if info.get(attr) is None] if missing: @@ -63,10 +61,10 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): (",".join(missing), fname, self.RenderXML(info))) - if ((Bcfg2.Options.MDATA_PARANOID.value and + if ((Bcfg2.Options.setup.default_paranoid == "true" and info.get("paranoid") is not None and info.get("paranoid").lower() == "false") or - (not Bcfg2.Options.MDATA_PARANOID.value and + (Bcfg2.Options.setup.default_paranoid == "false" and (info.get("paranoid") is None or info.get("paranoid").lower() != "true"))): self.LintError("paranoid-false", diff --git a/src/lib/Bcfg2/Server/Lint/Jinja2.py b/src/lib/Bcfg2/Server/Lint/Jinja2.py new file mode 100644 index 000000000..333249cc2 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Jinja2.py @@ -0,0 +1,41 @@ +""" Check Jinja2 templates for syntax errors. """ + +import sys +import Bcfg2.Server.Lint +from jinja2 import Template, TemplateSyntaxError +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator + + +class Jinja2(Bcfg2.Server.Lint.ServerPlugin): + """ Check Jinja2 templates for syntax errors. """ + + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + + @classmethod + def Errors(cls): + return {"jinja2-syntax-error": "error", + "unknown-jinja2-error": "error"} + + def check_template(self, entry): + """ Generic check for all jinja2 templates """ + try: + Template(entry.data.decode(entry.encoding)) + except TemplateSyntaxError: + err = sys.exc_info()[1] + self.LintError("jinja2-syntax-error", + "Jinja2 syntax error in %s: %s" % (entry.name, err)) + except: + err = sys.exc_info()[1] + self.LintError("unknown-jinja2-error", + "Unknown Jinja2 error in %s: %s" % (entry.name, + err)) + + def check_cfg(self): + """ Check jinja2 templates in Cfg for syntax errors. """ + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if (self.HandlesFile(entry.name) and + isinstance(entry, CfgJinja2Generator)): + self.check_template(entry) diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index 2419c3d43..8e6a926ae 100644 --- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -8,9 +8,24 @@ import Bcfg2.Server.Lint from Bcfg2.Server.Plugins.Cfg import CfgGenerator +def threshold(val): + """ Option type processor to accept either a percentage (e.g., + "threshold=75") or a ratio (e.g., "threshold=.75") """ + rv = float(val) + if rv > 1: + rv /= 100 + return rv + + class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): """ find Probes or Cfg files with multiple similar files that might be merged into one """ + + options = Bcfg2.Server.Lint.ServerPlugin.options + [ + Bcfg2.Options.Option( + cf=("MergeFiles", "threshold"), default="0.75", type=threshold, + help="The threshold at which to suggest merging files and probes")] + def Run(self): if 'Cfg' in self.core.plugins: self.check_cfg() @@ -20,14 +35,25 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): return {"merge-cfg": "warning", - "merge-probes": "warning"} + "identical-cfg": "error", + "merge-probes": "warning", + "identical-probes": "error"} def check_cfg(self): """ check Cfg for similar files """ + # ignore non-specific Cfg entries, e.g., privkey.xml + ignore = [] + for hdlr in Bcfg2.Options.setup.cfg_handlers: + if not hdlr.__specific__: + ignore.extend(hdlr.__basenames__) + for filename, entryset in self.core.plugins['Cfg'].entries.items(): candidates = dict([(f, e) for f, e in entryset.entries.items() - if isinstance(e, CfgGenerator)]) - for mset in self.get_similar(candidates): + if (isinstance(e, CfgGenerator) and + f not in ignore and + not f.endswith(".crypt"))]) + similar, identical = self.get_similar(candidates) + for mset in similar: self.LintError("merge-cfg", "The following files are similar: %s. " "Consider merging them into a single Genshi " @@ -35,54 +61,69 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): ", ".join([os.path.join(filename, p) for p in mset])) + for mset in identical: + self.LintError("identical-cfg", + "The following files are identical: %s. " + "Strongly consider merging them into a single " + "Genshi template." % + ", ".join([os.path.join(filename, p) + for p in mset])) + def check_probes(self): """ check Probes for similar files """ probes = self.core.plugins['Probes'].probes.entries - for mset in self.get_similar(probes): + similar, identical = self.get_similar(probes) + for mset in similar: self.LintError("merge-probes", "The following probes are similar: %s. " "Consider merging them into a single probe." % ", ".join([p for p in mset])) + for mset in identical: + self.LintError("identical-probes", + "The following probes are identical: %s. " + "Strongly consider merging them into a single " + "probe." % + ", ".join([p for p in mset])) def get_similar(self, entries): """ Get a list of similar files from the entry dict. Return value is a list of lists, each of which gives the filenames of similar files """ - if "threshold" in self.config: - # accept threshold either as a percent (e.g., "threshold=75") or - # as a ratio (e.g., "threshold=.75") - threshold = float(self.config['threshold']) - if threshold > 1: - threshold /= 100 - else: - threshold = 0.75 - rv = [] + similar = [] + identical = [] elist = list(entries.items()) while elist: - result = self._find_similar(elist.pop(0), copy.copy(elist), - threshold) - if len(result) > 1: - elist = [(fname, fdata) - for fname, fdata in elist - if fname not in result] - rv.append(result) - return rv + rv = self._find_similar(elist.pop(0), copy.copy(elist)) + if rv[0]: + similar.append(rv[0]) + if rv[1]: + identical.append(rv[1]) + elist = [(fname, fdata) + for fname, fdata in elist + if fname not in rv[0] | rv[1]] + return similar, identical - def _find_similar(self, ftuple, others, threshold): + def _find_similar(self, ftuple, others): """ Find files similar to the one described by ftupe in the list of other files. ftuple is a tuple of (filename, data); others is a list of such tuples. threshold is a float between 0 and 1 that describes how similar two files much be to rate as 'similar' """ fname, fdata = ftuple - rv = [fname] - while others: - cname, cdata = others.pop(0) + similar = set() + identical = set() + for cname, cdata in others: seqmatch = SequenceMatcher(None, fdata.data, cdata.data) # perform progressively more expensive comparisons - if (seqmatch.real_quick_ratio() > threshold and - seqmatch.quick_ratio() > threshold and - seqmatch.ratio() > threshold): - rv.extend(self._find_similar((cname, cdata), copy.copy(others), - threshold)) - return rv + if seqmatch.real_quick_ratio() == 1.0: + identical.add(cname) + elif ( + seqmatch.real_quick_ratio() > Bcfg2.Options.setup.threshold and + seqmatch.quick_ratio() > Bcfg2.Options.setup.threshold and + seqmatch.ratio() > Bcfg2.Options.setup.threshold): + similar.add(cname) + if similar: + similar.add(fname) + if identical: + identical.add(fname) + return (similar, identical) diff --git a/src/lib/Bcfg2/Server/Lint/Metadata.py b/src/lib/Bcfg2/Server/Lint/Metadata.py new file mode 100644 index 000000000..e445892d1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Metadata.py @@ -0,0 +1,172 @@ +""" ``bcfg2-lint`` plugin for :ref:`Metadata +<server-plugins-grouping-metadata>` """ + +from Bcfg2.Server.Lint import ServerPlugin + + +class Metadata(ServerPlugin): + """ ``bcfg2-lint`` plugin for :ref:`Metadata + <server-plugins-grouping-metadata>`. This checks for several things: + + * ``<Client>`` tags nested inside other ``<Client>`` tags; + * Deprecated options (like ``location="floating"``); + * Profiles that don't exist, or that aren't profile groups; + * Groups or clients that are defined multiple times; + * Multiple default groups or a default group that isn't a profile + group. + """ + __serverplugin__ = 'Metadata' + + def Run(self): + self.nested_clients() + self.deprecated_options() + self.bogus_profiles() + self.duplicate_groups() + self.duplicate_default_groups() + self.duplicate_clients() + self.default_is_profile() + + @classmethod + def Errors(cls): + return {"nested-client-tags": "warning", + "deprecated-clients-options": "warning", + "nonexistent-profile-group": "error", + "non-profile-set-as-profile": "error", + "duplicate-group": "error", + "duplicate-client": "error", + "multiple-default-groups": "error", + "default-is-not-profile": "error"} + + def deprecated_options(self): + """ Check for the ``location='floating'`` option, which has + been deprecated in favor of ``floating='true'``. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return + clientdata = self.metadata.clients_xml.xdata + for el in clientdata.xpath("//Client"): + loc = el.get("location") + if loc: + if loc == "floating": + floating = True + else: + floating = False + self.LintError("deprecated-clients-options", + "The location='%s' option is deprecated. " + "Please use floating='%s' instead:\n%s" % + (loc, floating, self.RenderXML(el))) + + def nested_clients(self): + """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, + which is either redundant or will never match. """ + groupdata = self.metadata.groups_xml.xdata + for el in groupdata.xpath("//Client//Client"): + self.LintError("nested-client-tags", + "Client %s nested within Client tag: %s" % + (el.get("name"), self.RenderXML(el))) + + def bogus_profiles(self): + """ Check for clients that have profiles that are either not + flagged as profile groups in ``groups.xml``, or don't exist. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return + for client in self.metadata.clients_xml.xdata.findall('.//Client'): + profile = client.get("profile") + if profile not in self.metadata.groups: + self.LintError("nonexistent-profile-group", + "%s has nonexistent profile group %s:\n%s" % + (client.get("name"), profile, + self.RenderXML(client))) + elif not self.metadata.groups[profile].is_profile: + self.LintError("non-profile-set-as-profile", + "%s is set as profile for %s, but %s is not a " + "profile group:\n%s" % + (profile, client.get("name"), profile, + self.RenderXML(client))) + + def duplicate_default_groups(self): + """ Check for multiple default groups. """ + defaults = [] + for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): + if grp.get("default", "false").lower() == "true": + defaults.append(self.RenderXML(grp)) + if len(defaults) > 1: + self.LintError("multiple-default-groups", + "Multiple default groups defined:\n%s" % + "\n".join(defaults)) + + def duplicate_clients(self): + """ Check for clients that are defined more than once. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return + self.duplicate_entries( + self.metadata.clients_xml.xdata.xpath("//Client"), + "client") + + def duplicate_groups(self): + """ Check for groups that are defined more than once. There + are two ways this can happen: + + 1. The group is listed twice with contradictory options. + 2. The group is listed with no options *first*, and then with + options later. + + In this context, 'first' refers to the order in which groups + are parsed; see the loop condition below and + _handle_groups_xml_event above for details. """ + groups = dict() + duplicates = dict() + for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): + grpname = grp.get("name") + if grpname in duplicates: + duplicates[grpname].append(grp) + elif len(grp.attrib) > 1: # group has options + if grpname in groups: + duplicates[grpname] = [grp, groups[grpname]] + else: + groups[grpname] = grp + else: # group has no options + groups[grpname] = grp + for grpname, grps in duplicates.items(): + self.LintError("duplicate-group", + "Group %s is defined multiple times:\n%s" % + (grpname, + "\n".join(self.RenderXML(g) for g in grps))) + + def duplicate_entries(self, allentries, etype): + """ Generic duplicate entry finder. + + :param allentries: A list of all entries to check for + duplicates. + :type allentries: list of lxml.etree._Element + :param etype: The entry type. This will be used to determine + the error name (``duplicate-<etype>``) and for + display to the end user. + :type etype: string + """ + entries = dict() + for el in allentries: + if el.get("name") in entries: + entries[el.get("name")].append(self.RenderXML(el)) + else: + entries[el.get("name")] = [self.RenderXML(el)] + for ename, els in entries.items(): + if len(els) > 1: + self.LintError("duplicate-%s" % etype, + "%s %s is defined multiple times:\n%s" % + (etype.title(), ename, "\n".join(els))) + + def default_is_profile(self): + """ Ensure that the default group is a profile group. """ + if (self.metadata.default and + not self.metadata.groups[self.metadata.default].is_profile): + xdata = \ + self.metadata.groups_xml.xdata.xpath("//Group[@name='%s']" % + self.metadata.default)[0] + self.LintError("default-is-not-profile", + "Default group is not a profile group:\n%s" % + self.RenderXML(xdata)) diff --git a/src/lib/Bcfg2/Server/Lint/Pkgmgr.py b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py new file mode 100644 index 000000000..eed6d4c19 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py @@ -0,0 +1,50 @@ +""" ``bcfg2-lint`` plugin for :ref:`Pkgmgr +<server-plugins-generators-pkgmgr>` """ + +import os +import glob +import lxml.etree +import Bcfg2.Options +from Bcfg2.Server.Lint import ServerlessPlugin + + +class Pkgmgr(ServerlessPlugin): + """ Find duplicate :ref:`Pkgmgr + <server-plugins-generators-pkgmgr>` entries with the same + priority. """ + __serverplugin__ = 'Pkgmgr' + + def Run(self): + pset = set() + for pfile in glob.glob(os.path.join(Bcfg2.Options.setup.repository, + 'Pkgmgr', '*.xml')): + if self.HandlesFile(pfile): + xdata = lxml.etree.parse(pfile).getroot() + # get priority, type, group + priority = xdata.get('priority') + ptype = xdata.get('type') + for pkg in xdata.xpath("//Package"): + if pkg.getparent().tag == 'Group': + grp = pkg.getparent().get('name') + if (type(grp) is not str and + grp.getparent().tag == 'Group'): + pgrp = grp.getparent().get('name') + else: + pgrp = 'none' + else: + grp = 'none' + pgrp = 'none' + ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) + # check if package is already listed with same + # priority, type, grp + if ptuple in pset: + self.LintError( + "duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) + else: + pset.add(ptuple) + + @classmethod + def Errors(cls): + return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index ce8b237b9..ebf4c4954 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -3,16 +3,10 @@ verified with an XML schema alone. """ import os import re -import lxml.etree import Bcfg2.Server.Lint import Bcfg2.Client.Tools.VCS from Bcfg2.Server.Plugins.Packages import Apt, Yum from Bcfg2.Client.Tools.POSIX.base import device_map -try: - from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile - HAS_GENSHI = True -except ImportError: - HAS_GENSHI = False # format verifying functions. TODO: These should be moved into XML @@ -162,7 +156,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): for source in self.core.plugins['Packages'].sources: if isinstance(source, Yum.YumSource): if (not source.pulp_id and not source.url and - not source.rawurl): + not source.rawurl): self.LintError( "required-attrs-missing", "A %s source must have either a url, rawurl, or " @@ -176,7 +170,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): (source.ptype, self.RenderXML(source.xsource))) if (not isinstance(source, Apt.AptSource) and - source.recommended): + source.recommended): self.LintError( "extra-attrs", "The recommended attribute is not supported on %s sources:" @@ -191,32 +185,34 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): for rules in self.core.plugins['Rules'].entries.values(): xdata = rules.pnode.data for path in xdata.xpath("//Path"): - self.check_entry(path, os.path.join(self.config['repo'], - rules.name)) + self.check_entry(path, + os.path.join(Bcfg2.Options.setup.repository, + rules.name)) def check_bundles(self): - """ Check bundles for BoundPath entries with missing + """ Check bundles for BoundPath and BoundPackage entries with missing attrs. """ if 'Bundler' not in self.core.plugins: return for bundle in self.core.plugins['Bundler'].entries.values(): - if (self.HandlesFile(bundle.name) and - (not HAS_GENSHI or - not isinstance(bundle, BundleTemplateFile))): - try: - xdata = lxml.etree.XML(bundle.data) - except (lxml.etree.XMLSyntaxError, AttributeError): - xdata = \ - lxml.etree.parse(bundle.template.filepath).getroot() - - for path in \ - xdata.xpath("//*[substring(name(), 1, 5) = 'Bound']"): + if self.HandlesFile(bundle.name) and bundle.template is None: + for path in bundle.xdata.xpath( + "//*[substring(name(), 1, 5) = 'Bound']"): self.check_entry(path, bundle.name) + # ensure that abstract Path tags have either name + # or glob specified + for path in bundle.xdata.xpath("//Path"): + if ('name' not in path.attrib and + 'glob' not in path.attrib): + self.LintError( + "required-attrs-missing", + "Path tags require either a 'name' or 'glob' " + "attribute: \n%s" % self.RenderXML(path)) # ensure that abstract Package tags have either name # or group specified - for package in xdata.xpath("//Package"): + for package in bundle.xdata.xpath("//Package"): if ('name' not in package.attrib and 'group' not in package.attrib): self.LintError( @@ -272,7 +268,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): fmt = required_attrs['__text__'] del required_attrs['__text__'] if (not entry.text and - not entry.get('empty', 'false').lower() == 'true'): + not entry.get('empty', 'false').lower() == 'true'): self.LintError("required-attrs-missing", "Text missing for %s %s in %s: %s" % (tag, name, filename, diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py index fca9d14a9..a437c1318 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py @@ -4,20 +4,24 @@ import os import stat import Bcfg2.Server.Lint from Bcfg2.Compat import any # pylint: disable=W0622 -from Bcfg2.Server.Plugin import DEFAULT_FILE_METADATA +from Bcfg2.Server.Plugin import default_path_metadata from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator import \ CfgEncryptedGenshiGenerator from Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator import \ CfgEncryptedCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator import \ + CfgEncryptedJinja2Generator class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin): """ Check for templated scripts or executables. """ - templates = [CfgGenshiGenerator, CfgCheetahGenerator, - CfgEncryptedGenshiGenerator, CfgEncryptedCheetahGenerator] + templates = [CfgGenshiGenerator, CfgCheetahGenerator, CfgJinja2Generator, + CfgEncryptedGenshiGenerator, CfgEncryptedCheetahGenerator, + CfgEncryptedJinja2Generator] extensions = [".pl", ".py", ".sh", ".rb"] def Run(self): @@ -58,10 +62,11 @@ class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin): # finally, check for executable permissions in info.xml for entry in entryset.entries.values(): if isinstance(entry, CfgInfoXML): - for pinfo in entry.infoxml.pnode.data.xpath("//FileInfo"): + for pinfo in entry.infoxml.xdata.xpath("//FileInfo/Info"): try: - mode = int(pinfo.get("mode", - DEFAULT_FILE_METADATA['mode']), 8) + mode = int( + pinfo.get("mode", + default_path_metadata()['mode']), 8) except ValueError: # LintError will be produced by RequiredAttrs plugin self.logger.warning("Non-octal mode: %s" % mode) diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py new file mode 100644 index 000000000..9d05516f1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -0,0 +1,97 @@ +""" ``bcfg2-lint`` plugin for :ref:`TemplateHelper +<server-plugins-connectors-templatehelper>` """ + +import sys +import imp +from Bcfg2.Server.Lint import ServerPlugin +from Bcfg2.Server.Plugins.TemplateHelper import HelperModule, MODULE_RE, \ + safe_module_name + + +class TemplateHelper(ServerPlugin): + """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper + <server-plugins-connectors-templatehelper>` modules are valid. + This can check for: + + * A TemplateHelper module that cannot be imported due to syntax or + other compile-time errors; + * A TemplateHelper module that does not have an ``__export__`` + attribute, or whose ``__export__`` is not a list; + * Bogus symbols listed in ``__export__``, including symbols that + don't exist, that are reserved, or that start with underscores. + """ + __serverplugin__ = 'TemplateHelper' + + def __init__(self, *args, **kwargs): + ServerPlugin.__init__(self, *args, **kwargs) + # we instantiate a dummy helper to discover which keywords and + # defaults are reserved + dummy = HelperModule("foo.py") + self.reserved_keywords = dir(dummy) + self.reserved_defaults = dummy.reserved_defaults + + def Run(self): + for helper in self.core.plugins['TemplateHelper'].entries.values(): + if self.HandlesFile(helper.name): + self.check_helper(helper.name) + + def check_helper(self, helper): + """ Check a single helper module. + + :param helper: The filename of the helper module + :type helper: string + """ + module_name = MODULE_RE.search(helper).group(1) + + try: + module = imp.load_source(safe_module_name(module_name), helper) + except: # pylint: disable=W0702 + err = sys.exc_info()[1] + self.LintError("templatehelper-import-error", + "Failed to import %s: %s" % + (helper, err)) + return + + if not hasattr(module, "__export__"): + self.LintError("templatehelper-no-export", + "%s has no __export__ list" % helper) + return + elif not isinstance(module.__export__, list): + self.LintError("templatehelper-nonlist-export", + "__export__ is not a list in %s" % helper) + return + + for sym in module.__export__: + if not hasattr(module, sym): + self.LintError("templatehelper-nonexistent-export", + "%s: exported symbol %s does not exist" % + (helper, sym)) + elif sym in self.reserved_keywords: + self.LintError("templatehelper-reserved-export", + "%s: exported symbol %s is reserved" % + (helper, sym)) + elif sym.startswith("_"): + self.LintError("templatehelper-underscore-export", + "%s: exported symbol %s starts with underscore" + % (helper, sym)) + if sym in getattr(module, "__default__", []): + self.LintError("templatehelper-export-and-default", + "%s: %s is listed in both __default__ and " + "__export__" % (helper, sym)) + + for sym in getattr(module, "__default__", []): + if sym in self.reserved_defaults: + self.LintError("templatehelper-reserved-default", + "%s: default symbol %s is reserved" % + (helper, sym)) + + @classmethod + def Errors(cls): + return {"templatehelper-import-error": "error", + "templatehelper-no-export": "error", + "templatehelper-nonlist-export": "error", + "templatehelper-nonexistent-export": "error", + "templatehelper-reserved-export": "error", + "templatehelper-reserved-default": "error", + "templatehelper-underscore-export": "warning", + "templatehelper-export-and-default": "warning"} diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 3efcc890d..cab5d248d 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -6,14 +6,21 @@ import sys import glob import fnmatch import lxml.etree -from subprocess import Popen, PIPE, STDOUT +import Bcfg2.Options import Bcfg2.Server.Lint +from Bcfg2.Utils import Executor class Validate(Bcfg2.Server.Lint.ServerlessPlugin): """ Ensure that all XML files in the Bcfg2 repository validate according to their respective schemas. """ + options = Bcfg2.Server.Lint.ServerlessPlugin.options + [ + Bcfg2.Options.PathOption( + "--schema", cf=("Validate", "schema"), + default="/usr/share/bcfg2/schemas", + help="The full path to the XML schema files")] + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) @@ -32,14 +39,14 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Cfg/**/pubkey.xml": "pubkey.xsd", "Cfg/**/authorizedkeys.xml": "authorizedkeys.xsd", "Cfg/**/authorized_keys.xml": "authorizedkeys.xsd", + "Cfg/**/sslcert.xml": "sslca-cert.xsd", + "Cfg/**/sslkey.xml": "sslca-key.xsd", "SSHbase/**/info.xml": "info.xsd", - "SSLCA/**/info.xml": "info.xsd", "TGenshi/**/info.xml": "info.xsd", "TCheetah/**/info.xml": "info.xsd", "Bundler/*.xml": "bundle.xsd", "Bundler/*.genshi": "bundle.xsd", "Pkgmgr/*.xml": "pkglist.xsd", - "Base/*.xml": "base.xsd", "Rules/*.xml": "rules.xsd", "Defaults/*.xml": "defaults.xsd", "etc/report-configuration.xml": "report-configuration.xsd", @@ -50,16 +57,14 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "AWSTags/config.xml": "awstags.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", - "SSLCA/**/cert.xml": "sslca-cert.xsd", - "SSLCA/**/key.xml": "sslca-key.xsd", "GroupLogic/groups.xml": "grouplogic.xsd" } self.filelists = {} self.get_filelists() + self.cmd = Executor() def Run(self): - schemadir = self.config['schema'] for path, schemaname in self.filesets.items(): try: @@ -69,7 +74,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if filelist: # avoid loading schemas for empty file lists - schemafile = os.path.join(schemadir, schemaname) + schemafile = os.path.join(Bcfg2.Options.setup.schema, + schemaname) schema = self._load_schema(schemafile) if schema: for filename in filelist: @@ -118,11 +124,10 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if self.files is None: cmd.append("--xinclude") cmd.append(filename) - lint = Popen(cmd, stdout=PIPE, stderr=STDOUT) + result = self.cmd.run(cmd) self.LintError("xml-failed-to-parse", - "%s fails to parse:\n%s" % (filename, - lint.communicate()[0])) - lint.wait() + "%s fails to parse:\n%s" % + (filename, result.stdout + result.stderr)) return False except IOError: self.LintError("xml-failed-to-read", @@ -184,14 +189,11 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if self.files is None: cmd.append("--xinclude") cmd.extend(["--noout", "--schema", schemafile, filename]) - lint = Popen(cmd, stdout=PIPE, stderr=STDOUT) - output = lint.communicate()[0] - # py3k fix - if not isinstance(output, str): - output = output.decode('utf-8') - if lint.wait(): + result = self.cmd.run(cmd) + if not result.success: self.LintError("xml-failed-to-verify", - "%s fails to verify:\n%s" % (filename, output)) + "%s fails to verify:\n%s" % + (filename, result.stdout + result.stderr)) return False return True @@ -208,8 +210,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): listfiles = lambda p: fnmatch.filter(self.files, os.path.join('*', p)) else: - listfiles = lambda p: glob.glob(os.path.join(self.config['repo'], - p)) + listfiles = lambda p: \ + glob.glob(os.path.join(Bcfg2.Options.setup.repository, p)) for path in self.filesets.keys(): if '/**/' in path: @@ -218,9 +220,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): else: # self.files is None fpath, fname = path.split('/**/') self.filelists[path] = [] - for root, _, files in \ - os.walk(os.path.join(self.config['repo'], - fpath)): + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, + fpath)): self.filelists[path].extend([os.path.join(root, f) for f in files if f == fname]) diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py index 1f55962eb..6383a3c99 100644 --- a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -52,8 +52,8 @@ class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): listfiles = lambda p: fnmatch.filter(self.files, os.path.join('*', p)) else: - listfiles = lambda p: glob.glob(os.path.join(self.config['repo'], - p)) + listfiles = lambda p: glob.glob( + os.path.join(Bcfg2.Options.setup.repository, p)) rv = [] for path in self.globs: @@ -62,9 +62,9 @@ class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): rv.extend(listfiles(path)) else: # self.files is None fpath, fname = path.split('/**/') - for root, _, files in \ - os.walk(os.path.join(self.config['repo'], - fpath)): + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, + fpath)): rv.extend([os.path.join(root, f) for f in files if f == fname]) else: diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 28644263f..526bdf159 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -2,16 +2,19 @@ import os import sys +import time +import copy +import fcntl +import struct +import termios import logging -from copy import copy import textwrap import lxml.etree -import fcntl -import termios -import struct +import Bcfg2.Options +import Bcfg2.Server.Core +import Bcfg2.Server.Plugins from Bcfg2.Compat import walk_packages - -plugins = [m[1] for m in walk_packages(path=__path__)] # pylint: disable=C0103 +from Bcfg2.Options import _debug def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -46,10 +49,15 @@ def get_termsize(): class Plugin(object): """ Base class for all bcfg2-lint plugins """ - def __init__(self, config, errorhandler=None, files=None): + #: Name of the matching server plugin or None if there is no + #: matching one. If this is None the lint plugin will only loaded + #: by default if the matching server plugin is enabled, too. + __serverplugin__ = None + + options = [Bcfg2.Options.Common.repository] + + def __init__(self, errorhandler=None, files=None): """ - :param config: A :mod:`Bcfg2.Options` setup dict - :type config: dict :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` that will be used to handle lint errors. If one is not provided, a new one will be @@ -63,9 +71,6 @@ class Plugin(object): #: The list of files that bcfg2-lint should be run against self.files = files - #: The Bcfg2.Options setup dict - self.config = config - self.logger = logging.getLogger('bcfg2-lint') if errorhandler is None: #: The error handler @@ -96,9 +101,10 @@ class Plugin(object): False otherwise. """ return (self.files is None or fname in self.files or - os.path.join(self.config['repo'], fname) in self.files or + os.path.join(Bcfg2.Options.setup.repository, + fname) in self.files or os.path.abspath(fname) in self.files or - os.path.abspath(os.path.join(self.config['repo'], + os.path.abspath(os.path.join(Bcfg2.Options.setup.repository, fname)) in self.files) def LintError(self, err, msg): @@ -125,7 +131,7 @@ class Plugin(object): """ xml = None if len(element) or element.text: - el = copy(element) + el = copy.copy(element) if el.text and not keep_text: el.text = '...' for child in el.iterchildren(): @@ -145,8 +151,8 @@ class ErrorHandler(object): def __init__(self, errors=None): """ - :param config: An initial dict of errors to register - :type config: dict + :param errors: An initial dict of errors to register + :type errors: dict """ #: The number of errors passed to this error handler self.errors = 0 @@ -267,12 +273,10 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 """ Base class for bcfg2-lint plugins that check things that require the running Bcfg2 server. """ - def __init__(self, core, config, errorhandler=None, files=None): + def __init__(self, core, errorhandler=None, files=None): """ :param core: The Bcfg2 server core :type core: Bcfg2.Server.Core.BaseCore - :param config: A :mod:`Bcfg2.Options` setup dict - :type config: dict :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` that will be used to handle lint errors. If one is not provided, a new one will be @@ -282,7 +286,7 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 the bcfg2-lint ``--stdin`` option.) :type files: list of strings """ - Plugin.__init__(self, config, errorhandler=errorhandler, files=files) + Plugin.__init__(self, errorhandler=errorhandler, files=files) #: The server core self.core = core @@ -290,3 +294,171 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 #: The metadata plugin self.metadata = self.core.metadata + + +class LintPluginAction(Bcfg2.Options.ComponentAction): + """ Option parser action to load lint plugins """ + bases = ['Bcfg2.Server.Lint'] + + +class LintPluginOption(Bcfg2.Options.Option): + """ Option class for the lint_plugins """ + + def early_parsing_hook(self, namespace): + """ + We want a usefull default for the enabled lint plugins. + Therfore we use all importable plugins, that either pertain + with enabled server plugins or that has no matching plugin. + """ + + plugins = [p.__name__ for p in namespace.plugins] + for loader, name, _is_pkg in walk_packages(path=__path__): + try: + module = loader.find_module(name).load_module(name) + plugin = getattr(module, name) + if plugin.__serverplugin__ is None or \ + plugin.__serverplugin__ in plugins: + _debug("Automatically adding lint plugin %s" % + plugin.__name__) + self.default.append(plugin.__name__) + except ImportError: + pass + + +class _EarlyOptions(object): + """ We need the server.plugins options in an early parsing hook + for determining the default value for the lint_plugins. So we + create a component that is parsed before the other options. """ + + parse_first = True + options = [Bcfg2.Options.Common.plugins] + + +class CLI(object): + """ The bcfg2-lint CLI """ + options = Bcfg2.Server.Core.Core.options + [ + Bcfg2.Options.PathOption( + '--lint-config', default='/etc/bcfg2-lint.conf', + action=Bcfg2.Options.ConfigFileAction, + help='Specify bcfg2-lint configuration file'), + LintPluginOption( + "--lint-plugins", cf=('lint', 'plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, action=LintPluginAction, + help='bcfg2-lint plugin list'), + Bcfg2.Options.BooleanOption( + '--list-errors', help='Show error handling'), + Bcfg2.Options.BooleanOption( + '--stdin', help='Operate on a list of files supplied on stdin'), + Bcfg2.Options.Option( + cf=("errors", '*'), dest="lint_errors", + help="How to handle bcfg2-lint errors")] + + def __init__(self): + parser = Bcfg2.Options.get_parser( + description="Manage a running Bcfg2 server", + components=[self, _EarlyOptions]) + parser.parse() + + self.logger = logging.getLogger(parser.prog) + + self.logger.debug("Running lint with plugins: %s" % + [p.__name__ + for p in Bcfg2.Options.setup.lint_plugins]) + + if Bcfg2.Options.setup.stdin: + self.files = [s.strip() for s in sys.stdin.readlines()] + else: + self.files = None + self.errorhandler = self.get_errorhandler() + self.serverlessplugins = [] + self.serverplugins = [] + for plugin in Bcfg2.Options.setup.lint_plugins: + if issubclass(plugin, ServerPlugin): + self.serverplugins.append(plugin) + else: + self.serverlessplugins.append(plugin) + + def run(self): + """ Run bcfg2-lint """ + if Bcfg2.Options.setup.list_errors: + for plugin in self.serverplugins + self.serverlessplugins: + self.errorhandler.RegisterErrors(getattr(plugin, 'Errors')()) + + print("%-35s %-35s" % ("Error name", "Handler")) + for err, handler in self.errorhandler.errortypes.items(): + print("%-35s %-35s" % (err, handler.__name__)) + return 0 + + if not self.serverplugins and not self.serverlessplugins: + self.logger.error("No lint plugins loaded!") + return 1 + + self.run_serverless_plugins() + + if self.serverplugins: + if self.errorhandler.errors: + # it would be swell if we could try to start the server + # even if there were errors with the serverless plugins, + # but since XML parsing errors occur in the FAM thread + # (not in the core server thread), there's no way we can + # start the server and try to catch exceptions -- + # bcfg2-lint isn't in the same stack as the exceptions. + # so we're forced to assume that a serverless plugin error + # will prevent the server from starting + print("Serverless plugins encountered errors, skipping server " + "plugins") + else: + self.run_server_plugins() + + if (self.errorhandler.errors or + self.errorhandler.warnings or + Bcfg2.Options.setup.verbose): + print("%d errors" % self.errorhandler.errors) + print("%d warnings" % self.errorhandler.warnings) + + if self.errorhandler.errors: + return 2 + elif self.errorhandler.warnings: + return 3 + else: + return 0 + + def get_errorhandler(self): + """ get a Bcfg2.Server.Lint.ErrorHandler object """ + return Bcfg2.Server.Lint.ErrorHandler( + errors=Bcfg2.Options.setup.lint_errors) + + def run_serverless_plugins(self): + """ Run serverless plugins """ + self.logger.debug("Running serverless plugins: %s" % + [p.__name__ for p in self.serverlessplugins]) + for plugin in self.serverlessplugins: + self.logger.debug(" Running %s" % plugin.__name__) + plugin(files=self.files, errorhandler=self.errorhandler).Run() + + def run_server_plugins(self): + """ run plugins that require a running server to run """ + core = Bcfg2.Server.Core.Core() + try: + core.load_plugins() + core.block_for_fam_events(handle_events=True) + self.logger.debug("Running server plugins: %s" % + [p.__name__ for p in self.serverplugins]) + for plugin in self.serverplugins: + self.logger.debug(" Running %s" % plugin.__name__) + plugin(core, + files=self.files, errorhandler=self.errorhandler).Run() + finally: + core.shutdown() + + def _run_plugin(self, plugin, args=None): + """ Run a single bcfg2-lint plugin """ + if args is None: + args = [] + start = time.time() + # python 2.5 doesn't support mixing *magic and keyword arguments + kwargs = dict(files=self.files, errorhandler=self.errorhandler) + rv = plugin(*args, **kwargs).Run() + self.logger.debug(" Ran %s in %0.2f seconds" % (plugin.__name__, + time.time() - start)) + return rv diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 2cb3adae3..724b34d8d 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -15,32 +15,16 @@ import time import threading import lxml.etree import multiprocessing +import Bcfg2.Options +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin from itertools import cycle -from Bcfg2.Cache import Cache -from Bcfg2.Compat import Empty, wraps -from Bcfg2.Server.Core import BaseCore, exposed -from Bcfg2.Server.BuiltinCore import Core as BuiltinCore +from Bcfg2.Compat import Queue, Empty, wraps +from Bcfg2.Server.Core import Core, exposed +from Bcfg2.Server.BuiltinCore import BuiltinCore from multiprocessing.connection import Listener, Client -class DispatchingCache(Cache, Bcfg2.Server.Plugin.Debuggable): - """ Implementation of :class:`Bcfg2.Cache.Cache` that propagates - cache expiration events to child nodes. """ - - #: The method to send over the pipe to expire the cache - method = "expire_metadata_cache" - - def __init__(self, *args, **kwargs): - self.rpc_q = kwargs.pop("queue") - Bcfg2.Server.Plugin.Debuggable.__init__(self) - Cache.__init__(self, *args, **kwargs) - - def expire(self, key=None): - self.rpc_q.publish(self.method, args=[key]) - Cache.expire(self, key=key) - - class RPCQueue(Bcfg2.Server.Plugin.Debuggable): """ An implementation of a :class:`multiprocessing.Queue` designed for several additional use patterns: @@ -148,7 +132,7 @@ class DualEvent(object): return self._threading_event.wait(timeout=timeout) -class ChildCore(BaseCore): +class ChildCore(Core): """ A child process for :class:`Bcfg2.MultiprocessingCore.Core`. This core builds configurations from a given :class:`multiprocessing.Pipe`. Note that this is a full-fledged @@ -167,12 +151,10 @@ class ChildCore(BaseCore): #: every ``poll_wait`` seconds. poll_wait = 3.0 - def __init__(self, name, setup, rpc_q, terminate): + def __init__(self, name, rpc_q, terminate): """ :param name: The name of this child :type name: string - :param setup: A Bcfg2 options dict - :type setup: Bcfg2.Options.OptionParser :param read_q: The queue the child will read from for RPC communications from the parent process. :type read_q: multiprocessing.Queue @@ -183,7 +165,7 @@ class ChildCore(BaseCore): themselves down. :type terminate: multiprocessing.Event """ - BaseCore.__init__(self, setup) + Core.__init__(self) #: The name of this child self.name = name @@ -197,7 +179,7 @@ class ChildCore(BaseCore): # override this setting so that the child doesn't try to write # the pidfile - self.setup['daemon'] = False + Bcfg2.Options.setup.daemon = False # ensure that the child doesn't start a perflog thread self.perflog_thread = None @@ -207,9 +189,6 @@ class ChildCore(BaseCore): def _run(self): return True - def _daemonize(self): - return True - def _dispatch(self, address, data): """ Method dispatcher used for commands received from the RPC queue. """ @@ -264,7 +243,7 @@ class ChildCore(BaseCore): self.shutdown() def shutdown(self): - BaseCore.shutdown(self) + Core.shutdown(self) self.logger.info("%s: Closing RPC command queue" % self.name) self.rpc_q.close() @@ -289,16 +268,9 @@ class ChildCore(BaseCore): return rmi @exposed - def expire_metadata_cache(self, client=None): - """ Expire the metadata cache for a client """ - self.metadata_cache.expire(client) - - @exposed - def RecvProbeData(self, address, _): - """ Expire the probe cache for a client """ - self.expire_caches_by_type(Bcfg2.Server.Plugin.Probing, - key=self.resolve_client(address, - metadata=False)[0]) + def expire_cache(self, *tags, **kwargs): + """ Expire cached data """ + Bcfg2.Server.Cache.expire(*tags, exact=kwargs.pop("exact", False)) @exposed def GetConfig(self, client): @@ -309,7 +281,7 @@ class ChildCore(BaseCore): return lxml.etree.tostring(self.BuildConfiguration(client)) -class Core(BuiltinCore): +class MultiprocessingCore(BuiltinCore): """ A multiprocessing core that delegates building the actual client configurations to :class:`Bcfg2.Server.MultiprocessingCore.ChildCore` objects. The @@ -317,14 +289,34 @@ class Core(BuiltinCore): :func:`GetConfig` are delegated to children. All other calls are handled by the parent process. """ + options = BuiltinCore.options + [ + Bcfg2.Options.Option( + '--children', dest="core_children", + cf=('server', 'children'), type=int, + default=multiprocessing.cpu_count(), + help='Spawn this number of children for the multiprocessing core')] + #: How long to wait for a child process to shut down cleanly #: before it is terminated. shutdown_timeout = 10.0 - def __init__(self, setup): - BuiltinCore.__init__(self, setup) - if setup['children'] is None: - setup['children'] = multiprocessing.cpu_count() + def __init__(self): + BuiltinCore.__init__(self) + + #: A dict of child name -> one end of the + #: :class:`multiprocessing.Pipe` object used to communicate + #: with that child. (The child is given the other end of the + #: Pipe.) + self.pipes = dict() + + #: A queue that keeps track of which children are available to + #: render a configuration. A child is popped from the queue + #: when it starts to render a config, then it's pushed back on + #: when it's done. This lets us use a blocking call to + #: :func:`Queue.Queue.get` when waiting for an available + #: child. + self.available_children = \ + Queue(maxsize=Bcfg2.Options.setup.core_children) #: The flag that indicates when to stop child threads and #: processes @@ -334,8 +326,6 @@ class Core(BuiltinCore): #: used to send or publish commands to children. self.rpc_q = RPCQueue() - self.metadata_cache = DispatchingCache(queue=self.rpc_q) - #: A list of children that will be cycled through self._all_children = [] @@ -343,13 +333,22 @@ class Core(BuiltinCore): #: to provide a round-robin distribution of render requests self.children = None + def __str__(self): + if hasattr(Bcfg2.Options.setup, "location"): + return "%s(%s; %s children)" % (self.__class__.__name__, + Bcfg2.Options.setup.location, + len(self._all_children)) + else: + return "%s(%s children)" % (self.__class__.__name__, + len(self._all_children)) + def _run(self): - for cnum in range(self.setup['children']): + for cnum in range(Bcfg2.Options.setup.core_children): name = "Child-%s" % cnum self.logger.debug("Starting child %s" % name) child_q = self.rpc_q.add_subscriber(name) - childcore = ChildCore(name, self.setup, child_q, self.terminate) + childcore = ChildCore(name, child_q, self.terminate) child = multiprocessing.Process(target=childcore.run, name=name) child.start() self.logger.debug("Child %s started with PID %s" % (name, @@ -358,6 +357,7 @@ class Core(BuiltinCore): self.logger.debug("Started %s children: %s" % (len(self._all_children), self._all_children)) self.children = cycle(self._all_children) + Bcfg2.Server.Cache.add_expire_hook(self.cache_dispatch) return BuiltinCore._run(self) def shutdown(self): @@ -430,16 +430,11 @@ class Core(BuiltinCore): def set_debug(self, address, debug): self.rpc_q.set_debug(debug) self.rpc_q.publish("set_debug", args=[address, debug]) - self.metadata_cache.set_debug(debug) return BuiltinCore.set_debug(self, address, debug) - @exposed - def RecvProbeData(self, address, probedata): - rv = BuiltinCore.RecvProbeData(self, address, probedata) - # we don't want the children to actually process probe data, - # so we don't send the data, just the fact that we got some. - self.rpc_q.publish("RecvProbeData", args=[address, None]) - return rv + def cache_dispatch(self, tags, exact, _): + """ Publish cache expiration events to child nodes. """ + self.rpc_q.publish("expire_cache", args=tags, kwargs=dict(exact=exact)) @exposed def GetConfig(self, address): diff --git a/src/lib/Bcfg2/Server/Plugin/__init__.py b/src/lib/Bcfg2/Server/Plugin/__init__.py index ed1282ba0..6599aa7a5 100644 --- a/src/lib/Bcfg2/Server/Plugin/__init__.py +++ b/src/lib/Bcfg2/Server/Plugin/__init__.py @@ -14,6 +14,7 @@ documentation it's not necessary to use the submodules. E.g., you can import os import sys +import Bcfg2.Options sys.path.append(os.path.dirname(__file__)) # pylint: disable=W0401 @@ -21,3 +22,33 @@ from Bcfg2.Server.Plugin.base import * from Bcfg2.Server.Plugin.interfaces import * from Bcfg2.Server.Plugin.helpers import * from Bcfg2.Server.Plugin.exceptions import * + + +class _OptionContainer(object): + """ Container for plugin options that are loaded at import time + """ + options = [ + Bcfg2.Options.Common.default_paranoid, + Bcfg2.Options.Option( + cf=('mdata', 'owner'), dest="default_owner", default='root', + help='Default Path owner'), + Bcfg2.Options.Option( + cf=('mdata', 'group'), dest="default_group", default='root', + help='Default Path group'), + Bcfg2.Options.Option( + cf=('mdata', 'important'), dest="default_important", + default='false', choices=['true', 'false'], + help='Default Path priority (importance)'), + Bcfg2.Options.Option( + cf=('mdata', 'mode'), dest="default_mode", default='644', + help='Default mode for Path'), + Bcfg2.Options.Option( + cf=('mdata', 'secontext'), dest="default_secontext", + default='__default__', help='Default SELinux context'), + Bcfg2.Options.Option( + cf=('mdata', 'sensitive'), dest="default_sensitive", + default='false', + help='Default Path sensitivity setting')] + + +Bcfg2.Options.get_parser().add_component(_OptionContainer) diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index 03feceb6f..549f7b543 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -1,66 +1,11 @@ """This module provides the base class for Bcfg2 server plugins.""" import os -import logging +import Bcfg2.Options +from Bcfg2.Logger import Debuggable from Bcfg2.Utils import ClassName -class Debuggable(object): - """ Mixin to add a debugging interface to an object and expose it - via XML-RPC on :class:`Bcfg2.Server.Plugin.base.Plugin` objects """ - - #: List of names of methods to be exposed as XML-RPC functions - __rmi__ = ['toggle_debug', 'set_debug'] - - #: How exposed XML-RPC functions should be dispatched to child - #: processes. - __child_rmi__ = __rmi__[:] - - def __init__(self, name=None): - """ - :param name: The name of the logger object to get. If none is - supplied, the full name of the class (including - module) will be used. - :type name: string - - .. autoattribute:: __rmi__ - """ - if name is None: - name = "%s.%s" % (self.__class__.__module__, - self.__class__.__name__) - self.debug_flag = False - self.logger = logging.getLogger(name) - - def set_debug(self, debug): - """ Explicitly enable or disable debugging. This method is exposed - via XML-RPC. - - :returns: bool - The new value of the debug flag - """ - self.debug_flag = debug - return debug - - def toggle_debug(self): - """ Turn debugging output on or off. This method is exposed - via XML-RPC. - - :returns: bool - The new value of the debug flag - """ - return self.set_debug(not self.debug_flag) - - def debug_log(self, message, flag=None): - """ Log a message at the debug level. - - :param message: The message to log - :type message: string - :param flag: Override the current debug flag with this value - :type flag: bool - :returns: None - """ - if (flag is None and self.debug_flag) or flag: - self.logger.error(message) - - class Plugin(Debuggable): """ The base class for all Bcfg2 Server plugins. """ @@ -109,13 +54,10 @@ class Plugin(Debuggable): #: but not ``__rmi__`` will be ignored. __child_rmi__ = Debuggable.__child_rmi__ - def __init__(self, core, datastore): + def __init__(self, core): """ :param core: The Bcfg2.Server.Core initializing the plugin :type core: Bcfg2.Server.Core - :param datastore: The path to the Bcfg2 repository on the - filesystem - :type datastore: string :raises: :exc:`OSError` if adding a file monitor failed; :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` on other errors @@ -125,7 +67,7 @@ class Plugin(Debuggable): Debuggable.__init__(self, name=self.name) self.Entries = {} self.core = core - self.data = os.path.join(datastore, self.name) + self.data = os.path.join(Bcfg2.Options.setup.repository, self.name) if self.create and not os.path.exists(self.data): self.logger.warning("%s: %s does not exist, creating" % (self.name, self.data)) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 55dd255cd..559612d1e 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -3,56 +3,89 @@ import os import re import sys -import copy import time +import copy import glob import logging +import genshi import operator import lxml.etree import Bcfg2.Server import Bcfg2.Options -import Bcfg2.Statistics +import Bcfg2.Server.FileMonitor +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import CmpMixin, wraps -from Bcfg2.Server.Plugin.base import Debuggable, Plugin -from Bcfg2.Server.Plugin.interfaces import Generator +from Bcfg2.Server.Plugin.base import Plugin +from Bcfg2.Server.Plugin.interfaces import Generator, TemplateDataProvider from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ PluginExecutionError, PluginInitError try: + import Bcfg2.Server.Encryption + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False + +try: import django # pylint: disable=W0611 HAS_DJANGO = True except ImportError: HAS_DJANGO = False -#: A dict containing default metadata for Path entries from bcfg2.conf -DEFAULT_FILE_METADATA = Bcfg2.Options.OptionParser( - dict(configfile=Bcfg2.Options.CFILE, - owner=Bcfg2.Options.MDATA_OWNER, - group=Bcfg2.Options.MDATA_GROUP, - mode=Bcfg2.Options.MDATA_MODE, - secontext=Bcfg2.Options.MDATA_SECONTEXT, - important=Bcfg2.Options.MDATA_IMPORTANT, - paranoid=Bcfg2.Options.MDATA_PARANOID, - sensitive=Bcfg2.Options.MDATA_SENSITIVE)) -DEFAULT_FILE_METADATA.parse([Bcfg2.Options.CFILE.cmd, Bcfg2.Options.CFILE]) -del DEFAULT_FILE_METADATA['args'] -del DEFAULT_FILE_METADATA['configfile'] - LOGGER = logging.getLogger(__name__) -#: a compiled regular expression for parsing info and :info files -INFO_REGEX = re.compile(r'owner:\s*(?P<owner>\S+)|' + - r'group:\s*(?P<group>\S+)|' + - r'mode:\s*(?P<mode>\w+)|' + - r'secontext:\s*(?P<secontext>\S+)|' + - r'paranoid:\s*(?P<paranoid>\S+)|' + - r'sensitive:\s*(?P<sensitive>\S+)|' + - r'encoding:\s*(?P<encoding>\S+)|' + - r'important:\s*(?P<important>\S+)|' + - r'mtime:\s*(?P<mtime>\w+)') + +class track_statistics(object): # pylint: disable=C0103 + """ Decorator that tracks execution time for the given + :class:`Plugin` method with :mod:`Bcfg2.Statistics` for reporting + via ``bcfg2-admin perf`` """ + + def __init__(self, name=None): + """ + :param name: The name under which statistics for this function + will be tracked. By default, the name will be + the name of the function concatenated with the + name of the class the function is a member of. + :type name: string + """ + # if this is None, it will be set later during __call_ + self.name = name + + def __call__(self, func): + if self.name is None: + self.name = func.__name__ + + @wraps(func) + def inner(obj, *args, **kwargs): + """ The decorated function """ + name = "%s:%s" % (obj.__class__.__name__, self.name) + + start = time.time() + try: + return func(obj, *args, **kwargs) + finally: + Bcfg2.Server.Statistics.stats.add_value(name, + time.time() - start) + + return inner + + +def removecomment(stream): + """ A Genshi filter that removes comments from the stream. This + function is a generator. + + :param stream: The Genshi stream to remove comments from + :type stream: genshi.core.Stream + :returns: tuple of ``(kind, data, pos)``, as when iterating + through a Genshi stream + """ + for kind, data, pos in stream: + if kind is genshi.core.COMMENT: + continue + yield kind, data, pos -def bind_info(entry, metadata, infoxml=None, default=DEFAULT_FILE_METADATA): +def bind_info(entry, metadata, infoxml=None, default=None): """ Bind the file metadata in the given :class:`Bcfg2.Server.Plugin.helpers.InfoXML` object to the given entry. @@ -69,6 +102,8 @@ def bind_info(entry, metadata, infoxml=None, default=DEFAULT_FILE_METADATA): :returns: None :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` """ + if default is None: + default = default_path_metadata() for attr, val in list(default.items()): entry.set(attr, val) if infoxml: @@ -82,81 +117,126 @@ def bind_info(entry, metadata, infoxml=None, default=DEFAULT_FILE_METADATA): entry.set(attr, val) -class track_statistics(object): # pylint: disable=C0103 - """ Decorator that tracks execution time for the given - :class:`Plugin` method with :mod:`Bcfg2.Statistics` for reporting - via ``bcfg2-admin perf`` """ +def default_path_metadata(): + """ Get the default Path entry metadata from the config. - def __init__(self, name=None): - """ - :param name: The name under which statistics for this function - will be tracked. By default, the name will be - the name of the function concatenated with the - name of the class the function is a member of. - :type name: string - """ - # if this is None, it will be set later during __call_ - self.name = name + :returns: dict of metadata attributes and their default values + """ + return dict([(k, getattr(Bcfg2.Options.setup, "default_%s" % k)) + for k in ['owner', 'group', 'mode', 'secontext', 'important', + 'paranoid', 'sensitive']]) + + +class DefaultTemplateDataProvider(TemplateDataProvider): + """ A base + :class:`Bcfg2.Server.Plugin.interfaces.TemplateDataProvider` that + provides default data for text and XML templates. + + Note that, since Cheetah and Genshi text templates treat the + ``path`` variable differently, this is overridden, by + :class:`Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.DefaultCheetahDataProvider` + and + :class:`Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.DefaultGenshiDataProvider`, + respectively. """ + + def get_template_data(self, entry, metadata, template): + return dict(name=entry.get('realname', entry.get('name')), + metadata=metadata, + source_path=template, + repo=Bcfg2.Options.setup.repository) + + def get_xml_template_data(self, _, metadata): + return dict(metadata=metadata, + repo=Bcfg2.Options.setup.repository) + +_sentinel = object() # pylint: disable=C0103 + + +def _get_template_data(func_name, args, default=_sentinel): + """ Generic template data getter for both text and XML templates. + + :param func_name: The name of the function to call on + :class:`Bcfg2.Server.Plugin.interfaces.TemplateDataProvider` + objects to get data for this template type. + Should be one of either ``get_template_data`` + for text templates, or ``get_xml_template_data`` + for XML templates. + :type func_name: string + :param args: The arguments to pass to the data retrieval function + :type args: list + :param default: An object that provides a set of base values. If + this is not provided, an instance of + :class:`Bcfg2.Server.Plugin.helpers.DefaultTemplateDataProvider` + is used. This can be set to None to avoid setting + any base values at all. + :type default: Bcfg2.Server.Plugin.interfaces.TemplateDataProvider + """ + if default is _sentinel: + default = DefaultTemplateDataProvider() + providers = Bcfg2.Server.core.plugins_by_type(TemplateDataProvider) + if default is not None: + providers.insert(0, default) + + rv = dict() + source = dict() + for prov in providers: + pdata = getattr(prov, func_name)(*args) + for key, val in pdata.items(): + if key not in rv: + rv[key] = val + source[key] = prov + else: + LOGGER.warning("Duplicate template variable %s provided by " + "both %s and %s" % (key, prov, source[key])) + return rv - def __call__(self, func): - if self.name is None: - self.name = func.__name__ - @wraps(func) - def inner(obj, *args, **kwargs): - """ The decorated function """ - name = "%s:%s" % (obj.__class__.__name__, self.name) +def get_template_data(entry, metadata, template, default=_sentinel): + """ Get all template variables for a text (i.e., Cfg) template """ + return _get_template_data("get_template_data", [entry, metadata, template], + default=default) - start = time.time() - try: - return func(obj, *args, **kwargs) - finally: - Bcfg2.Statistics.stats.add_value(name, time.time() - start) - return inner +def get_xml_template_data(structfile, metadata, default=_sentinel): + """ Get all template variables for an XML template """ + return _get_template_data("get_xml_template_data", [structfile, metadata], + default=default) class DatabaseBacked(Plugin): """ Provides capabilities for a plugin to read and write to a - database. + database. The plugin must add an option to flag database use with + something like: + + options = Bcfg2.Server.Plugin.Plugins.options + [ + Bcfg2.Options.BooleanOption( + cf=('metadata', 'use_database'), dest="metadata_db", + help="Use database capabilities of the Metadata plugin") + + This must be done manually due to various limitations in Python. .. private-include: _use_db .. private-include: _must_lock """ - #: The option to look up in :attr:`section` to determine whether or - #: not to use the database capabilities of this plugin. The option - #: is retrieved with - #: :py:func:`ConfigParser.SafeConfigParser.getboolean`, and so must - #: conform to the possible values that function can handle. - option = "use_database" - - def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - use_db = self.core.setup.cfp.getboolean(self.section, - self.option, - default=False) + def __init__(self, core): + Plugin.__init__(self, core) + use_db = getattr(Bcfg2.Options.setup, "%s_db" % self.name.lower(), + False) if use_db and not HAS_DJANGO: - raise PluginInitError("%s.%s is True but Django not found" % - (self.section, self.option)) + raise PluginInitError("%s is configured to use the database but " + "Django libraries are not found" % self.name) elif use_db and not self.core.database_available: - raise PluginInitError("%s.%s is True but the database is " - "unavailable due to prior errors" % - (self.section, self.option)) - - def _section(self): - """ The section to look in for :attr:`DatabaseBacked.option` - """ - return self.name.lower() - section = property(_section) + raise PluginInitError("%s is configured to use the database but " + "the database is unavailable due to prior " + "errors" % self.name) @property def _use_db(self): """ Whether or not this plugin is configured to use the database. """ - use_db = self.core.setup.cfp.getboolean(self.section, - self.option, - default=False) + use_db = getattr(Bcfg2.Options.setup, "%s_db" % self.name.lower(), + False) if use_db and HAS_DJANGO and self.core.database_available: return True else: @@ -167,11 +247,7 @@ class DatabaseBacked(Plugin): """ Whether or not the backend database must acquire a thread lock before writing, because it does not allow multiple threads to write.""" - engine = \ - self.core.setup.cfp.get(Bcfg2.Options.DB_ENGINE.cf[0], - Bcfg2.Options.DB_ENGINE.cf[1], - default=Bcfg2.Options.DB_ENGINE.default) - return engine == 'sqlite3' + return self._use_db and Bcfg2.Options.setup.db_engine == 'sqlite3' @staticmethod def get_db_lock(func): @@ -209,13 +285,10 @@ class FileBacked(Debuggable): principally meant to be used as a part of :class:`Bcfg2.Server.Plugin.helpers.DirectoryBacked`. """ - def __init__(self, name, fam=None): + def __init__(self, name): """ :param name: The full path to the file to cache and monitor :type name: string - :param fam: The FAM object used to receive notifications of - changes - :type fam: Bcfg2.Server.FileMonitor.FileMonitor """ Debuggable.__init__(self) @@ -226,7 +299,7 @@ class FileBacked(Debuggable): self.name = name #: The FAM object used to receive notifications of changes - self.fam = fam + self.fam = Bcfg2.Server.FileMonitor.get_fam() def HandleEvent(self, event=None): """ HandleEvent is called whenever the FAM registers an event. @@ -279,14 +352,11 @@ class DirectoryBacked(Debuggable): #: :attr:`patterns` or ``ignore``, then a warning will be produced. ignore = None - def __init__(self, data, fam): + def __init__(self, data): """ :param data: The path to the data directory that will be monitored :type data: string - :param fam: The FAM object used to receive notifications of - changes - :type fam: Bcfg2.Server.FileMonitor.FileMonitor .. ----- .. autoattribute:: __child__ @@ -294,7 +364,7 @@ class DirectoryBacked(Debuggable): Debuggable.__init__(self) self.data = os.path.normpath(data) - self.fam = fam + self.fam = Bcfg2.Server.FileMonitor.get_fam() #: self.entries contains information about the files monitored #: by this object. The keys of the dict are the relative @@ -366,8 +436,7 @@ class DirectoryBacked(Debuggable): :returns: None """ self.entries[relative] = self.__child__(os.path.join(self.data, - relative), - self.fam) + relative)) self.entries[relative].HandleEvent(event) def HandleEvent(self, event): # pylint: disable=R0912 @@ -492,13 +561,10 @@ class XMLFileBacked(FileBacked): #: to the constructor. create = None - def __init__(self, filename, fam=None, should_monitor=False, create=None): + def __init__(self, filename, should_monitor=False, create=None): """ :param filename: The full path to the file to cache and monitor :type filename: string - :param fam: The FAM object used to receive notifications of - changes - :type fam: Bcfg2.Server.FileMonitor.FileMonitor :param should_monitor: Whether or not to monitor this file for changes. It may be useful to disable monitoring when, for instance, the file @@ -518,7 +584,7 @@ class XMLFileBacked(FileBacked): .. ----- .. autoattribute:: __identifier__ """ - FileBacked.__init__(self, filename, fam=fam) + FileBacked.__init__(self, filename) #: The raw XML data contained in the file as an #: :class:`lxml.etree.ElementTree` object, with XIncludes @@ -542,7 +608,7 @@ class XMLFileBacked(FileBacked): self.extra_monitors = [] if ((create is not None or self.create not in [None, False]) and - not os.path.exists(self.name)): + not os.path.exists(self.name)): toptag = create or self.create self.logger.warning("%s does not exist, creating" % self.name) if hasattr(toptag, "getroottree"): @@ -554,7 +620,7 @@ class XMLFileBacked(FileBacked): #: Whether or not to monitor this file for changes. self.should_monitor = should_monitor - if fam and should_monitor: + if should_monitor: self.fam.AddMonitor(filename, self) def _follow_xincludes(self, fname=None, xdata=None): @@ -619,18 +685,14 @@ class XMLFileBacked(FileBacked): Index.__doc__ = FileBacked.Index.__doc__ def add_monitor(self, fpath): - """ Add a FAM monitor to a file that has been XIncluded. This - is only done if the constructor got a ``fam`` object, - regardless of whether ``should_monitor`` is set to True (i.e., - whether or not the base file is monitored). + """ Add a FAM monitor to a file that has been XIncluded. :param fpath: The full path to the file to monitor :type fpath: string :returns: None """ self.extra_monitors.append(fpath) - if self.fam: - self.fam.AddMonitor(fpath, self) + self.fam.AddMonitor(fpath, self) def __iter__(self): return iter(self.entries) @@ -642,44 +704,171 @@ class XMLFileBacked(FileBacked): class StructFile(XMLFileBacked): """ StructFiles are XML files that contain a set of structure file formatting logic for handling ``<Group>`` and ``<Client>`` - tags. """ + tags. + + .. ----- + .. autoattribute:: __identifier__ + .. automethod:: _include_element + """ #: If ``__identifier__`` is not None, then it must be the name of #: an XML attribute that will be required on the top-level tag of #: the file being cached __identifier__ = None - def _include_element(self, item, metadata): - """ determine if an XML element matches the metadata """ + #: Whether or not to enable encryption + encryption = True + + #: Callbacks used to determine if children of items with the given + #: tags should be included in the return value of + #: :func:`Bcfg2.Server.Plugin.helpers.StructFile.Match` and + #: :func:`Bcfg2.Server.Plugin.helpers.StructFile.XMLMatch`. Each + #: callback is passed the same arguments as + #: :func:`Bcfg2.Server.Plugin.helpers.StructFile._include_element`. + #: It should return True if children of the element should be + #: included in the match, False otherwise. The callback does + #: *not* need to consider negation; that will be handled in + #: :func:`Bcfg2.Server.Plugin.helpers.StructFile._include_element` + _include_tests = \ + dict(Group=lambda el, md, *args: el.get('name') in md.groups, + Client=lambda el, md, *args: el.get('name') == md.hostname) + + def __init__(self, filename, should_monitor=False, create=None): + XMLFileBacked.__init__(self, filename, should_monitor=should_monitor, + create=create) + self.template = None + + def Index(self): + XMLFileBacked.Index(self) + if (self.name.endswith('.genshi') or + ('py' in self.xdata.nsmap and + self.xdata.nsmap['py'] == 'http://genshi.edgewall.org/')): + try: + loader = genshi.template.TemplateLoader() + self.template = \ + loader.load(self.name, + cls=genshi.template.MarkupTemplate, + encoding=Bcfg2.Options.setup.encoding) + except LookupError: + err = sys.exc_info()[1] + self.logger.error('Genshi lookup error in %s: %s' % (self.name, + err)) + except genshi.template.TemplateError: + err = sys.exc_info()[1] + self.logger.error('Genshi template error in %s: %s' % + (self.name, err)) + except genshi.input.ParseError: + err = sys.exc_info()[1] + self.logger.error('Genshi parse error in %s: %s' % (self.name, + err)) + + if HAS_CRYPTO and self.encryption: + for el in self.xdata.xpath("//*[@encrypted]"): + try: + el.text = self._decrypt(el).encode('ascii', + 'xmlcharrefreplace') + except UnicodeDecodeError: + self.logger.info("%s: Decrypted %s to gibberish, skipping" + % (self.name, el.tag)) + except Bcfg2.Server.Encryption.EVPError: + lax_decrypt = self.xdata.get( + "lax_decryption", + str(Bcfg2.Options.setup.lax_decryption)).lower() == \ + "true" + msg = "Failed to decrypt %s element in %s" % (el.tag, + self.name) + if lax_decrypt: + self.logger.debug(msg) + else: + raise PluginExecutionError(msg) + Index.__doc__ = XMLFileBacked.Index.__doc__ + + def _decrypt(self, element): + """ Decrypt a single encrypted properties file element """ + if not element.text or not element.text.strip(): + return + passes = Bcfg2.Options.setup.passphrases + try: + passphrase = passes[element.get("encrypted")] + return Bcfg2.Server.Encryption.ssl_decrypt(element.text, + passphrase) + except KeyError: + raise Bcfg2.Server.Encryption.EVPError("No passphrase named '%s'" % + element.get("encrypted")) + raise Bcfg2.Server.Encryption.EVPError("Failed to decrypt") + + def _include_element(self, item, metadata, *args): + """ Determine if an XML element matches the other arguments. + + The first argument is always the XML element to match, and the + second will always be a single + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` object + representing the metadata to match against. Subsequent + arguments are as given to + :func:`Bcfg2.Server.Plugin.helpers.StructFile.Match` or + :func:`Bcfg2.Server.Plugin.helpers.StructFile.XMLMatch`. In + the base StructFile implementation, there are no additional + arguments; in classes that inherit from StructFile, see the + :func:`Match` and :func:`XMLMatch` method signatures.""" if isinstance(item, lxml.etree._Comment): # pylint: disable=W0212 return False - negate = item.get('negate', 'false').lower() == 'true' - if item.tag == 'Group': - return negate == (item.get('name') not in metadata.groups) - elif item.tag == 'Client': - return negate == (item.get('name') != metadata.hostname) + if item.tag in self._include_tests: + negate = item.get('negate', 'false').lower() == 'true' + return negate != self._include_tests[item.tag](item, metadata, + *args) else: return True - def _match(self, item, metadata): - """ recursive helper for Match() """ - if self._include_element(item, metadata): - if item.tag == 'Group' or item.tag == 'Client': + def _render(self, metadata): + """ Render the template for the given client metadata + + :param metadata: Client metadata to match against. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: lxml.etree._Element object representing the rendered + XML data + """ + stream = self.template.generate( + **get_xml_template_data(self, metadata)).filter(removecomment) + return lxml.etree.XML(stream.render('xml', + strip_whitespace=False).encode(), + parser=Bcfg2.Server.XMLParser) + + def _match(self, item, metadata, *args): + """ recursive helper for + :func:`Bcfg2.Server.Plugin.helpers.StructFile.Match` """ + if self._include_element(item, metadata, *args): + if item.tag in self._include_tests.keys(): rv = [] - if self._include_element(item, metadata): + if self._include_element(item, metadata, *args): for child in item.iterchildren(): - rv.extend(self._match(child, metadata)) + rv.extend(self._match(child, metadata, *args)) return rv else: rv = copy.deepcopy(item) for child in rv.iterchildren(): rv.remove(child) for child in item.iterchildren(): - rv.extend(self._match(child, metadata)) + rv.extend(self._match(child, metadata, *args)) return [rv] else: return [] + def _do_match(self, metadata, *args): + """ Helper for + :func:`Bcfg2.Server.Plugin.helpers.StructFile.Match` that lets + a subclass of StructFile easily redefine the public Match() + interface to accept a different number of arguments. This + provides a sane prototype for the Match() function while + keeping the internals consistent. """ + rv = [] + if self.template is None: + entries = self.entries + else: + entries = self._render(metadata).getchildren() + for child in entries: + rv.extend(self._match(child, metadata, *args)) + return rv + def Match(self, metadata): """ Return matching fragments of the data in this file. A tag is considered to match if all ``<Group>`` and ``<Client>`` @@ -690,22 +879,22 @@ class StructFile(XMLFileBacked): Match() (and *not* their descendents) should be considered to match the metadata. + Match() returns matching fragments in document order. + :param metadata: Client metadata to match against. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: list of lxml.etree._Element objects """ - rv = [] - for child in self.entries: - rv.extend(self._match(child, metadata)) - return rv + return self._do_match(metadata) - def _xml_match(self, item, metadata): - """ recursive helper for XMLMatch """ - if self._include_element(item, metadata): - if item.tag == 'Group' or item.tag == 'Client': + def _xml_match(self, item, metadata, *args): + """ recursive helper for + :func:`Bcfg2.Server.Plugin.helpers.StructFile.XMLMatch` """ + if self._include_element(item, metadata, *args): + if item.tag in self._include_tests.keys(): for child in item.iterchildren(): item.remove(child) item.getparent().append(child) - self._xml_match(child, metadata) + self._xml_match(child, metadata, *args) if item.text: if item.getparent().text is None: item.getparent().text = item.text @@ -714,10 +903,25 @@ class StructFile(XMLFileBacked): item.getparent().remove(item) else: for child in item.iterchildren(): - self._xml_match(child, metadata) + self._xml_match(child, metadata, *args) else: item.getparent().remove(item) + def _do_xmlmatch(self, metadata, *args): + """ Helper for + :func:`Bcfg2.Server.Plugin.helpers.StructFile.XMLMatch` that lets + a subclass of StructFile easily redefine the public Match() + interface to accept a different number of arguments. This + provides a sane prototype for the Match() function while + keeping the internals consistent. """ + if self.template is None: + rv = copy.deepcopy(self.xdata) + else: + rv = self._render(metadata) + for child in rv.iterchildren(): + self._xml_match(child, metadata, *args) + return rv + def XMLMatch(self, metadata): """ Return a rebuilt XML document that only contains the matching portions of the original file. A tag is considered @@ -727,169 +931,58 @@ class StructFile(XMLFileBacked): All ``<Group>`` and ``<Client>`` tags will have been stripped out. + The new document produced by XMLMatch() is not necessarily in + the same order as the original document. + :param metadata: Client metadata to match against. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: lxml.etree._Element """ - rv = copy.deepcopy(self.xdata) - for child in rv.iterchildren(): - self._xml_match(child, metadata) - return rv + return self._do_xmlmatch(metadata) -class INode(object): - """ INodes provide lists of things available at a particular group - intersection. INodes are deprecated; new plugins should use - :class:`Bcfg2.Server.Plugin.helpers.StructFile` instead. """ - - raw = dict( - Client="lambda m, e:'%(name)s' == m.hostname and predicate(m, e)", - Group="lambda m, e:'%(name)s' in m.groups and predicate(m, e)") - nraw = dict( - Client="lambda m, e:'%(name)s' != m.hostname and predicate(m, e)", - Group="lambda m, e:'%(name)s' not in m.groups and predicate(m, e)") - containers = ['Group', 'Client'] - ignore = [] - - def __init__(self, data, idict, parent=None): - self.data = data - self.contents = {} - if parent is None: - self.predicate = lambda m, e: True - else: - predicate = parent.predicate - if data.get('negate', 'false').lower() == 'true': - psrc = self.nraw - else: - psrc = self.raw - if data.tag in list(psrc.keys()): - self.predicate = eval(psrc[data.tag] % - {'name': data.get('name')}, - {'predicate': predicate}) - else: - raise PluginExecutionError("Unknown tag: %s" % data.tag) - self.children = [] - self._load_children(data, idict) +class InfoXML(StructFile): + """ InfoXML files contain Group, Client, and Path tags to set the + metadata (permissions, owner, etc.) of files. """ + encryption = False - def _load_children(self, data, idict): - """ load children """ - for item in data.getchildren(): - if item.tag in self.ignore: - continue - elif item.tag in self.containers: - self.children.append(self.__class__(item, idict, self)) - else: - try: - self.contents[item.tag][item.get('name')] = \ - dict(item.attrib) - except KeyError: - self.contents[item.tag] = \ - {item.get('name'): dict(item.attrib)} - if item.text: - self.contents[item.tag][item.get('name')]['__text__'] = \ - item.text - if item.getchildren(): - self.contents[item.tag][item.get('name')]['__children__'] \ - = item.getchildren() - try: - idict[item.tag].append(item.get('name')) - except KeyError: - idict[item.tag] = [item.get('name')] - - def Match(self, metadata, data, entry=lxml.etree.Element("None")): - """Return a dictionary of package mappings.""" - if self.predicate(metadata, entry): - for key in self.contents: - try: - data[key].update(self.contents[key]) - except: # pylint: disable=W0702 - data[key] = {} - data[key].update(self.contents[key]) - for child in self.children: - child.Match(metadata, data, entry=entry) - - -class InfoNode (INode): - """ :class:`Bcfg2.Server.Plugin.helpers.INode` implementation that - includes ``<Path>`` tags, suitable for use with :file:`info.xml` - files.""" - - raw = dict( - Client="lambda m, e: '%(name)s' == m.hostname and predicate(m, e)", - Group="lambda m, e: '%(name)s' in m.groups and predicate(m, e)", - Path="lambda m, e: ('%(name)s' == e.get('name') or " + - "'%(name)s' == e.get('realname')) and " + - "predicate(m, e)") - nraw = dict( - Client="lambda m, e: '%(name)s' != m.hostname and predicate(m, e)", - Group="lambda m, e: '%(name)s' not in m.groups and predicate(m, e)", - Path="lambda m, e: '%(name)s' != e.get('name') and " + - "'%(name)s' != e.get('realname') and " + - "predicate(m, e)") - containers = ['Group', 'Client', 'Path'] - - -class XMLSrc(XMLFileBacked): - """ XMLSrc files contain a - :class:`Bcfg2.Server.Plugin.helpers.INode` hierarchy that returns - matching entries. XMLSrc objects are deprecated and - :class:`Bcfg2.Server.Plugin.helpers.StructFile` should be - preferred where possible.""" - __node__ = INode - __cacheobj__ = dict - __priority_required__ = True - - def __init__(self, filename, fam=None, should_monitor=False, create=None): - XMLFileBacked.__init__(self, filename, fam, should_monitor, create) - self.items = {} - self.cache = None - self.pnode = None - self.priority = -1 + _include_tests = copy.copy(StructFile._include_tests) + _include_tests['Path'] = lambda el, md, entry, *args: \ + entry.get('realname', entry.get('name')) == el.get("name") - def HandleEvent(self, _=None): - """Read file upon update.""" - self.items = {} - try: - xdata = lxml.etree.parse(self.name, - parser=Bcfg2.Server.XMLParser).getroot() - except lxml.etree.XMLSyntaxError: - msg = "Failed to parse file %s: %s" % (self.name, - sys.exc_info()[1]) - self.logger.error(msg) - raise PluginExecutionError(msg) - self.pnode = self.__node__(xdata, self.items) - self.cache = None - try: - self.priority = int(xdata.get('priority')) - except (ValueError, TypeError): - if self.__priority_required__: - msg = "Got bogus priority %s for file %s" % \ - (xdata.get('priority'), self.name) - self.logger.error(msg) - raise PluginExecutionError(msg) + def Match(self, metadata, entry): # pylint: disable=W0221 + """ Implementation of + :func:`Bcfg2.Server.Plugin.helpers.StructFile.Match` that + considers Path tags to allow ``info.xml`` files to set + different file metadata for different file paths. """ + return self._do_match(metadata, entry) - def Cache(self, metadata): - """Build a package dict for a given host.""" - if self.cache is None or self.cache[0] != metadata: - cache = (metadata, self.__cacheobj__()) - if self.pnode is None: - self.logger.error("Cache method called early for %s; " - "forcing data load" % self.name) - self.HandleEvent() - return - self.pnode.Match(metadata, cache[1]) - self.cache = cache - - def __str__(self): - return str(self.items) + def XMLMatch(self, metadata, entry): # pylint: disable=W0221 + """ Implementation of + :func:`Bcfg2.Server.Plugin.helpers.StructFile.XMLMatch` that + considers Path tags to allow ``info.xml`` files to set + different file metadata for different file paths. """ + return self._do_xmlmatch(metadata, entry) + def BindEntry(self, entry, metadata): + """ Bind the matching file metadata for this client and entry + to the entry. -class InfoXML(XMLSrc): - """ InfoXML files contain a - :class:`Bcfg2.Server.Plugin.helpers.InfoNode` hierarchy that - returns matching entries, suitable for use with :file:`info.xml` - files.""" - __node__ = InfoNode - __priority_required__ = False + :param entry: The abstract entry to bind the info to. This + will be modified in place + :type entry: lxml.etree._Element + :param metadata: The client metadata to get info for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: None + """ + fileinfo = self.Match(metadata, entry) + if len(fileinfo) == 0: + raise PluginExecutionError("No metadata found in %s for %s" % + (self.name, entry.get('name'))) + elif len(fileinfo) > 1: + self.logger.warning("Multiple file metadata found in %s for %s" % + (self.name, entry.get('name'))) + for attr, val in fileinfo[0].attrib.items(): + entry.set(attr, val) class XMLDirectoryBacked(DirectoryBacked): @@ -905,6 +998,25 @@ class XMLDirectoryBacked(DirectoryBacked): __child__ = XMLFileBacked +class PriorityStructFile(StructFile): + """ A StructFile where each file has a priority, given as a + top-level XML attribute. """ + + def __init__(self, filename, should_monitor=False): + StructFile.__init__(self, filename, should_monitor=should_monitor) + self.priority = -1 + __init__.__doc__ = StructFile.__init__.__doc__ + + def Index(self): + StructFile.Index(self) + try: + self.priority = int(self.xdata.get('priority')) + except (ValueError, TypeError): + raise PluginExecutionError("Got bogus priority %s for file %s" % + (self.xdata.get('priority'), self.name)) + Index.__doc__ = StructFile.Index.__doc__ + + class PrioDir(Plugin, Generator, XMLDirectoryBacked): """ PrioDir handles a directory of XML files where each file has a set priority. @@ -915,42 +1027,43 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): #: The type of child objects to create for files contained within #: the directory that is tracked. Default is - #: :class:`Bcfg2.Server.Plugin.helpers.XMLSrc` - __child__ = XMLSrc + #: :class:`Bcfg2.Server.Plugin.helpers.PriorityStructFile` + __child__ = PriorityStructFile - def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) + def __init__(self, core): + Plugin.__init__(self, core) Generator.__init__(self) - XMLDirectoryBacked.__init__(self, self.data, self.core.fam) + XMLDirectoryBacked.__init__(self, self.data) __init__.__doc__ = Plugin.__init__.__doc__ def HandleEvent(self, event): XMLDirectoryBacked.HandleEvent(self, event) self.Entries = {} - for src in list(self.entries.values()): - for itype, children in list(src.items.items()): - for child in children: - try: - self.Entries[itype][child] = self.BindEntry - except KeyError: - self.Entries[itype] = {child: self.BindEntry} + for src in self.entries.values(): + for child in src.xdata.iterchildren(): + if child.tag in ['Group', 'Client']: + continue + if child.tag not in self.Entries: + self.Entries[child.tag] = dict() + self.Entries[child.tag][child.get("name")] = self.BindEntry HandleEvent.__doc__ = XMLDirectoryBacked.HandleEvent.__doc__ - def _matches(self, entry, metadata, rules): # pylint: disable=W0613 - """ Whether or not a given entry has a matching entry in this - PrioDir. By default this does strict matching (i.e., the - entry name is in ``rules.keys()``), but this can be overridden - to provide regex matching, etc. + def _matches(self, entry, metadata, candidate): # pylint: disable=W0613 + """ Whether or not a given candidate matches the abstract + entry given. By default this does strict matching (i.e., the + entry name matches the candidate name), but this can be + overridden to provide regex matching, etc. :param entry: The entry to find a match for :type entry: lxml.etree._Element :param metadata: The metadata to get attributes for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :rules: A dict of rules to look in for a matching rule - :type rules: dict + :candidate: A candidate concrete entry to match with + :type candidate: lxml.etree._Element :returns: bool """ - return entry.get('name') in rules + return (entry.tag == candidate.tag and + entry.get('name') == candidate.get('name')) def BindEntry(self, entry, metadata): """ Bind the attributes that apply to an entry to it. The @@ -962,71 +1075,40 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: None """ - attrs = self.get_attrs(entry, metadata) - for key, val in list(attrs.items()): - entry.attrib[key] = val - - def get_attrs(self, entry, metadata): - """ Get a list of attributes to add to the entry during the - bind. This is a complex method, in that it both modifies the - entry, and returns attributes that need to be added to the - entry. That seems sub-optimal, and should probably be changed - at some point. Namely: - - * The return value includes all XML attributes that need to be - added to the entry, but it does not add them. - * If text contents or child tags need to be added to the - entry, they are added to the entry in place. - - :param entry: The entry to add attributes to. - :type entry: lxml.etree._Element - :param metadata: The metadata to get attributes for - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :returns: dict of <attr name>:<attr value> - :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` - """ + matching = [] for src in self.entries.values(): - src.Cache(metadata) - - matching = [src for src in list(self.entries.values()) - if (src.cache and - entry.tag in src.cache[1] and - self._matches(entry, metadata, - src.cache[1][entry.tag]))] + for candidate in src.XMLMatch(metadata).xpath("//%s" % entry.tag): + if self._matches(entry, metadata, candidate): + matching.append((src, candidate)) if len(matching) == 0: raise PluginExecutionError("No matching source for entry when " - "retrieving attributes for %s(%s)" % - (entry.tag, entry.attrib.get('name'))) + "retrieving attributes for %s:%s" % + (entry.tag, entry.get('name'))) elif len(matching) == 1: - index = 0 + data = matching[0][1] else: - prio = [int(src.priority) for src in matching] - if prio.count(max(prio)) > 1: - msg = "Found conflicting sources with same priority for " + \ - "%s:%s for %s" % (entry.tag, entry.get("name"), - metadata.hostname) + prio = [int(m[0].priority) for m in matching] + priority = max(prio) + if prio.count(priority) > 1: + msg = "Found conflicting sources with same priority (%s) " \ + "for %s:%s for %s" % (priority, entry.tag, + entry.get("name"), metadata.hostname) self.logger.error(msg) - self.logger.error([item.name for item in matching]) - self.logger.error("Priority was %s" % max(prio)) + self.logger.error([m[0].name for m in matching]) raise PluginExecutionError(msg) - index = prio.index(max(prio)) - for rname in list(matching[index].cache[1][entry.tag].keys()): - if self._matches(entry, metadata, [rname]): - data = matching[index].cache[1][entry.tag][rname] - break - else: - # Fall back on __getitem__. Required if override used - data = matching[index].cache[1][entry.tag][entry.get('name')] - if '__text__' in data: - entry.text = data['__text__'] - if '__children__' in data: - for item in data['__children__']: - entry.append(copy.copy(item)) + for src, candidate in matching: + if int(src.priority) == priority: + data = candidate + break + + entry.text = data.text + for item in data.getchildren(): + entry.append(copy.copy(item)) - return dict([(key, data[key]) - for key in list(data.keys()) - if not key.startswith('__')]) + for key, val in list(data.attrib.items()): + if key not in entry.attrib: + entry.attrib[key] = val class Specificity(CmpMixin): @@ -1115,11 +1197,11 @@ class Specificity(CmpMixin): return "".join(rv) -class SpecificData(object): +class SpecificData(Debuggable): """ A file that is specific to certain clients, groups, or all clients. """ - def __init__(self, name, specific, encoding): # pylint: disable=W0613 + def __init__(self, name, specific): # pylint: disable=W0613 """ :param name: The full path to the file :type name: string @@ -1128,9 +1210,8 @@ class SpecificData(object): object describing what clients this file applies to. :type specific: Bcfg2.Server.Plugin.helpers.Specificity - :param encoding: The encoding to use for data in this file - :type encoding: string """ + Debuggable.__init__(self) self.name = name self.specific = specific self.data = None @@ -1152,8 +1233,7 @@ class SpecificData(object): except UnicodeDecodeError: self.data = open(self.name, mode='rb').read() except: # pylint: disable=W0201 - LOGGER.error("Failed to read file %s: %s" % (self.name, - sys.exc_info()[1])) + self.logger.error("Failed to read file %s" % self.name) class EntrySet(Debuggable): @@ -1177,7 +1257,7 @@ class EntrySet(Debuggable): #: considered a plain string and filenames must match exactly. basename_is_regex = False - def __init__(self, basename, path, entry_type, encoding): + def __init__(self, basename, path, entry_type): """ :param basename: The filename or regular expression that files in this EntrySet must match. See @@ -1192,12 +1272,10 @@ class EntrySet(Debuggable): be an object factory or similar callable. See below for the expected signature. :type entry_type: callable - :param encoding: The encoding of all files in this entry set. - :type encoding: string The ``entry_type`` callable must have the following signature:: - entry_type(filepath, specificity, encoding) + entry_type(filepath, specificity) Where the parameters are: @@ -1208,8 +1286,6 @@ class EntrySet(Debuggable): object describing what clients this file applies to. :type specific: Bcfg2.Server.Plugin.helpers.Specificity - :param encoding: The encoding to use for data in this file - :type encoding: string Additionally, the object returned by ``entry_type`` must have a ``specific`` attribute that is sortable (e.g., a @@ -1222,9 +1298,8 @@ class EntrySet(Debuggable): self.path = path self.entry_type = entry_type self.entries = {} - self.metadata = DEFAULT_FILE_METADATA.copy() + self.metadata = default_path_metadata() self.infoxml = None - self.encoding = encoding if self.basename_is_regex: base_pat = basename @@ -1241,6 +1316,12 @@ class EntrySet(Debuggable): #: be overridden on a per-entry basis in :func:`entry_init`. self.specific = re.compile(pattern) + def set_debug(self, debug): + rv = Debuggable.set_debug(self, debug) + for entry in self.entries.values(): + entry.set_debug(debug) + return rv + def get_matching(self, metadata): """ Get a list of all entries that apply to the given client. This gets all matching entries; for example, there could be an @@ -1299,7 +1380,7 @@ class EntrySet(Debuggable): """ action = event.code2str() - if event.filename in ['info', 'info.xml', ':info']: + if event.filename == 'info.xml': if action in ['exists', 'created', 'changed']: self.update_metadata(event) elif action == 'deleted': @@ -1358,8 +1439,7 @@ class EntrySet(Debuggable): self.logger.error("Could not process filename %s; ignoring" % fpath) return - self.entries[event.filename] = entry_type(fpath, spec, - self.encoding) + self.entries[event.filename] = entry_type(fpath, spec) self.entries[event.filename].handle_event(event) def specificity_from_filename(self, fname, specific=None): @@ -1404,8 +1484,8 @@ class EntrySet(Debuggable): return Specificity(**kwargs) def update_metadata(self, event): - """ Process changes to or creation of info, :info, and - info.xml files for the EntrySet. + """ Process changes to or creation of info.xml files for the + EntrySet. :param event: An event that applies to an info handled by this EntrySet @@ -1417,24 +1497,9 @@ class EntrySet(Debuggable): if not self.infoxml: self.infoxml = InfoXML(fpath) self.infoxml.HandleEvent(event) - elif event.filename in [':info', 'info']: - for line in open(fpath).readlines(): - match = INFO_REGEX.match(line) - if not match: - self.logger.warning("Failed to match line in %s: %s" % - (fpath, line)) - continue - else: - mgd = match.groupdict() - for key, value in list(mgd.items()): - if value: - self.metadata[key] = value - if len(self.metadata['mode']) == 3: - self.metadata['mode'] = "0%s" % self.metadata['mode'] def reset_metadata(self, event): - """ Reset metadata to defaults if info. :info, or info.xml are - removed. + """ Reset metadata to defaults if info.xml is removed. :param event: An event that applies to an info handled by this EntrySet @@ -1443,12 +1508,10 @@ class EntrySet(Debuggable): """ if event.filename == 'info.xml': self.infoxml = None - elif event.filename in [':info', 'info']: - self.metadata = DEFAULT_FILE_METADATA.copy() def bind_info_to_entry(self, entry, metadata): - """ Shortcut to call :func:`bind_info` with the base - info/info.xml for this EntrySet. + """ Bind the metadata for the given client in the base + info.xml for this EntrySet to the entry. :param entry: The abstract entry to bind the info to. This will be modified in place @@ -1457,7 +1520,10 @@ class EntrySet(Debuggable): :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: None """ - bind_info(entry, metadata, infoxml=self.infoxml, default=self.metadata) + for attr, val in list(self.metadata.items()): + entry.set(attr, val) + if self.infoxml is not None: + self.infoxml.BindEntry(entry, metadata) def bind_entry(self, entry, metadata): """ Return the single best fully-bound entry from the set of @@ -1503,10 +1569,12 @@ class GroupSpool(Plugin, Generator): #: object. entry_type = 'Path' - def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) + def __init__(self, core): + Plugin.__init__(self, core) Generator.__init__(self) + self.fam = Bcfg2.Server.FileMonitor.get_fam() + #: See :class:`Bcfg2.Server.Plugins.interfaces.Generator` for #: details on the Entries attribute. self.Entries[self.entry_type] = {} @@ -1518,7 +1586,6 @@ class GroupSpool(Plugin, Generator): self.entries = {} self.handles = {} self.AddDirectoryMonitor('') - self.encoding = core.setup['encoding'] __init__.__doc__ = Plugin.__init__.__doc__ def add_entry(self, event): @@ -1542,8 +1609,7 @@ class GroupSpool(Plugin, Generator): dirpath = self.data + ident self.entries[ident] = self.es_cls(self.filename_pattern, dirpath, - self.es_child_cls, - self.encoding) + self.es_child_cls) self.Entries[self.entry_type][ident] = \ self.entries[ident].bind_entry if not os.path.isdir(epath): @@ -1653,5 +1719,5 @@ class GroupSpool(Plugin, Generator): if not os.path.isdir(name): self.logger.error("Failed to open directory %s" % name) return - reqid = self.core.fam.AddMonitor(name, self) + reqid = self.fam.AddMonitor(name, self) self.handles[reqid] = relative diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 07717a710..c45d6fa84 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -6,11 +6,17 @@ import copy import threading import lxml.etree import Bcfg2.Server +import Bcfg2.Options from Bcfg2.Compat import Queue, Empty, Full, cPickle from Bcfg2.Server.Plugin.base import Plugin from Bcfg2.Server.Plugin.exceptions import PluginInitError, \ MetadataRuntimeError, MetadataConsistencyError +# Since this file basically just contains abstract interface +# descriptions, just about every function declaration has unused +# arguments. Disable this pylint warning for the whole file. +# pylint: disable=W0613 + class Generator(object): """ Generator plugins contribute to literal client configurations. @@ -26,13 +32,12 @@ class Generator(object): generate the content. The callable will receive two arguments: the abstract entry (as an lxml.etree._Element object), and the client metadata object the entry is being generated for. - #. If the entry is not listed in ``Entries``, the Bcfg2 core calls :func:`HandlesEntry`; if that returns True, then it calls :func:`HandleEntry`. """ - def HandlesEntry(self, entry, metadata): # pylint: disable=W0613 + def HandlesEntry(self, entry, metadata): """ HandlesEntry is the slow path method for routing configuration binding requests. It is called if the ``Entries`` dict does not contain a method for binding the @@ -47,7 +52,7 @@ class Generator(object): """ return False - def HandleEntry(self, entry, metadata): # pylint: disable=W0613 + def HandleEntry(self, entry, metadata): """ HandleEntry is the slow path method for binding configuration binding requests. It is called if the ``Entries`` dict does not contain a method for binding the @@ -137,7 +142,6 @@ class Metadata(object): """ pass - # pylint: disable=W0613 def resolve_client(self, address, cleanup_cache=False): """ Resolve the canonical name of this client. If this method is not implemented, the hostname claimed by the client is @@ -155,7 +159,6 @@ class Metadata(object): :class:`Bcfg2.Server.Plugin.exceptions.MetadataConsistencyError` """ return address[1] - # pylint: enable=W0613 def AuthenticateConnection(self, cert, user, password, address): """ Authenticate the given client. @@ -222,7 +225,7 @@ class Connector(object): """ Connector plugins augment client metadata instances with additional data, additional groups, or both. """ - def get_additional_groups(self, metadata): # pylint: disable=W0613 + def get_additional_groups(self, metadata): """ Return a list of additional groups for the given client. Each group can be either the name of a group (a string), or a :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object @@ -253,7 +256,7 @@ class Connector(object): """ return list() - def get_additional_data(self, metadata): # pylint: disable=W0613 + def get_additional_data(self, metadata): """ Return arbitrary additional data for the given ClientMetadata object. By convention this is usually a dict object, but doesn't need to be. @@ -346,14 +349,14 @@ class ThreadedStatistics(Statistics, Threaded, threading.Thread): """ ThreadedStatistics plugins process client statistics in a separate thread. """ - def __init__(self, core, datastore): - Statistics.__init__(self, core, datastore) + def __init__(self, core): + Statistics.__init__(self, core) Threaded.__init__(self) threading.Thread.__init__(self) # Event from the core signaling an exit self.terminate = core.terminate self.work_queue = Queue(100000) - self.pending_file = os.path.join(datastore, "etc", + self.pending_file = os.path.join(Bcfg2.Options.setup.repository, "etc", "%s.pending" % self.name) self.daemon = False @@ -476,7 +479,7 @@ class ThreadedStatistics(Statistics, Threaded, threading.Thread): # Someone who understands these interfaces better needs to write docs # for PullSource and PullTarget class PullSource(object): - def GetExtra(self, client): # pylint: disable=W0613 + def GetExtra(self, client): return [] def GetCurrentEntry(self, client, e_type, e_name): @@ -556,20 +559,23 @@ class Version(Plugin): create = False + options = Plugin.options + [ + Bcfg2.Options.PathOption(cf=('server', 'vcs_root'), + default='<repository>', + help='Server VCS repository root')] + #: The path to the VCS metadata file or directory, relative to the #: base of the Bcfg2 repository. E.g., for Subversion this would #: be ".svn" __vcs_metadata_path__ = None - def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) + __rmi__ = Plugin.__rmi__ + ['get_revision'] + + def __init__(self, core): + Plugin.__init__(self, core) - if core.setup['vcs_root']: - self.vcs_root = core.setup['vcs_root'] - else: - self.vcs_root = datastore if self.__vcs_metadata_path__: - self.vcs_path = os.path.join(self.vcs_root, + self.vcs_path = os.path.join(Bcfg2.Options.setup.vcs_root, self.__vcs_metadata_path__) if not os.path.exists(self.vcs_path): @@ -626,20 +632,46 @@ class ClientRunHooks(object): pass -class Caching(object): - """ A plugin that caches more than just the data received from the - FAM. This presents a unified interface to clear the cache. """ +class ClientACLs(object): + """ ClientACLs are used to grant or deny access to different + XML-RPC calls based on client IP or metadata. """ - def expire_cache(self, key=None): - """ Expire the cache associated with the given key. + def check_acl_ip(self, address, rmi): + """ Check if the given IP address is authorized to make the + named XML-RPC call. - :param key: The key to expire the cache for. Because cache - implementations vary tremendously between plugins, - this could be any number of things, but generally - a hostname. It also may or may not be possible to - expire the cache for a single host; this interface - does not require any guarantee about that. - :type key: varies - :returns: None + :param address: The address pair of the client to check ACLs for + :type address: tuple of (<ip address>, <port>) + :param rmi: The fully-qualified name of the RPC call + :param rmi: string + :returns: bool or None - True to allow, False to deny, None to + defer to metadata ACLs """ - raise NotImplementedError + return True + + def check_acl_metadata(self, metadata, rmi): + """ Check if the given client is authorized to make the named + XML-RPC call. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param rmi: The fully-qualified name of the RPC call + :param rmi: string + :returns: bool + """ + return True + + +class TemplateDataProvider(object): + """ TemplateDataProvider plugins provide variables to templates + for use in rendering. """ + + def get_template_data(self, entry, metadata, template): + """ Get a dict of variables that will be supplied to a Cfg + template for rendering """ + return dict() + + def get_xml_template_data(self, structfile, metadata): + """ Get a dict of variables that will be supplied to an XML + template (e.g., a bundle) for rendering """ + return dict() diff --git a/src/lib/Bcfg2/Server/Plugins/ACL.py b/src/lib/Bcfg2/Server/Plugins/ACL.py new file mode 100644 index 000000000..37f51a2a1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/ACL.py @@ -0,0 +1,146 @@ +""" Support for client ACLs based on IP address and client metadata """ + +import os +import struct +import socket +import Bcfg2.Server.Plugin + + +def rmi_names_equal(first, second): + """ Compare two XML-RPC method names and see if they match. + Resolves some limited wildcards; see + :ref:`server-plugins-misc-acl-wildcards` for details. + + :param first: One of the ACLs to compare + :type first: string + :param second: The other ACL to compare + :type second: string + :returns: bool """ + if first == second: + # single wildcard is special, and matches everything + return True + if first is None or second is None: + return False + if '*' not in first + second: + # no wildcards, and not exactly equal + return False + first_parts = first.split('.') + second_parts = second.split('.') + if len(first_parts) != len(second_parts): + return False + for i in range(len(first_parts)): + if (first_parts[i] != second_parts[i] and first_parts[i] != '*' and + second_parts[i] != '*'): + return False + return True + + +def ip2int(ip): + """ convert a dotted-quad IP address into an integer + representation of the same """ + return struct.unpack('>L', socket.inet_pton(socket.AF_INET, ip))[0] + + +def ip_matches(ip, entry): + """ Return True if the given IP matches the IP or IP and netmask + in the given ACL entry; False otherwise """ + if entry.get("netmask"): + try: + mask = int("1" * int(entry.get("netmask")) + + "0" * (32 - int(entry.get("netmask"))), 2) + except ValueError: + mask = ip2int(entry.get("netmask")) + return ip2int(ip) & mask == ip2int(entry.get("address")) & mask + elif entry.get("address") is None: + # no address, no netmask -- match all + return True + elif ip == entry.get("address"): + # just a plain ip address + return True + return False + + +class IPACLFile(Bcfg2.Server.Plugin.XMLFileBacked): + """ representation of ACL ip.xml, for IP-based ACLs """ + __identifier__ = None + actions = dict(Allow=True, + Deny=False, + Defer=None) + + def check_acl(self, address, rmi): + """ Check a client address against the ACL list """ + if not len(self.entries): + # default defer if no ACLs are defined. + self.debug_log("ACL: %s requests %s: No IP ACLs, defer" % + (address, rmi)) + return self.actions["Defer"] + for entry in self.entries: + if (ip_matches(address, entry) and + rmi_names_equal(entry.get("method"), rmi)): + self.debug_log("ACL: %s requests %s: Found matching IP ACL, " + "%s" % (address, rmi, entry.tag.lower())) + return self.actions[entry.tag] + if address == "127.0.0.1": + self.debug_log("ACL: %s requests %s: No matching IP ACLs, " + "localhost allowed" % (address, rmi)) + return self.actions['Allow'] # default allow for localhost + + self.debug_log("ACL: %s requests %s: No matching IP ACLs, defer" % + (address, rmi)) + return self.actions["Defer"] # default defer for other machines + + +class MetadataACLFile(Bcfg2.Server.Plugin.StructFile): + """ representation of ACL metadata.xml, for metadata-based ACLs """ + def check_acl(self, metadata, rmi): + """ check client metadata against the ACL list """ + if not len(self.entries): + # default allow if no ACLs are defined. + self.debug_log("ACL: %s requests %s: No metadata ACLs, allow" % + (metadata.hostname, rmi)) + return True + for el in self.Match(metadata): + if rmi_names_equal(el.get("method"), rmi): + self.debug_log("ACL: %s requests %s: Found matching metadata " + "ACL, %s" % (metadata.hostname, rmi, + el.tag.lower())) + return el.tag == "Allow" + if metadata.hostname in ['localhost', 'localhost.localdomain']: + # default allow for localhost + self.debug_log("ACL: %s requests %s: No matching metadata ACLs, " + "localhost allowed" % (metadata.hostname, rmi)) + return True + self.debug_log("ACL: %s requests %s: No matching metadata ACLs, deny" % + (metadata.hostname, rmi)) + return False # default deny for other machines + + +class ACL(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.ClientACLs): + """ allow connections to bcfg-server based on IP address """ + + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) + Bcfg2.Server.Plugin.ClientACLs.__init__(self) + self.ip_acls = IPACLFile(os.path.join(self.data, 'ip.xml'), + should_monitor=True) + self.metadata_acls = MetadataACLFile(os.path.join(self.data, + 'metadata.xml'), + should_monitor=True) + + def check_acl_ip(self, address, rmi): + self.debug_log("ACL: %s requests %s: Checking IP ACLs" % + (address[0], rmi)) + return self.ip_acls.check_acl(address[0], rmi) + + def check_acl_metadata(self, metadata, rmi): + self.debug_log("ACL: %s requests %s: Checking metadata ACLs" % + (metadata.hostname, rmi)) + return self.metadata_acls.check_acl(metadata, rmi) + + def set_debug(self, debug): + rv = Bcfg2.Server.Plugin.Plugin.set_debug(self, debug) + self.ip_acls.set_debug(debug) + self.metadata_acls.set_debug(debug) + return rv + set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py index 147f37fbf..3f92542e7 100644 --- a/src/lib/Bcfg2/Server/Plugins/AWSTags.py +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -3,10 +3,9 @@ import os import re import sys -import Bcfg2.Server.Lint import Bcfg2.Server.Plugin from boto import connect_ec2 -from Bcfg2.Cache import Cache +from Bcfg2.Server.Cache import Cache from Bcfg2.Compat import ConfigParser @@ -66,11 +65,7 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): create = 'AWSTags' def __init__(self, filename, core=None): - try: - fam = core.fam - except AttributeError: - fam = None - Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, should_monitor=True) self.core = core self.tags = [] @@ -107,15 +102,13 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): class AWSTags(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.Connector): """ Query tags from AWS via boto, optionally setting group membership """ __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) try: @@ -178,6 +171,8 @@ class AWSTags(Bcfg2.Server.Plugin.Plugin, return self._tagcache[metadata.hostname] def expire_cache(self, key=None): + """ Expire the cache for one host, or for all hosts. This is + exposed as an XML-RPC RMI. """ self._tagcache.expire(key=key) def start_client_run(self, metadata): @@ -189,29 +184,3 @@ class AWSTags(Bcfg2.Server.Plugin.Plugin, def get_additional_groups(self, metadata): return self.config.get_groups(metadata.hostname, self.get_tags(metadata)) - - -class AWSTagsLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags - <server-plugins-connectors-awstags>` patterns for validity. """ - - def Run(self): - cfg = self.core.plugins['AWSTags'].config - for entry in cfg.xdata.xpath('//Tag'): - self.check(entry, "name") - if entry.get("value"): - self.check(entry, "value") - - @classmethod - def Errors(cls): - return {"pattern-fails-to-initialize": "error"} - - def check(self, entry, attr): - """ Check a single attribute (``name`` or ``value``) of a - single entry for validity. """ - try: - re.compile(entry.get(attr)) - except re.error: - self.LintError("pattern-fails-to-initialize", - "'%s' regex could not be compiled: %s\n %s" % - (attr, sys.exc_info()[1], entry.get("name"))) diff --git a/src/lib/Bcfg2/Server/Plugins/Account.py b/src/lib/Bcfg2/Server/Plugins/Account.py deleted file mode 100644 index fd49d3655..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Account.py +++ /dev/null @@ -1,102 +0,0 @@ -"""This handles authentication setup.""" - -import Bcfg2.Server.Plugin - - -class Account(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Generator): - """This module generates account config files, - based on an internal data repo: - static.(passwd|group|limits.conf) -> static entries - dyn.(passwd|group) -> dynamic entries (usually acquired from yp or somesuch) - useraccess -> users to be granted login access on some hosts - superusers -> users to be granted root privs on all hosts - rootlike -> users to be granted root privs on some hosts - - """ - name = 'Account' - __author__ = 'bcfg-dev@mcs.anl.gov' - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Generator.__init__(self) - self.Entries = {'ConfigFile': {'/etc/passwd': self.from_yp_cb, - '/etc/group': self.from_yp_cb, - '/etc/security/limits.conf': self.gen_limits_cb, - '/root/.ssh/authorized_keys': self.gen_root_keys_cb, - '/etc/sudoers': self.gen_sudoers}} - try: - self.repository = Bcfg2.Server.Plugin.DirectoryBacked(self.data, - self.core.fam) - except: - self.logger.error("Failed to load repos: %s, %s" % \ - (self.data, "%s/ssh" % (self.data))) - raise Bcfg2.Server.Plugin.PluginInitError - - def from_yp_cb(self, entry, metadata): - """Build password file from cached yp data.""" - fname = entry.attrib['name'].split('/')[-1] - entry.text = self.repository.entries["static.%s" % (fname)].data - entry.text += self.repository.entries["dyn.%s" % (fname)].data - perms = {'owner': 'root', - 'group': 'root', - 'mode': '0644'} - [entry.attrib.__setitem__(key, value) for (key, value) in \ - list(perms.items())] - - def gen_limits_cb(self, entry, metadata): - """Build limits entries based on current ACLs.""" - entry.text = self.repository.entries["static.limits.conf"].data - superusers = self.repository.entries["superusers"].data.split() - useraccess = [line.split(':') for line in \ - self.repository.entries["useraccess"].data.split()] - users = [user for (user, host) in \ - useraccess if host == metadata.hostname.split('.')[0]] - perms = {'owner': 'root', - 'group': 'root', - 'mode': '0600'} - [entry.attrib.__setitem__(key, value) for (key, value) in \ - list(perms.items())] - entry.text += "".join(["%s hard maxlogins 1024\n" % uname for uname in superusers + users]) - if "*" not in users: - entry.text += "* hard maxlogins 0\n" - - def gen_root_keys_cb(self, entry, metadata): - """Build root authorized keys file based on current ACLs.""" - superusers = self.repository.entries['superusers'].data.split() - try: - rootlike = [line.split(':', 1) for line in \ - self.repository.entries['rootlike'].data.split()] - superusers += [user for (user, host) in rootlike \ - if host == metadata.hostname.split('.')[0]] - except: - pass - rdata = self.repository.entries - entry.text = "".join([rdata["%s.key" % user].data for user \ - in superusers if \ - ("%s.key" % user) in rdata]) - perms = {'owner': 'root', - 'group': 'root', - 'mode': '0600'} - [entry.attrib.__setitem__(key, value) for (key, value) \ - in list(perms.items())] - - def gen_sudoers(self, entry, metadata): - """Build root authorized keys file based on current ACLs.""" - superusers = self.repository.entries['superusers'].data.split() - try: - rootlike = [line.split(':', 1) for line in \ - self.repository.entries['rootlike'].data.split()] - superusers += [user for (user, host) in rootlike \ - if host == metadata.hostname.split('.')[0]] - except: - pass - entry.text = self.repository.entries['static.sudoers'].data - entry.text += "".join(["%s ALL=(ALL) ALL\n" % uname \ - for uname in superusers]) - perms = {'owner': 'root', - 'group': 'root', - 'mode': '0440'} - [entry.attrib.__setitem__(key, value) for (key, value) \ - in list(perms.items())] diff --git a/src/lib/Bcfg2/Server/Plugins/Base.py b/src/lib/Bcfg2/Server/Plugins/Base.py deleted file mode 100644 index a18204d60..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Base.py +++ /dev/null @@ -1,33 +0,0 @@ -"""This module sets up a base list of configuration entries.""" - -import copy -import lxml.etree -import Bcfg2.Server.Plugin -from itertools import chain - - -class Base(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): - """This Structure is good for the pile of independent configs - needed for most actual systems. - """ - name = 'Base' - __author__ = 'bcfg-dev@mcs.anl.gov' - __child__ = Bcfg2.Server.Plugin.StructFile - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data, - self.core.fam) - - def BuildStructures(self, metadata): - """Build structures for client described by metadata.""" - ret = lxml.etree.Element("Independent", version='2.0') - fragments = list(chain(*[base.Match(metadata) - for base in list(self.entries.values())])) - for frag in fragments: - ret.append(copy.copy(frag)) - return [ret] diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 58f8f4430..41ee57b6d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -1,201 +1,132 @@ """This provides bundle clauses with translation functionality.""" -import copy -import logging -import lxml.etree import os -import os.path import re import sys -import Bcfg2.Server -import Bcfg2.Server.Plugin -import Bcfg2.Server.Lint - -try: - import genshi.template.base - from Bcfg2.Server.Plugins.TGenshi import removecomment, TemplateFile - HAS_GENSHI = True -except ImportError: - HAS_GENSHI = False - - -SETUP = None +import copy +import fnmatch +import lxml.etree +from Bcfg2.Server.Plugin import StructFile, Plugin, Structure, \ + StructureValidator, XMLDirectoryBacked, Generator +from genshi.template import TemplateError -class BundleFile(Bcfg2.Server.Plugin.StructFile): +class BundleFile(StructFile): """ Representation of a bundle XML file """ - def get_xml_value(self, metadata): - """ get the XML data that applies to the given client """ - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - for item in self.Match(metadata): - bundle.append(copy.copy(item)) - return bundle - - -if HAS_GENSHI: - class BundleTemplateFile(TemplateFile, - Bcfg2.Server.Plugin.StructFile): - """ Representation of a Genshi-templated bundle XML file """ - - def __init__(self, name, specific, encoding, fam=None): - TemplateFile.__init__(self, name, specific, encoding) - Bcfg2.Server.Plugin.StructFile.__init__(self, name, fam=fam) - self.logger = logging.getLogger(name) - - def get_xml_value(self, metadata): - """ get the rendered XML data that applies to the given - client """ - if not hasattr(self, 'template'): - msg = "No parsed template information for %s" % self.name - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - stream = self.template.generate( - metadata=metadata, - repo=SETUP['repo']).filter(removecomment) - data = lxml.etree.XML( - stream.render('xml', strip_whitespace=False).encode(), - parser=Bcfg2.Server.XMLParser) - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - for item in self.Match(metadata, data): - bundle.append(copy.deepcopy(item)) - return bundle - - def Match(self, metadata, xdata): # pylint: disable=W0221 - """Return matching fragments of parsed template.""" - rv = [] - for child in xdata.getchildren(): - rv.extend(self._match(child, metadata)) - self.logger.debug("File %s got %d match(es)" % (self.name, - len(rv))) - return rv - - class SGenshiTemplateFile(BundleTemplateFile): - """ provided for backwards compat with the deprecated SGenshi - plugin """ - pass - - -class Bundler(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): + bundle_name_re = re.compile(r'^(?P<name>.*)\.(xml|genshi)$') + + def __init__(self, filename, should_monitor=False): + StructFile.__init__(self, filename, should_monitor=should_monitor) + if self.name.endswith(".genshi"): + self.logger.warning("Bundler: %s: Bundle filenames ending with " + ".genshi are deprecated; add the Genshi XML " + "namespace to a .xml bundle instead" % + self.name) + + def Index(self): + StructFile.Index(self) + if self.xdata.get("name"): + self.logger.warning("Bundler: %s: Explicitly specifying bundle " + "names is deprecated" % self.name) + + @property + def bundle_name(self): + """ The name of the bundle, as determined from the filename """ + return self.bundle_name_re.match( + os.path.basename(self.name)).group("name") + + +class Bundler(Plugin, + Structure, + StructureValidator, + XMLDirectoryBacked): """ The bundler creates dependent clauses based on the bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' - patterns = re.compile(r'^(?P<name>.*)\.(xml|genshi)$') - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - self.encoding = core.setup['encoding'] - self.__child__ = self.template_dispatch - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data, - self.core.fam) - global SETUP - SETUP = core.setup - - def template_dispatch(self, name, _): - """ Add the correct child entry type to Bundler depending on - whether the XML file in question is a plain XML file or a - templated bundle """ - bundle = lxml.etree.parse(name, parser=Bcfg2.Server.XMLParser) - nsmap = bundle.getroot().nsmap - if (name.endswith('.genshi') or - ('py' in nsmap and - nsmap['py'] == 'http://genshi.edgewall.org/')): - if HAS_GENSHI: - spec = Bcfg2.Server.Plugin.Specificity() - return BundleTemplateFile(name, spec, self.encoding, - fam=self.core.fam) - else: - raise Bcfg2.Server.Plugin.PluginExecutionError("Genshi not " - "available: %s" - % name) - else: - return BundleFile(name, fam=self.fam) + __child__ = BundleFile + patterns = re.compile(r'^.*\.(?:xml|genshi)$') + + def __init__(self, core): + Plugin.__init__(self, core) + Structure.__init__(self) + StructureValidator.__init__(self) + XMLDirectoryBacked.__init__(self, self.data) + #: Bundles by bundle name, rather than filename + self.bundles = dict() + + def HandleEvent(self, event): + XMLDirectoryBacked.HandleEvent(self, event) + self.bundles = dict([(b.bundle_name, b) + for b in self.entries.values()]) + + def validate_structures(self, metadata, structures): + """ Translate <Path glob='...'/> entries into <Path name='...'/> + entries """ + for struct in structures: + for pathglob in struct.xpath("//Path[@glob]"): + for plugin in self.core.plugins_by_type(Generator): + for match in fnmatch.filter(plugin.Entries['Path'].keys(), + pathglob.get("glob")): + lxml.etree.SubElement(pathglob.getparent(), + "Path", name=match) + pathglob.getparent().remove(pathglob) def BuildStructures(self, metadata): - """Build all structures for client (metadata).""" bundleset = [] - - bundle_entries = {} - for key, item in self.entries.items(): - bundle_entries.setdefault( - self.patterns.match(os.path.basename(key)).group('name'), - []).append(item) - - for bundlename in metadata.bundles: + bundles = copy.copy(metadata.bundles) + bundles_added = set(bundles) + while bundles: + bundlename = bundles.pop() try: - entries = bundle_entries[bundlename] + bundle = self.bundles[bundlename] except KeyError: self.logger.error("Bundler: Bundle %s does not exist" % bundlename) continue + try: - bundleset.append(entries[0].get_xml_value(metadata)) - except genshi.template.base.TemplateError: + data = bundle.XMLMatch(metadata) + except TemplateError: err = sys.exc_info()[1] self.logger.error("Bundler: Failed to render templated bundle " "%s: %s" % (bundlename, err)) + continue except: self.logger.error("Bundler: Unexpected bundler error for %s" % bundlename, exc_info=1) - return bundleset - + continue -class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various :ref:`Bundler - <server-plugins-structures-bundler-index>` checks. """ - - def Run(self): - self.missing_bundles() - for bundle in self.core.plugins['Bundler'].entries.values(): - if (self.HandlesFile(bundle.name) and - (not HAS_GENSHI or - not isinstance(bundle, BundleTemplateFile))): - self.bundle_names(bundle) - - @classmethod - def Errors(cls): - return {"bundle-not-found": "error", - "inconsistent-bundle-name": "warning"} - - def missing_bundles(self): - """ Find bundles listed in Metadata but not implemented in - Bundler. """ - if self.files is None: - # when given a list of files on stdin, this check is - # useless, so skip it - groupdata = self.metadata.groups_xml.xdata - ref_bundles = set([b.get("name") - for b in groupdata.findall("//Bundle")]) - - allbundles = self.core.plugins['Bundler'].entries.keys() - for bundle in ref_bundles: - xmlbundle = "%s.xml" % bundle - genshibundle = "%s.genshi" % bundle - if (xmlbundle not in allbundles and - genshibundle not in allbundles): - self.LintError("bundle-not-found", - "Bundle %s referenced, but does not exist" % - bundle) - - def bundle_names(self, bundle): - """ Verify bundle name attribute matches filename. - - :param bundle: The bundle to verify - :type bundle: Bcfg2.Server.Plugins.Bundler.BundleFile - """ - try: - xdata = lxml.etree.XML(bundle.data) - except AttributeError: - # genshi template - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - - fname = os.path.splitext(os.path.basename(bundle.name))[0] - bname = xdata.get('name') - if fname != bname: - self.LintError("inconsistent-bundle-name", - "Inconsistent bundle name: filename is %s, " - "bundle name is %s" % (fname, bname)) + if data.get("independent", "false").lower() == "true": + data.tag = "Independent" + del data.attrib['independent'] + + data.set("name", bundlename) + + for child in data.findall("Bundle"): + if child.getchildren(): + # XInclude'd bundle -- "flatten" it so there + # aren't extra Bundle tags, since other bits in + # Bcfg2 only handle the direct children of the + # top-level Bundle tag + if data.get("name"): + self.logger.warning("Bundler: In file XIncluded from " + "%s: Explicitly specifying " + "bundle names is deprecated" % + self.name) + for el in child.getchildren(): + data.append(el) + data.remove(child) + elif child.get("name"): + # dependent bundle -- add it to the list of + # bundles for this client + if child.get("name") not in bundles_added: + bundles.append(child.get("name")) + bundles_added.add(child.get("name")) + data.remove(child) + else: + # neither name or children -- wat + self.logger.warning("Bundler: Useless empty Bundle tag " + "in %s" % self.name) + data.remove(child) + bundleset.append(data) + return bundleset diff --git a/src/lib/Bcfg2/Server/Plugins/Bzr.py b/src/lib/Bcfg2/Server/Plugins/Bzr.py index e0cbdf72a..01b51ace4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bzr.py +++ b/src/lib/Bcfg2/Server/Plugins/Bzr.py @@ -11,19 +11,19 @@ class Bzr(Bcfg2.Server.Plugin.Version): using bazaar. """ __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) self.logger.debug("Initialized Bazaar plugin with directory %s at " - "revision = %s" % (self.vcs_root, + "revision = %s" % (Bcfg2.Options.setup.vcs_root, self.get_revision())) def get_revision(self): """Read Bazaar revision information for the Bcfg2 repository.""" try: - working_tree = WorkingTree.open(self.vcs_root) + working_tree = WorkingTree.open(Bcfg2.Options.setup.vcs_root) revision = str(working_tree.branch.revno()) if (working_tree.has_changes(working_tree.basis_tree()) or - working_tree.unknowns()): + working_tree.unknowns()): revision += "+" except errors.NotBranchError: msg = "Failed to read Bazaar branch" diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py index 41d5588e4..7792d7e5c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py @@ -3,8 +3,9 @@ based on an XML specification of which SSH keypairs should granted access. """ import lxml.etree +import Bcfg2.Options from Bcfg2.Server.Plugin import StructFile, PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP, CFG +from Bcfg2.Server.Plugins.Cfg import CfgGenerator, get_cfg from Bcfg2.Server.Plugins.Metadata import ClientMetadata @@ -20,26 +21,13 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): #: Handle authorized keys XML files __basenames__ = ['authorizedkeys.xml', 'authorized_keys.xml'] - #: This handler is experimental, in part because it depends upon - #: the (experimental) CfgPrivateKeyCreator handler - experimental = True - def __init__(self, fname): - CfgGenerator.__init__(self, fname, None, None) + CfgGenerator.__init__(self, fname, None) StructFile.__init__(self, fname) self.cache = dict() - self.core = CFG.core + self.core = get_cfg().core __init__.__doc__ = CfgGenerator.__init__.__doc__ - @property - def category(self): - """ The name of the metadata category that generated keys are - specific to """ - if (SETUP.cfp.has_section("sshkeys") and - SETUP.cfp.has_option("sshkeys", "category")): - return SETUP.cfp.get("sshkeys", "category") - return None - def handle_event(self, event): CfgGenerator.handle_event(self, event) StructFile.HandleEvent(self, event) @@ -51,12 +39,6 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): rv = [] for allow in spec.findall("Allow"): options = [] - if allow.find("Params") is not None: - self.logger.warning("Use of <Params> in authorized_keys.xml " - "is deprecated; use <Option> instead") - options.extend("=".join(p) - for p in allow.find("Params").attrib.items()) - for opt in allow.findall("Option"): if opt.get("value"): options.append("%s=%s" % (opt.get("name"), @@ -68,7 +50,8 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): if pubkey_name: host = allow.get("host") group = allow.get("group") - category = allow.get("category", self.category) + category = allow.get("category", + Bcfg2.Options.setup.sshkeys_category) if host: key_md = self.core.build_metadata(host) elif group: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py deleted file mode 100644 index 49a5a85b3..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py +++ /dev/null @@ -1,28 +0,0 @@ -""" Handle .cat files, which append lines to and remove lines from -plaintext files """ - -from Bcfg2.Server.Plugins.Cfg import CfgFilter - - -class CfgCatFilter(CfgFilter): - """ CfgCatFilter appends lines to and remove lines from plaintext - :ref:`server-plugins-generators-Cfg` files""" - - #: Handle .cat files - __extensions__ = ['cat'] - - #: .cat files are deprecated - deprecated = True - - def modify_data(self, entry, metadata, data): - datalines = data.strip().split('\n') - for line in self.data.split('\n'): - if not line: - continue - if line.startswith('+'): - datalines.append(line[1:]) - elif line.startswith('-'): - if line[1:] in datalines: - datalines.remove(line[1:]) - return "\n".join(datalines) + "\n" - modify_data.__doc__ = CfgFilter.modify_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index 724164cf5..84309b5dd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -2,8 +2,10 @@ <http://www.cheetahtemplate.org/>`_ templating system to generate :ref:`server-plugins-generators-cfg` files. """ -from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP +import Bcfg2.Options +from Bcfg2.Server.Plugin import PluginExecutionError, \ + DefaultTemplateDataProvider, get_template_data +from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: from Cheetah.Template import Template @@ -12,6 +14,18 @@ except ImportError: HAS_CHEETAH = False +class DefaultCheetahDataProvider(DefaultTemplateDataProvider): + """ Template data provider for Cheetah templates. Cheetah and + Genshi currently differ over the value of the ``path`` variable, + which is why this is necessary. """ + + def get_template_data(self, entry, metadata, template): + rv = DefaultTemplateDataProvider.get_template_data(self, entry, + metadata, template) + rv['path'] = rv['name'] + return rv + + class CfgCheetahGenerator(CfgGenerator): """ The CfgCheetahGenerator allows you to use the `Cheetah <http://www.cheetahtemplate.org/>`_ templating system to generate @@ -27,19 +41,18 @@ class CfgCheetahGenerator(CfgGenerator): #: :class:`Cheetah.Template.Template` compiler settings settings = dict(useStackFrames=False) - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) if not HAS_CHEETAH: raise PluginExecutionError("Cheetah is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): - template = Template(self.data.decode(self.encoding), + template = Template(self.data.decode(Bcfg2.Options.setup.encoding), compilerSettings=self.settings) - template.metadata = metadata - template.name = entry.get('realname', entry.get('name')) - template.path = entry.get('realname', entry.get('name')) - template.source_path = self.name - template.repo = SETUP['repo'] + for key, val in get_template_data( + entry, metadata, self.name, + default=DefaultCheetahDataProvider()).items(): + setattr(template, key, val) return template.respond() get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py deleted file mode 100644 index da506a195..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py +++ /dev/null @@ -1,35 +0,0 @@ -""" Handle .diff files, which apply diffs to plaintext files """ - -import os -import tempfile -from Bcfg2.Server.Plugin import PluginExecutionError -from subprocess import Popen, PIPE -from Bcfg2.Server.Plugins.Cfg import CfgFilter - - -class CfgDiffFilter(CfgFilter): - """ CfgDiffFilter applies diffs to plaintext - :ref:`server-plugins-generators-Cfg` files """ - - #: Handle .diff files - __extensions__ = ['diff'] - - #: .diff files are deprecated - deprecated = True - - def modify_data(self, entry, metadata, data): - basehandle, basename = tempfile.mkstemp() - open(basename, 'w').write(data) - os.close(basehandle) - - cmd = ["patch", "-u", "-f", basename] - patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - stderr = patch.communicate(input=self.data)[1] - ret = patch.wait() - output = open(basename, 'r').read() - os.unlink(basename) - if ret != 0: - raise PluginExecutionError("Error applying diff %s: %s" % - (self.name, stderr)) - return output - modify_data.__doc__ = CfgFilter.modify_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index 0a30a070a..849c75f70 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -1,11 +1,11 @@ """ CfgEncryptedGenerator lets you encrypt your plaintext :ref:`server-plugins-generators-cfg` files on the server. """ +import Bcfg2.Options from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP +from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: - from Bcfg2.Encryption import bruteforce_decrypt, EVPError, \ - get_algorithm, CFG_SECTION + from Bcfg2.Server.Encryption import bruteforce_decrypt, EVPError HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False @@ -22,8 +22,8 @@ class CfgEncryptedGenerator(CfgGenerator): #: .genshi.crypt and .cheetah.crypt files __priority__ = 50 - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") @@ -33,17 +33,13 @@ class CfgEncryptedGenerator(CfgGenerator): return # todo: let the user specify a passphrase by name try: - self.data = bruteforce_decrypt( - self.data, setup=SETUP, - algorithm=get_algorithm(SETUP)) + self.data = bruteforce_decrypt(self.data) except EVPError: - strict = SETUP.cfp.get(CFG_SECTION, "decrypt", - default="strict") msg = "Cfg: Failed to decrypt %s" % self.name - if strict: - raise PluginExecutionError(msg) - else: + if Bcfg2.Options.setup.lax_decryption: self.logger.debug(msg) + else: + raise PluginExecutionError(msg) def get_data(self, entry, metadata): if self.data is None: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 130652aef..f69ab8e5f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -1,32 +1,24 @@ """ Handle encrypted Genshi templates (.crypt.genshi or .genshi.crypt files) """ +from genshi.template import TemplateLoader from Bcfg2.Compat import StringIO from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import SETUP from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator try: - from Bcfg2.Encryption import bruteforce_decrypt, get_algorithm + from Bcfg2.Server.Encryption import bruteforce_decrypt HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False -try: - from genshi.template import TemplateLoader -except ImportError: - # CfgGenshiGenerator will raise errors if genshi doesn't exist - TemplateLoader = object # pylint: disable=C0103 - class EncryptedTemplateLoader(TemplateLoader): """ Subclass :class:`genshi.template.TemplateLoader` to decrypt the data on the fly as it's read in using - :func:`Bcfg2.Encryption.bruteforce_decrypt` """ + :func:`Bcfg2.Server.Encryption.bruteforce_decrypt` """ def _instantiate(self, cls, fileobj, filepath, filename, encoding=None): - plaintext = \ - StringIO(bruteforce_decrypt(fileobj.read(), - algorithm=get_algorithm(SETUP))) + plaintext = StringIO(bruteforce_decrypt(fileobj.read())) return TemplateLoader._instantiate(self, cls, plaintext, filepath, filename, encoding=encoding) @@ -45,7 +37,7 @@ class CfgEncryptedGenshiGenerator(CfgGenshiGenerator): #: when it's read in __loader_cls__ = EncryptedTemplateLoader - def __init__(self, fname, spec, encoding): - CfgGenshiGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenshiGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py new file mode 100644 index 000000000..c8da84ae0 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py @@ -0,0 +1,25 @@ +""" Handle encrypted Jinja2 templates (.crypt.jinja2 or +.jinja2.crypt files)""" + +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator \ + import CfgEncryptedGenerator + + +class CfgEncryptedJinja2Generator(CfgJinja2Generator, CfgEncryptedGenerator): + """ CfgEncryptedJinja2Generator lets you encrypt your Jinja2 + :ref:`server-plugins-generators-cfg` files on the server """ + + #: handle .crypt.jinja2 or .jinja2.crypt files + __extensions__ = ['jinja2.crypt', 'crypt.jinja2'] + + #: Override low priority from parent class + __priority__ = 0 + + def handle_event(self, event): + CfgEncryptedGenerator.handle_event(self, event) + handle_event.__doc__ = CfgEncryptedGenerator.handle_event.__doc__ + + def get_data(self, entry, metadata): + return CfgJinja2Generator.get_data(self, entry, metadata) + get_data.__doc__ = CfgJinja2Generator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index 313e53ee9..953473a12 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -3,8 +3,8 @@ import os import sys import shlex +from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import PluginExecutionError -from subprocess import Popen, PIPE from Bcfg2.Server.Plugins.Cfg import CfgVerifier, CfgVerificationError @@ -15,27 +15,19 @@ class CfgExternalCommandVerifier(CfgVerifier): #: Handle :file:`:test` files __basenames__ = [':test'] - def __init__(self, name, specific, encoding): - CfgVerifier.__init__(self, name, specific, encoding) + def __init__(self, name, specific): + CfgVerifier.__init__(self, name, specific) self.cmd = [] + self.exc = Executor(timeout=30) __init__.__doc__ = CfgVerifier.__init__.__doc__ def verify_entry(self, entry, metadata, data): try: - proc = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - out, err = proc.communicate(input=data) - rv = proc.wait() - if rv != 0: - # pylint: disable=E1103 - raise CfgVerificationError(err.strip() or out.strip() or - "Non-zero return value %s" % rv) - # pylint: enable=E1103 - except CfgVerificationError: - raise - except: - err = sys.exc_info()[1] - raise CfgVerificationError("Error running external command " - "verifier: %s" % err) + result = self.exc.run(self.cmd, inputdata=data) + if not result.success: + raise CfgVerificationError(result.error) + except OSError: + raise CfgVerificationError(sys.exc_info()[1]) verify_entry.__doc__ = CfgVerifier.verify_entry.__doc__ def handle_event(self, event): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index 83a5c1165..ef4e6a656 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -5,63 +5,54 @@ import re import sys import traceback -from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP - -try: - import genshi.core - from genshi.template import TemplateLoader, NewTextTemplate - from genshi.template.eval import UndefinedError, Suite - #: True if Genshi libraries are available - HAS_GENSHI = True - - def _genshi_removes_blank_lines(): - """ Genshi 0.5 uses the Python :mod:`compiler` package to - compile genshi snippets to AST. Genshi 0.6 uses some bespoke - magic, because compiler has been deprecated. - :func:`compiler.parse` produces an AST that removes all excess - whitespace (e.g., blank lines), while - :func:`genshi.template.astutil.parse` does not. In order to - determine which actual line of code an error occurs on, we - need to know which is in use and how it treats blank lines. - I've beat my head against this for hours and the best/only way - I can find is to compile some genshi code with an error and - see which line it's on.""" - code = """d = dict() - +import Bcfg2.Options +from Bcfg2.Server.Plugin import PluginExecutionError, removecomment, \ + DefaultTemplateDataProvider, get_template_data +from Bcfg2.Server.Plugins.Cfg import CfgGenerator +from genshi.template import TemplateLoader, NewTextTemplate +from genshi.template.eval import UndefinedError, Suite + + +def _genshi_removes_blank_lines(): + """ Genshi 0.5 uses the Python :mod:`compiler` package to + compile genshi snippets to AST. Genshi 0.6 uses some bespoke + magic, because compiler has been deprecated. + :func:`compiler.parse` produces an AST that removes all excess + whitespace (e.g., blank lines), while + :func:`genshi.template.astutil.parse` does not. In order to + determine which actual line of code an error occurs on, we + need to know which is in use and how it treats blank lines. + I've beat my head against this for hours and the best/only way + I can find is to compile some genshi code with an error and + see which line it's on.""" + code = """d = dict() d['a']""" - try: - Suite(code).execute(dict()) - except KeyError: - line = traceback.extract_tb(sys.exc_info()[2])[-1][1] - if line == 2: - return True - else: - return False - - #: True if Genshi removes all blank lines from a code block before - #: executing it; False indicates that Genshi only removes leading - #: and trailing blank lines. See - #: :func:`_genshi_removes_blank_lines` for an explanation of this. - GENSHI_REMOVES_BLANK_LINES = _genshi_removes_blank_lines() -except ImportError: - TemplateLoader = None # pylint: disable=C0103 - HAS_GENSHI = False - - -def removecomment(stream): - """ A Genshi filter that removes comments from the stream. This - function is a generator. - - :param stream: The Genshi stream to remove comments from - :type stream: genshi.core.Stream - :returns: tuple of ``(kind, data, pos)``, as when iterating - through a Genshi stream - """ - for kind, data, pos in stream: - if kind is genshi.core.COMMENT: - continue - yield kind, data, pos + try: + Suite(code).execute(dict()) + except KeyError: + line = traceback.extract_tb(sys.exc_info()[2])[-1][1] + if line == 2: + return True + else: + return False + +#: True if Genshi removes all blank lines from a code block before +#: executing it; False indicates that Genshi only removes leading +#: and trailing blank lines. See +#: :func:`_genshi_removes_blank_lines` for an explanation of this. +GENSHI_REMOVES_BLANK_LINES = _genshi_removes_blank_lines() + + +class DefaultGenshiDataProvider(DefaultTemplateDataProvider): + """ Template data provider for Genshi templates. Cheetah and + Genshi currently differ over the value of the ``path`` variable, + which is why this is necessary. """ + + def get_template_data(self, entry, metadata, template): + rv = DefaultTemplateDataProvider.get_template_data(self, entry, + metadata, template) + rv['path'] = template + return rv class CfgGenshiGenerator(CfgGenerator): @@ -92,10 +83,8 @@ class CfgGenshiGenerator(CfgGenerator): #: occurred. pyerror_re = re.compile(r'<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>') - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) - if not HAS_GENSHI: - raise PluginExecutionError("Genshi is not available") + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) self.template = None self.loader = self.__loader_cls__(max_cache_size=0) __init__.__doc__ = CfgGenerator.__init__.__doc__ @@ -105,19 +94,18 @@ class CfgGenshiGenerator(CfgGenerator): raise PluginExecutionError("Failed to load template %s" % self.name) - fname = entry.get('realname', entry.get('name')) - stream = \ - self.template.generate(name=fname, - metadata=metadata, - path=self.name, - source_path=self.name, - repo=SETUP['repo']).filter(removecomment) + stream = self.template.generate( + **get_template_data( + entry, metadata, self.name, + default=DefaultGenshiDataProvider())).filter(removecomment) try: try: - return stream.render('text', encoding=self.encoding, + return stream.render('text', + encoding=Bcfg2.Options.setup.encoding, strip_whitespace=False) except TypeError: - return stream.render('text', encoding=self.encoding) + return stream.render('text', + encoding=Bcfg2.Options.setup.encoding) except UndefinedError: # a failure in a genshi expression _other_ than %{ python ... %} err = sys.exc_info()[1] @@ -196,8 +184,9 @@ class CfgGenshiGenerator(CfgGenerator): def handle_event(self, event): CfgGenerator.handle_event(self, event) try: - self.template = self.loader.load(self.name, cls=NewTextTemplate, - encoding=self.encoding) + self.template = \ + self.loader.load(self.name, cls=NewTextTemplate, + encoding=Bcfg2.Options.setup.encoding) except: raise PluginExecutionError("Failed to load template: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py index 3b6fc8fa0..886b3993b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py @@ -1,6 +1,6 @@ """ Handle info.xml files """ -from Bcfg2.Server.Plugin import PluginExecutionError, InfoXML +from Bcfg2.Server.Plugin import InfoXML from Bcfg2.Server.Plugins.Cfg import CfgInfo @@ -17,21 +17,9 @@ class CfgInfoXML(CfgInfo): __init__.__doc__ = CfgInfo.__init__.__doc__ def bind_info_to_entry(self, entry, metadata): - mdata = dict() - self.infoxml.pnode.Match(metadata, mdata, entry=entry) - if 'Info' not in mdata: - raise PluginExecutionError("Failed to set metadata for file %s" % - entry.get('name')) - self._set_info(entry, mdata['Info'][None]) + self.infoxml.BindEntry(entry, metadata) bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ def handle_event(self, event): self.infoxml.HandleEvent() handle_event.__doc__ = CfgInfo.handle_event.__doc__ - - def _set_info(self, entry, info): - CfgInfo._set_info(self, entry, info) - if '__children__' in info: - for child in info['__children__']: - entry.append(child) - _set_info.__doc__ = CfgInfo._set_info.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py new file mode 100644 index 000000000..e36ee78aa --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py @@ -0,0 +1,52 @@ +""" The CfgJinja2Generator allows you to use the `Jinja2 +<http://jinja.pocoo.org/>`_ templating system to generate +:ref:`server-plugins-generators-cfg` files. """ + +import Bcfg2.Options +from Bcfg2.Server.Plugin import PluginExecutionError, \ + DefaultTemplateDataProvider, get_template_data +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + +try: + from jinja2 import Template + HAS_JINJA2 = True +except ImportError: + HAS_JINJA2 = False + + +class DefaultJinja2DataProvider(DefaultTemplateDataProvider): + """ Template data provider for Jinja2 templates. Jinja2 and + Genshi currently differ over the value of the ``path`` variable, + which is why this is necessary. """ + + def get_template_data(self, entry, metadata, template): + rv = DefaultTemplateDataProvider.get_template_data(self, entry, + metadata, template) + rv['path'] = rv['name'] + return rv + + +class CfgJinja2Generator(CfgGenerator): + """ The CfgJinja2Generator allows you to use the `Jinja2 + <http://jinja.pocoo.org/>`_ templating system to generate + :ref:`server-plugins-generators-cfg` files. """ + + #: Handle .jinja2 files + __extensions__ = ['jinja2'] + + #: Low priority to avoid matching host- or group-specific + #: .crypt.jinja2 files + __priority__ = 50 + + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) + if not HAS_JINJA2: + raise PluginExecutionError("Jinja2 is not available") + __init__.__doc__ = CfgGenerator.__init__.__doc__ + + def get_data(self, entry, metadata): + template = Template(self.data.decode(Bcfg2.Options.setup.encoding)) + return template.render( + get_template_data(entry, metadata, self.name, + default=DefaultJinja2DataProvider())) + get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py deleted file mode 100644 index 5122d9aa1..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py +++ /dev/null @@ -1,46 +0,0 @@ -""" Handle info and :info files """ - -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugins.Cfg import CfgInfo - - -class CfgLegacyInfo(CfgInfo): - """ CfgLegacyInfo handles :file:`info` and :file:`:info` files for - :ref:`server-plugins-generators-cfg` """ - - #: Handle :file:`info` and :file:`:info` - __basenames__ = ['info', ':info'] - - #: CfgLegacyInfo is deprecated. Use - #: :class:`Bcfg2.Server.Plugins.Cfg.CfgInfoXML.CfgInfoXML` instead. - deprecated = True - - def __init__(self, path): - CfgInfo.__init__(self, path) - self.path = path - - #: The set of info metadata stored in the file - self.metadata = None - __init__.__doc__ = CfgInfo.__init__.__doc__ - - def bind_info_to_entry(self, entry, metadata): - self._set_info(entry, self.metadata) - bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ - - def handle_event(self, event): - if event.code2str() == 'deleted': - return - self.metadata = dict() - for line in open(self.path).readlines(): - match = Bcfg2.Server.Plugin.INFO_REGEX.match(line) - if not match: - self.logger.warning("Failed to parse line in %s: %s" % - (event.filename, line)) - continue - else: - for key, value in list(match.groupdict().items()): - if value: - self.metadata[key] = value - if ('mode' in self.metadata and len(self.metadata['mode']) == 3): - self.metadata['mode'] = "0%s" % self.metadata['mode'] - handle_event.__doc__ = CfgInfo.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index ac031461a..8cc3f7b21 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -3,18 +3,13 @@ import os import shutil import tempfile -import subprocess -from Bcfg2.Server.Plugin import PluginExecutionError, StructFile -from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, SETUP +import Bcfg2.Options +from Bcfg2.Utils import Executor +from Bcfg2.Server.Plugins.Cfg import XMLCfgCreator, CfgCreationError from Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator import CfgPublicKeyCreator -try: - import Bcfg2.Encryption - HAS_CRYPTO = True -except ImportError: - HAS_CRYPTO = False -class CfgPrivateKeyCreator(CfgCreator, StructFile): +class CfgPrivateKeyCreator(XMLCfgCreator): """The CfgPrivateKeyCreator creates SSH keys on the fly. """ #: Different configurations for different clients/groups can be @@ -24,36 +19,21 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): #: Handle XML specifications of private keys __basenames__ = ['privkey.xml'] - def __init__(self, fname): - CfgCreator.__init__(self, fname) - StructFile.__init__(self, fname) + cfg_section = "sshkeys" + options = [ + Bcfg2.Options.Option( + cf=("sshkeys", "category"), dest="sshkeys_category", + help="Metadata category that generated SSH keys are specific to"), + Bcfg2.Options.Option( + cf=("sshkeys", "passphrase"), dest="sshkeys_passphrase", + help="Passphrase used to encrypt generated SSH private keys")] + def __init__(self, fname): + XMLCfgCreator.__init__(self, fname) pubkey_path = os.path.dirname(self.name) + ".pub" pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path)) self.pubkey_creator = CfgPublicKeyCreator(pubkey_name) - - @property - def category(self): - """ The name of the metadata category that generated keys are - specific to """ - if (SETUP.cfp.has_section("sshkeys") and - SETUP.cfp.has_option("sshkeys", "category")): - return SETUP.cfp.get("sshkeys", "category") - return None - - @property - def passphrase(self): - """ The passphrase used to encrypt private keys """ - if (HAS_CRYPTO and - SETUP.cfp.has_section("sshkeys") and - SETUP.cfp.has_option("sshkeys", "passphrase")): - return Bcfg2.Encryption.get_passphrases(SETUP)[ - SETUP.cfp.get("sshkeys", "passphrase")] - return None - - def handle_event(self, event): - CfgCreator.handle_event(self, event) - StructFile.HandleEvent(self, event) + self.cmd = Executor() def _gen_keypair(self, metadata, spec=None): """ Generate a keypair according to the given client medata @@ -100,62 +80,22 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): log_cmd.append("''") self.debug_log("Cfg: Generating new SSH key pair: %s" % " ".join(log_cmd)) - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - err = proc.communicate()[1] - if proc.wait(): + result = self.cmd.run(cmd) + if not result.success: raise CfgCreationError("Cfg: Failed to generate SSH key pair " "at %s for %s: %s" % - (filename, metadata.hostname, err)) - elif err: + (filename, metadata.hostname, + result.error)) + elif result.stderr: self.logger.warning("Cfg: Generated SSH key pair at %s for %s " "with errors: %s" % (filename, metadata.hostname, - err)) + result.stderr)) return filename except: shutil.rmtree(tempdir) raise - def get_specificity(self, metadata, spec=None): - """ Get config settings for key generation specificity - (per-host or per-group). - - :param metadata: The client metadata to create data for - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param spec: The key specification to follow when creating the - keys. This should be an XML document that only - contains key specification data that applies to - the given client metadata, and may be obtained by - doing ``self.XMLMatch(metadata)`` - :type spec: lxml.etree._Element - :returns: dict - A dict of specificity arguments suitable for - passing to - :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.write_data` - or - :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.get_filename` - """ - if spec is None: - spec = self.XMLMatch(metadata) - category = spec.get("category", self.category) - if category is None: - per_host_default = "true" - else: - per_host_default = "false" - per_host = spec.get("perhost", per_host_default).lower() == "true" - - specificity = dict(host=metadata.hostname) - if category and not per_host: - group = metadata.group_in_category(category) - if group: - specificity = dict(group=group, - prio=int(spec.get("priority", 50))) - else: - self.logger.info("Cfg: %s has no group in category %s, " - "creating host-specific key" % - (metadata.hostname, category)) - return specificity - # pylint: disable=W0221 def create_data(self, entry, metadata): """ Create data for the given entry on the given client @@ -168,7 +108,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): :returns: string - The private key data """ spec = self.XMLMatch(metadata) - specificity = self.get_specificity(metadata, spec) + specificity = self.get_specificity(metadata) filename = self._gen_keypair(metadata, spec) try: @@ -182,63 +122,8 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): # encrypt the private key, write to the proper place, and # return it privkey = open(filename).read() - if HAS_CRYPTO and self.passphrase: - self.debug_log("Cfg: Encrypting key data at %s" % filename) - privkey = Bcfg2.Encryption.ssl_encrypt( - privkey, - self.passphrase, - algorithm=Bcfg2.Encryption.get_algorithm(SETUP)) - specificity['ext'] = '.crypt' - self.write_data(privkey, **specificity) return privkey finally: shutil.rmtree(os.path.dirname(filename)) # pylint: enable=W0221 - - def Index(self): - StructFile.Index(self) - if HAS_CRYPTO: - for el in self.xdata.xpath("//*[@encrypted]"): - try: - el.text = self._decrypt(el).encode('ascii', - 'xmlcharrefreplace') - except UnicodeDecodeError: - self.logger.info("Cfg: Decrypted %s to gibberish, skipping" - % el.tag) - except Bcfg2.Encryption.EVPError: - default_strict = SETUP.cfp.get( - Bcfg2.Encryption.CFG_SECTION, "decrypt", - default="strict") - strict = self.xdata.get("decrypt", - default_strict) == "strict" - msg = "Cfg: Failed to decrypt %s element in %s" % \ - (el.tag, self.name) - if strict: - raise PluginExecutionError(msg) - else: - self.logger.debug(msg) - - def _decrypt(self, element): - """ Decrypt a single encrypted element """ - if not element.text or not element.text.strip(): - return - passes = Bcfg2.Encryption.get_passphrases(SETUP) - try: - passphrase = passes[element.get("encrypted")] - try: - return Bcfg2.Encryption.ssl_decrypt( - element.text, - passphrase, - algorithm=Bcfg2.Encryption.get_algorithm(SETUP)) - except Bcfg2.Encryption.EVPError: - # error is raised below - pass - except KeyError: - # bruteforce_decrypt raises an EVPError with a sensible - # error message, so we just let it propagate up the stack - return Bcfg2.Encryption.bruteforce_decrypt( - element.text, - passphrases=passes.values(), - algorithm=Bcfg2.Encryption.get_algorithm(SETUP)) - raise Bcfg2.Encryption.EVPError("Failed to decrypt") diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py index 4bd8690ed..3f2d1030b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py @@ -8,7 +8,7 @@ import tempfile import lxml.etree from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import StructFile, PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, CFG +from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, get_cfg class CfgPublicKeyCreator(CfgCreator, StructFile): @@ -21,17 +21,20 @@ class CfgPublicKeyCreator(CfgCreator, StructFile): creation of a keypair when a public key is created. """ #: Different configurations for different clients/groups can be - #: handled with Client and Group tags within privkey.xml + #: handled with Client and Group tags within pubkey.xml __specific__ = False #: Handle XML specifications of private keys __basenames__ = ['pubkey.xml'] + #: No text content on any tags, so encryption support disabled + encryption = False + def __init__(self, fname): CfgCreator.__init__(self, fname) StructFile.__init__(self, fname) - self.cfg = CFG - self.core = CFG.core + self.cfg = get_cfg() + self.core = self.cfg.core self.cmd = Executor() def create_data(self, entry, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py new file mode 100644 index 000000000..92fcc4cd8 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py @@ -0,0 +1,255 @@ +""" Cfg creator that creates SSL certs """ + +import os +import sys +import tempfile +import lxml.etree +import Bcfg2.Options +from Bcfg2.Utils import Executor +from Bcfg2.Compat import ConfigParser +from Bcfg2.Server.FileMonitor import get_fam +from Bcfg2.Server.Plugin import PluginExecutionError +from Bcfg2.Server.Plugins.Cfg import CfgCreationError, XMLCfgCreator, \ + CfgCreator, CfgVerifier, CfgVerificationError, get_cfg + + +class CfgSSLCACertCreator(XMLCfgCreator, CfgVerifier): + """ This class acts as both a Cfg creator that creates SSL certs, + and as a Cfg verifier that verifies SSL certs. """ + + #: Different configurations for different clients/groups can be + #: handled with Client and Group tags within pubkey.xml + __specific__ = False + + #: Handle XML specifications of private keys + __basenames__ = ['sslcert.xml'] + + cfg_section = "sslca" + options = [ + Bcfg2.Options.Option( + cf=("sslca", "category"), dest="sslca_category", + help="Metadata category that generated SSL keys are specific to"), + Bcfg2.Options.Option( + cf=("sslca", "passphrase"), dest="sslca_passphrase", + help="Passphrase used to encrypt generated SSL keys"), + Bcfg2.Options.WildcardSectionGroup( + Bcfg2.Options.PathOption( + cf=("sslca_*", "config"), + help="Path to the openssl config for the CA"), + Bcfg2.Options.Option( + cf=("sslca_*", "passphrase"), + help="Passphrase for the CA private key"), + Bcfg2.Options.PathOption( + cf=("sslca_*", "chaincert"), + help="Path to the SSL chaining certificate for verification"), + Bcfg2.Options.BooleanOption( + cf=("sslca_*", "root_ca"), + help="Whether or not <chaincert> is a root CA (as opposed to " + "an intermediate cert"), + prefix="")] + + def __init__(self, fname): + XMLCfgCreator.__init__(self, fname) + CfgVerifier.__init__(self, fname, None) + self.cmd = Executor() + self.cfg = get_cfg() + + def build_req_config(self, metadata): + """ Generates a temporary openssl configuration file that is + used to generate the required certificate request. """ + fd, fname = tempfile.mkstemp() + cfp = ConfigParser.ConfigParser({}) + cfp.optionxform = str + defaults = dict( + req=dict( + default_md='sha1', + distinguished_name='req_distinguished_name', + req_extensions='v3_req', + x509_extensions='v3_req', + prompt='no'), + req_distinguished_name=dict(), + v3_req=dict(subjectAltName='@alt_names'), + alt_names=dict()) + for section in list(defaults.keys()): + cfp.add_section(section) + for key in defaults[section]: + cfp.set(section, key, defaults[section][key]) + spec = self.XMLMatch(metadata) + cert = spec.find("Cert") + altnamenum = 1 + altnames = spec.findall('subjectAltName') + altnames.extend(list(metadata.aliases)) + altnames.append(metadata.hostname) + for altname in altnames: + cfp.set('alt_names', 'DNS.' + str(altnamenum), altname) + altnamenum += 1 + for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: + if cert.get(item): + cfp.set('req_distinguished_name', item, cert.get(item)) + cfp.set('req_distinguished_name', 'CN', metadata.hostname) + self.debug_log("Cfg: Writing temporary CSR config to %s" % fname) + try: + cfp.write(os.fdopen(fd, 'w')) + except IOError: + raise CfgCreationError("Cfg: Failed to write temporary CSR config " + "file: %s" % sys.exc_info()[1]) + return fname + + def build_request(self, keyfile, metadata): + """ Create the certificate request """ + req_config = self.build_req_config(metadata) + try: + fd, req = tempfile.mkstemp() + os.close(fd) + cert = self.XMLMatch(metadata).find("Cert") + days = cert.get("days", "365") + cmd = ["openssl", "req", "-new", "-config", req_config, + "-days", days, "-key", keyfile, "-text", "-out", req] + result = self.cmd.run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate CSR: %s" % + result.error) + return req + finally: + try: + os.unlink(req_config) + except OSError: + self.logger.error("Cfg: Failed to unlink temporary CSR " + "config: %s" % sys.exc_info()[1]) + + def get_ca(self, name): + """ get a dict describing a CA from the config file """ + rv = dict() + prefix = "sslca_%s_" % name + for attr in dir(Bcfg2.Options.setup): + if attr.startswith(prefix): + rv[attr[len(prefix):]] = getattr(Bcfg2.Options.setup, attr) + return rv + + def create_data(self, entry, metadata): + """ generate a new cert """ + self.logger.info("Cfg: Generating new SSL cert for %s" % self.name) + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get('ca', 'default')) + req = self.build_request(self._get_keyfile(cert, metadata), metadata) + try: + days = cert.get('days', '365') + cmd = ["openssl", "ca", "-config", ca['config'], "-in", req, + "-days", days, "-batch"] + passphrase = ca.get('passphrase') + if passphrase: + cmd.extend(["-passin", "pass:%s" % passphrase]) + result = self.cmd.run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate cert: %s" % + result.error) + except KeyError: + raise CfgCreationError("Cfg: [sslca_%s] section has no 'config' " + "option" % cert.get('ca', 'default')) + finally: + try: + os.unlink(req) + except OSError: + self.logger.error("Cfg: Failed to unlink temporary CSR: %s " % + sys.exc_info()[1]) + data = result.stdout + if cert.get('append_chain') and 'chaincert' in ca: + data += open(ca['chaincert']).read() + + self.write_data(data, **self.get_specificity(metadata)) + return data + + def verify_entry(self, entry, metadata, data): + fd, fname = tempfile.mkstemp() + self.debug_log("Cfg: Writing SSL cert %s to temporary file %s for " + "verification" % (entry.get("name"), fname)) + os.fdopen(fd, 'w').write(data) + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get('ca', 'default')) + try: + if ca.get('chaincert'): + self.verify_cert_against_ca(fname, entry, metadata) + self.verify_cert_against_key(fname, + self._get_keyfile(cert, metadata)) + finally: + os.unlink(fname) + + def _get_keyfile(self, cert, metadata): + """ Given a <Cert/> element and client metadata, return the + full path to the file on the filesystem that the key lives in.""" + keypath = cert.get("key") + eset = self.cfg.entries[keypath] + try: + return eset.best_matching(metadata).name + except PluginExecutionError: + # SSL key needs to be created + try: + creator = eset.best_matching(metadata, + eset.get_handlers(metadata, + CfgCreator)) + except PluginExecutionError: + raise CfgCreationError("Cfg: No SSL key or key creator " + "defined for %s" % keypath) + + keyentry = lxml.etree.Element("Path", name=keypath) + creator.create_data(keyentry, metadata) + + tries = 0 + while True: + if tries >= 10: + raise CfgCreationError("Cfg: Timed out waiting for event " + "on SSL key at %s" % keypath) + get_fam().handle_events_in_interval(1) + try: + return eset.best_matching(metadata).name + except PluginExecutionError: + tries += 1 + continue + + def verify_cert_against_ca(self, filename, entry, metadata): + """ + check that a certificate validates against the ca cert, + and that it has not expired. + """ + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get("ca", "default")) + chaincert = ca.get('chaincert') + cmd = ["openssl", "verify"] + is_root = ca.get('root_ca', "false").lower() == 'true' + if is_root: + cmd.append("-CAfile") + else: + # verifying based on an intermediate cert + cmd.extend(["-purpose", "sslserver", "-untrusted"]) + cmd.extend([chaincert, filename]) + self.debug_log("Cfg: Verifying %s against CA" % entry.get("name")) + result = self.cmd.run(cmd) + if result.stdout == cert + ": OK\n": + self.debug_log("Cfg: %s verified successfully against CA" % + entry.get("name")) + else: + raise CfgVerificationError("%s failed verification against CA: %s" + % (entry.get("name"), result.error)) + + def _get_modulus(self, fname, ftype="x509"): + """ get the modulus from the given file """ + cmd = ["openssl", ftype, "-noout", "-modulus", "-in", fname] + self.debug_log("Cfg: Getting modulus of %s for verification: %s" % + (fname, " ".join(cmd))) + result = self.cmd.run(cmd) + if not result.success: + raise CfgVerificationError("Failed to get modulus of %s: %s" % + (fname, result.error)) + return result.stdout.strip() + + def verify_cert_against_key(self, filename, keyfile): + """ check that a certificate validates against its private + key. """ + cert = self._get_modulus(filename) + key = self._get_modulus(keyfile, ftype="rsa") + if cert == key: + self.debug_log("Cfg: %s verified successfully against key %s" % + (filename, keyfile)) + else: + raise CfgVerificationError("%s failed verification against key %s" + % (filename, keyfile)) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py new file mode 100644 index 000000000..a158302be --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py @@ -0,0 +1,36 @@ +""" Cfg creator that creates SSL keys """ + +from Bcfg2.Utils import Executor +from Bcfg2.Server.Plugins.Cfg import CfgCreationError, XMLCfgCreator + + +class CfgSSLCAKeyCreator(XMLCfgCreator): + """ Cfg creator that creates SSL keys """ + + #: Different configurations for different clients/groups can be + #: handled with Client and Group tags within sslkey.xml + __specific__ = False + + __basenames__ = ["sslkey.xml"] + + cfg_section = "sslca" + + def create_data(self, entry, metadata): + self.logger.info("Cfg: Generating new SSL key for %s" % self.name) + spec = self.XMLMatch(metadata) + key = spec.find("Key") + if not key: + key = dict() + ktype = key.get('type', 'rsa') + bits = key.get('bits', '2048') + if ktype == 'rsa': + cmd = ["openssl", "genrsa", bits] + elif ktype == 'dsa': + cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] + result = Executor().run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate key %s for %s: %s" % + (self.name, metadata.hostname, + result.error)) + self.write_data(result.stdout, **self.get_specificity(metadata)) + return result.stdout diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index c6e2d0acb..5dc3d98eb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -3,60 +3,39 @@ import re import os import sys -import stat import errno import operator import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin -import Bcfg2.Server.Lint -from fnmatch import fnmatch from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 -from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ - any, oct_mode +from Bcfg2.Compat import u_str, unicode, b64encode, any, walk_packages # pylint: enable=W0622 -#: SETUP contains a reference to the -#: :class:`Bcfg2.Options.OptionParser` created by the Bcfg2 core for -#: parsing command-line and config file options. -#: :class:`Bcfg2.Server.Plugins.Cfg.Cfg` stores it in a module global -#: so that the handler objects can access it, because there is no other -#: facility for passing a setup object from a -#: :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` to its -#: :class:`Bcfg2.Server.Plugin.helpers.EntrySet` objects and thence to -#: the EntrySet children. -SETUP = None - -#: CFG is a reference to the :class:`Bcfg2.Server.Plugins.Cfg.Cfg` -#: plugin object created by the Bcfg2 core. This is provided so that -#: the handler objects can access it as necessary, since the existing -#: :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` and -#: :class:`Bcfg2.Server.Plugin.helpers.EntrySet` classes have no -#: facility for passing it otherwise. -CFG = None - -_HANDLERS = [] - - -def handlers(): - """ A list of Cfg handler classes. Loading the handlers must - be done at run-time, not at compile-time, or it causes a - circular import and Bad Things Happen.""" - if not _HANDLERS: - for submodule in walk_packages(path=__path__, prefix=__name__ + "."): - mname = submodule[1].rsplit('.', 1)[-1] - module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, - mname) - hdlr = getattr(module, mname) - if issubclass(hdlr, CfgBaseFileMatcher): - _HANDLERS.append(hdlr) - _HANDLERS.sort(key=operator.attrgetter("__priority__")) - return _HANDLERS - - -class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, - Bcfg2.Server.Plugin.Debuggable): +try: + import Bcfg2.Server.Encryption + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False + +_handlers = [m[1] # pylint: disable=C0103 + for m in walk_packages(path=__path__)] + +_CFG = None + + +def get_cfg(): + """ Get the :class:`Bcfg2.Server.Plugins.Cfg.Cfg` plugin object + created by the Bcfg2 core. This is provided so that the handler + objects can access it as necessary, since the existing + :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` and + :class:`Bcfg2.Server.Plugin.helpers.EntrySet` classes have no + facility for passing it otherwise.""" + return _CFG + + +class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): """ .. currentmodule:: Bcfg2.Server.Plugins.Cfg CfgBaseFileMatcher is the parent class for all Cfg handler @@ -100,13 +79,10 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, #: Flag to indicate an experimental handler. experimental = False - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): if not self.__specific__ and not specific: specific = Bcfg2.Server.Plugin.Specificity(all=True) - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) - Bcfg2.Server.Plugin.Debuggable.__init__(self) - self.encoding = encoding + Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific) __init__.__doc__ = Bcfg2.Server.Plugin.SpecificData.__init__.__doc__ + \ """ .. ----- @@ -197,7 +173,7 @@ class CfgGenerator(CfgBaseFileMatcher): client. See :class:`Bcfg2.Server.Plugin.helpers.EntrySet` for more details on how the best handler is chosen.""" - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # we define an __init__ that just calls the parent __init__, # so that we can set the docstring on __init__ to something # different from the parent __init__ -- namely, the parent @@ -205,7 +181,7 @@ class CfgGenerator(CfgBaseFileMatcher): # which we use to delineate the actual docs from the # .. autoattribute hacks we have to do to get private # attributes included in sphinx 1.0 """ - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def get_data(self, entry, metadata): # pylint: disable=W0613 @@ -225,9 +201,9 @@ class CfgFilter(CfgBaseFileMatcher): """ CfgFilters modify the initial content of a file after it has been generated by a :class:`Bcfg2.Server.Plugins.Cfg.CfgGenerator`. """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # see comment on CfgGenerator.__init__ above - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def modify_data(self, entry, metadata, data): @@ -249,10 +225,7 @@ class CfgFilter(CfgBaseFileMatcher): class CfgInfo(CfgBaseFileMatcher): """ CfgInfo handlers provide metadata (owner, group, paranoid, - etc.) for a file entry. - - .. private-include: _set_info - """ + etc.) for a file entry. """ #: Whether or not the files handled by this handler are permitted #: to have specificity indicators in their filenames -- e.g., @@ -268,7 +241,7 @@ class CfgInfo(CfgBaseFileMatcher): .. ----- .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgInfo.__specific__ """ - CfgBaseFileMatcher.__init__(self, fname, None, None) + CfgBaseFileMatcher.__init__(self, fname, None) def bind_info_to_entry(self, entry, metadata): """ Assign the appropriate attributes to the entry, modifying @@ -282,20 +255,6 @@ class CfgInfo(CfgBaseFileMatcher): """ raise NotImplementedError - def _set_info(self, entry, info): - """ Helper function to assign a dict of info attributes to an - entry object. ``entry`` is modified in-place. - - :param entry: The abstract entry to bind the info to - :type entry: lxml.etree._Element - :param info: A dict of attribute: value pairs - :type info: dict - :returns: None - """ - for key, value in list(info.items()): - if not key.startswith("__"): - entry.attrib[key] = value - class CfgVerifier(CfgBaseFileMatcher): """ CfgVerifier handlers validate entry data once it has been @@ -305,9 +264,9 @@ class CfgVerifier(CfgBaseFileMatcher): etc.), or both. """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # see comment on CfgGenerator.__init__ above - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def verify_entry(self, entry, metadata, data): @@ -338,18 +297,15 @@ class CfgCreator(CfgBaseFileMatcher): #: file, and are thus not specific __specific__ = False - #: The CfgCreator interface is experimental at this time - experimental = True - def __init__(self, fname): """ :param name: The full path to the file :type name: string .. ----- - .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgCreator.__specific__ + .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgInfo.__specific__ """ - CfgBaseFileMatcher.__init__(self, fname, None, None) + CfgBaseFileMatcher.__init__(self, fname, None) def create_data(self, entry, metadata): """ Create new data for the given entry and write it to disk @@ -369,7 +325,9 @@ class CfgCreator(CfgBaseFileMatcher): ``host`` is given, it will be host-specific. It will be group-specific if ``group`` and ``prio`` are given. If neither ``host`` nor ``group`` is given, the filename will be - non-specific. + non-specific. In general, this will be called as:: + + self.get_filename(**self.get_specificity(metadata)) :param host: The file applies to the given host :type host: bool @@ -400,6 +358,9 @@ class CfgCreator(CfgBaseFileMatcher): written as a host-specific file, or as a group-specific file if ``group`` and ``prio`` are given. If neither ``host`` nor ``group`` is given, it will be written as a non-specific file. + In general, this will be called as:: + + self.write_data(data, **self.get_specificity(metadata)) :param data: The data to write :type data: string @@ -419,7 +380,7 @@ class CfgCreator(CfgBaseFileMatcher): :raises: :exc:`Bcfg2.Server.Plugins.Cfg.CfgCreationError` """ fileloc = self.get_filename(host=host, group=group, prio=prio, ext=ext) - self.debug_log("%s: Writing new file %s" % (self.name, fileloc)) + self.debug_log("Cfg: Writing new file %s" % fileloc) try: os.makedirs(os.path.dirname(fileloc)) except OSError: @@ -435,6 +396,95 @@ class CfgCreator(CfgBaseFileMatcher): raise CfgCreationError("Could not write %s: %s" % (fileloc, err)) +class XMLCfgCreator(CfgCreator, # pylint: disable=W0223 + Bcfg2.Server.Plugin.StructFile): + """ A CfgCreator that uses XML to describe how data should be + generated. """ + + #: Whether or not the created data from this class can be + #: encrypted + encryptable = True + + #: Encryption and creation settings can be stored in bcfg2.conf, + #: either under the [cfg] section, or under the named section. + cfg_section = None + + def __init__(self, name): + CfgCreator.__init__(self, name) + Bcfg2.Server.Plugin.StructFile.__init__(self, name) + + def handle_event(self, event): + CfgCreator.handle_event(self, event) + Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event) + + @property + def passphrase(self): + """ The passphrase used to encrypt created data """ + if self.cfg_section: + localopt = "%s_passphrase" % self.cfg_section + passphrase = getattr(Bcfg2.Options.setup, localopt, + Bcfg2.Options.setup.cfg_passphrase) + else: + passphrase = Bcfg2.Options.setup.cfg_passphrase + if passphrase is None: + return None + try: + return Bcfg2.Options.setup.passphrases[passphrase] + except KeyError: + raise CfgCreationError("%s: No such passphrase: %s" % + (self.__class__.__name__, passphrase)) + + @property + def category(self): + """ The category to which created data is specific """ + if self.cfg_section: + localopt = "%s_category" % self.cfg_section + return getattr(Bcfg2.Options.setup, localopt, + Bcfg2.Options.setup.cfg_category) + else: + return Bcfg2.Options.setup.cfg_category + + def write_data(self, data, host=None, group=None, prio=0, ext=''): + if HAS_CRYPTO and self.encryptable and self.passphrase: + self.debug_log("Cfg: Encrypting created data") + data = Bcfg2.Server.Encryption.ssl_encrypt(data, self.passphrase) + ext = '.crypt' + CfgCreator.write_data(self, data, host=host, group=group, prio=prio, + ext=ext) + + def get_specificity(self, metadata): + """ Get config settings for key generation specificity + (per-host or per-group). + + :param metadata: The client metadata to create data for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: dict - A dict of specificity arguments suitable for + passing to + :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.write_data` + or + :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.get_filename` + """ + category = self.xdata.get("category", self.category) + if category is None: + per_host_default = "true" + else: + per_host_default = "false" + per_host = self.xdata.get("perhost", + per_host_default).lower() == "true" + + specificity = dict(host=metadata.hostname) + if category and not per_host: + group = metadata.group_in_category(category) + if group: + specificity = dict(group=group, + prio=int(self.xdata.get("priority", 50))) + else: + self.logger.info("Cfg: %s has no group in category %s, " + "creating host-specific data" % + (metadata.hostname, category)) + return specificity + + class CfgVerificationError(Exception): """ Raised by :func:`Bcfg2.Server.Plugins.Cfg.CfgVerifier.verify_entry` when an @@ -453,37 +503,27 @@ class CfgDefaultInfo(CfgInfo): """ :class:`Bcfg2.Server.Plugins.Cfg.Cfg` handler that supplies a default set of file metadata """ - def __init__(self, defaults): + def __init__(self): CfgInfo.__init__(self, '') - self.defaults = defaults __init__.__doc__ = CfgInfo.__init__.__doc__.split(".. -----")[0] - def bind_info_to_entry(self, entry, metadata): - self._set_info(entry, self.defaults) + def bind_info_to_entry(self, entry, _): + for key, value in Bcfg2.Server.Plugin.default_path_metadata().items(): + entry.attrib[key] = value bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ -#: A :class:`CfgDefaultInfo` object instantiated with -#: :attr:`Bcfg2.Server.Plugin.helper.DEFAULT_FILE_METADATA` as its -#: default metadata. This is used to set a default file metadata set -#: on an entry before a "real" :class:`CfgInfo` handler applies its -#: metadata to the entry. -DEFAULT_INFO = CfgDefaultInfo(Bcfg2.Server.Plugin.DEFAULT_FILE_METADATA) - -class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, - Bcfg2.Server.Plugin.Debuggable): +class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ Handle a collection of host- and group-specific Cfg files with multiple different Cfg handlers in a single directory. """ - def __init__(self, basename, path, entry_type, encoding): - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, - entry_type, encoding) - Bcfg2.Server.Plugin.Debuggable.__init__(self) + def __init__(self, basename, path, entry_type): + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type) self.specific = None __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ def set_debug(self, debug): - rv = Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) + rv = Bcfg2.Server.Plugin.EntrySet.set_debug(self, debug) for entry in self.entries.values(): entry.set_debug(debug) return rv @@ -504,7 +544,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # process a bogus changed event like a created return - for hdlr in handlers(): + for hdlr in Bcfg2.Options.setup.cfg_handlers: if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -597,7 +637,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # most specific to least specific. data = fltr.modify_data(entry, metadata, data) - if SETUP['validate']: + if Bcfg2.Options.setup.cfg_validation: try: self._validate_data(entry, metadata, data) except CfgVerificationError: @@ -613,7 +653,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, if not isinstance(data, unicode): if not isinstance(data, str): data = data.decode('utf-8') - data = u_str(data, self.encoding) + data = u_str(data, Bcfg2.Options.setup.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) @@ -652,7 +692,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, rv = [] for ent in self.entries.values(): if (isinstance(ent, handler_type) and - (not ent.__specific__ or ent.specific.matches(metadata))): + (not ent.__specific__ or ent.specific.matches(metadata))): rv.append(ent) return rv @@ -668,7 +708,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, :returns: None """ info_handlers = self.get_handlers(metadata, CfgInfo) - DEFAULT_INFO.bind_info_to_entry(entry, metadata) + CfgDefaultInfo().bind_info_to_entry(entry, metadata) if len(info_handlers) > 1: self.logger.error("More than one info supplier found for %s: %s" % (entry.get("name"), info_handlers)) @@ -717,13 +757,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # raises an appropriate exception return (self._create_data(entry, metadata), None) - if entry.get('mode').lower() == 'inherit': - # use on-disk permissions - self.logger.warning("Cfg: %s: Use of mode='inherit' is deprecated" - % entry.get("name")) - fname = os.path.join(self.path, generator.name) - entry.set('mode', - oct_mode(stat.S_IMODE(os.stat(fname).st_mode))) try: return (generator.get_data(entry, metadata), generator) except: @@ -801,10 +834,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, self.logger.error(msg) raise PluginExecutionError(msg) try: - etext = new_entry['text'].encode(self.encoding) + etext = new_entry['text'].encode(Bcfg2.Options.setup.encoding) except: msg = "Cfg: Cannot encode content of %s as %s" % \ - (name, self.encoding) + (name, Bcfg2.Options.setup.encoding) self.logger.error(msg) raise PluginExecutionError(msg) open(name, 'w').write(etext) @@ -812,13 +845,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, badattr = [attr for attr in ['owner', 'group', 'mode'] if attr in new_entry] if badattr: - # check for info files and inform user of their removal - for ifile in ['info', ':info']: - info = os.path.join(self.path, ifile) - if os.path.exists(info): - self.logger.info("Removing %s and replacing with info.xml" - % info) - os.remove(info) metadata_updates = {} metadata_updates.update(self.metadata) for attr in badattr: @@ -836,28 +862,45 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, flag=log) +class CfgHandlerAction(Bcfg2.Options.ComponentAction): + """ Option parser action to load Cfg handlers """ + bases = ['Bcfg2.Server.Plugins.Cfg'] + + class Cfg(Bcfg2.Server.Plugin.GroupSpool, Bcfg2.Server.Plugin.PullTarget): """ The Cfg plugin provides a repository to describe configuration file contents for clients. In its simplest form, the Cfg repository is just a directory tree modeled off of the directory tree on your client - machines. - """ + machines. """ __author__ = 'bcfg-dev@mcs.anl.gov' es_cls = CfgEntrySet es_child_cls = Bcfg2.Server.Plugin.SpecificData - def __init__(self, core, datastore): - global SETUP, CFG # pylint: disable=W0603 - Bcfg2.Server.Plugin.GroupSpool.__init__(self, core, datastore) + options = Bcfg2.Server.Plugin.GroupSpool.options + [ + Bcfg2.Options.BooleanOption( + '--cfg-validation', cf=('cfg', 'validation'), default=True, + help='Run validation on Cfg files'), + Bcfg2.Options.Option( + cf=('cfg', 'category'), dest="cfg_category", + help='The default name of the metadata category that created data ' + 'is specific to'), + Bcfg2.Options.Option( + cf=('cfg', 'passphrase'), dest="cfg_passphrase", + help='The default passphrase name used to encrypt created data'), + Bcfg2.Options.Option( + cf=("cfg", "handlers"), dest="cfg_handlers", + help="Cfg handlers to load", + type=Bcfg2.Options.Types.comma_list, action=CfgHandlerAction, + default=_handlers)] + + def __init__(self, core): + global _CFG # pylint: disable=W0603 + Bcfg2.Server.Plugin.GroupSpool.__init__(self, core) Bcfg2.Server.Plugin.PullTarget.__init__(self) - - CFG = self - - SETUP = core.setup - if 'validate' not in SETUP: - SETUP.add_option('validate', Bcfg2.Options.CFG_VALIDATION) - SETUP.reparse() + Bcfg2.Options.setup.cfg_handlers.sort( + key=operator.attrgetter("__priority__")) + _CFG = self __init__.__doc__ = Bcfg2.Server.Plugin.GroupSpool.__init__.__doc__ def has_generator(self, entry, metadata): @@ -891,127 +934,3 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, log) AcceptPullData.__doc__ = \ Bcfg2.Server.Plugin.PullTarget.AcceptPullData.__doc__ - - -class CfgLint(Bcfg2.Server.Lint.ServerPlugin): - """ warn about usage of .cat and .diff files """ - - def Run(self): - for basename, entry in list(self.core.plugins['Cfg'].entries.items()): - self.check_delta(basename, entry) - self.check_pubkey(basename, entry) - self.check_missing_files() - self.check_conflicting_handlers() - - @classmethod - def Errors(cls): - return {"cat-file-used": "warning", - "diff-file-used": "warning", - "no-pubkey-xml": "warning", - "unknown-cfg-files": "error", - "extra-cfg-files": "error", - "multiple-global-handlers": "error"} - - def check_delta(self, basename, entry): - """ check that no .cat or .diff files are in use """ - for fname, handler in entry.entries.items(): - path = handler.name - if self.HandlesFile(path) and isinstance(handler, CfgFilter): - extension = fname.split(".")[-1] - if extension in ["cat", "diff"]: - self.LintError("%s-file-used" % extension, - "%s file used on %s: %s" % (extension, - basename, - fname)) - - def check_pubkey(self, basename, entry): - """ check that privkey.xml files have corresponding pubkey.xml - files """ - if "privkey.xml" not in entry.entries: - return - privkey = entry.entries["privkey.xml"] - if not self.HandlesFile(privkey.name): - return - - pubkey = basename + ".pub" - if pubkey not in self.core.plugins['Cfg'].entries: - self.LintError("no-pubkey-xml", - "%s has no corresponding pubkey.xml at %s" % - (basename, pubkey)) - else: - pubset = self.core.plugins['Cfg'].entries[pubkey] - if "pubkey.xml" not in pubset.entries: - self.LintError("no-pubkey-xml", - "%s has no corresponding pubkey.xml at %s" % - (basename, pubkey)) - - def _list_path_components(self, path): - """ Get a list of all components of a path. E.g., - ``self._list_path_components("/foo/bar/foobaz")`` would return - ``["foo", "bar", "foo", "baz"]``. The list is not guaranteed - to be in order.""" - rv = [] - remaining, component = os.path.split(path) - while component != '': - rv.append(component) - remaining, component = os.path.split(remaining) - return rv - - def check_conflicting_handlers(self): - """ Check that a single entryset doesn't have multiple - non-specific (i.e., 'all') handlers. """ - cfg = self.core.plugins['Cfg'] - for eset in cfg.entries.values(): - alls = [e for e in eset.entries.values() - if (e.specific.all and - issubclass(e.__class__, CfgGenerator))] - if len(alls) > 1: - self.LintError("multiple-global-handlers", - "%s has multiple global handlers: %s" % - (eset.path, ", ".join(os.path.basename(e.name) - for e in alls))) - - def check_missing_files(self): - """ check that all files on the filesystem are known to Cfg """ - cfg = self.core.plugins['Cfg'] - - # first, collect ignore patterns from handlers - ignore = set() - for hdlr in handlers(): - ignore.update(hdlr.__ignore__) - - # next, get a list of all non-ignored files on the filesystem - all_files = set() - for root, _, files in os.walk(cfg.data): - for fname in files: - fpath = os.path.join(root, fname) - # check against the handler ignore patterns and the - # global FAM ignore list - if (not any(fname.endswith("." + i) for i in ignore) and - not any(fnmatch(fpath, p) - for p in self.config['ignore']) and - not any(fnmatch(c, p) - for p in self.config['ignore'] - for c in self._list_path_components(fpath))): - all_files.add(fpath) - - # next, get a list of all files known to Cfg - cfg_files = set() - for root, eset in cfg.entries.items(): - cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) - for fname in eset.entries.keys()) - - # finally, compare the two - unknown_files = all_files - cfg_files - extra_files = cfg_files - all_files - if unknown_files: - self.LintError( - "unknown-cfg-files", - "Files on the filesystem could not be understood by Cfg: %s" % - "; ".join(unknown_files)) - if extra_files: - self.LintError( - "extra-cfg-files", - "Cfg has entries for files that do not exist on the " - "filesystem: %s\nThis is probably a bug." % - "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py index 22cacaa76..35bff0835 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cvs.py +++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py @@ -1,7 +1,7 @@ """ The Cvs plugin provides a revision interface for Bcfg2 repos using cvs. """ -from subprocess import Popen, PIPE +from Bcfg2.Utils import Executor import Bcfg2.Server.Plugin @@ -11,22 +11,19 @@ class Cvs(Bcfg2.Server.Plugin.Version): __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = "CVSROOT" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) + self.cmd = Executor() self.logger.debug("Initialized cvs plugin with CVS directory %s" % self.vcs_path) def get_revision(self): """Read cvs revision information for the Bcfg2 repository.""" + result = self.cmd.run(["env LC_ALL=C", "cvs", "log"], + shell=True, cwd=Bcfg2.Options.setup.vcs_root) try: - data = Popen("env LC_ALL=C cvs log", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() - return data[3].strip('\n') - except IndexError: - msg = "Failed to read CVS log" + return result.stdout.splitlines()[0].strip() + except (IndexError, AttributeError): + msg = "Failed to read revision from CVS: %s" % result.error self.logger.error(msg) - self.logger.error('Ran command "cvs log" from directory %s' % - self.vcs_root) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index e6ef50fa1..88cb6d17c 100644 --- a/src/lib/Bcfg2/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -6,8 +6,8 @@ import Bcfg2.Server.Plugin class DBStats(Bcfg2.Server.Plugin.Plugin): """ DBstats provides a database-backed statistics handler """ - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) self.logger.error("DBStats has been replaced with Reporting") self.logger.error("DBStats: Be sure to migrate your data " "before running the report collector") diff --git a/src/lib/Bcfg2/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py index b4abafb0e..01e42b2d0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Darcs.py +++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py @@ -1,7 +1,7 @@ """ Darcs is a version plugin for dealing with Bcfg2 repos stored in the Darcs VCS. """ -from subprocess import Popen, PIPE +from Bcfg2.Utils import Executor import Bcfg2.Server.Plugin @@ -11,23 +11,19 @@ class Darcs(Bcfg2.Server.Plugin.Version): __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = "_darcs" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) + self.cmd = Executor() self.logger.debug("Initialized Darcs plugin with darcs directory %s" % self.vcs_path) def get_revision(self): """Read Darcs changeset information for the Bcfg2 repository.""" - try: - data = Popen("env LC_ALL=C darcs changes", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() - revision = data[0].strip('\n') - except: - msg = "Failed to read darcs repository" + result = self.cmd.run(["env LC_ALL=C", "darcs", "changes"], + shell=True, cwd=Bcfg2.Options.setup.vcs_root) + if result.success: + return result.stdout.splitlines()[0].strip() + else: + msg = "Failed to read revision from darcs: %s" % result.error self.logger.error(msg) - self.logger.error('Ran command "darcs changes" from directory %s' % - self.vcs_root) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - return revision diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index 66f299bc9..b30a9acea 100644 --- a/src/lib/Bcfg2/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -2,57 +2,33 @@ blacklist certain entries. """ import os -import lxml.etree import Bcfg2.Server.Plugin +import Bcfg2.Server.FileMonitor -class DecisionFile(Bcfg2.Server.Plugin.SpecificData): +class DecisionFile(Bcfg2.Server.Plugin.StructFile): """ Representation of a Decisions XML file """ - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) - self.contents = None - - def handle_event(self, event): - Bcfg2.Server.Plugin.SpecificData.handle_event(self, event) - self.contents = lxml.etree.XML(self.data) - - def get_decisions(self): + def get_decisions(self, metadata): """ Get a list of whitelist or blacklist tuples """ + if self.xdata is None: + # no white/blacklist has been read yet, probably because + # it doesn't exist + return [] return [(x.get('type'), x.get('name')) - for x in self.contents.xpath('.//Decision')] + for x in self.XMLMatch(metadata).xpath('.//Decision')] -class Decisions(Bcfg2.Server.Plugin.EntrySet, - Bcfg2.Server.Plugin.Plugin, +class Decisions(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Decision): - """ Decisions plugin - - Arguments: - - `core`: Bcfg2.Core instance - - `datastore`: File repository location - """ - basename_is_regex = True + """ Decisions plugin """ __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Decision.__init__(self) - Bcfg2.Server.Plugin.EntrySet.__init__(self, '(white|black)list', - self.data, DecisionFile, - core.setup['encoding']) - core.fam.AddMonitor(self.data, self) - - def HandleEvent(self, event): - """ Handle events on Decision files by passing them off to - EntrySet.handle_event """ - if event.filename != self.path: - return self.handle_event(event) + self.whitelist = DecisionFile(os.path.join(self.data, "whitelist.xml")) + self.blacklist = DecisionFile(os.path.join(self.data, "blacklist.xml")) def GetDecisions(self, metadata, mode): - ret = [] - for cdt in self.get_matching(metadata): - if os.path.basename(cdt.name).startswith(mode): - ret.extend(cdt.get_decisions()) - return ret + return getattr(self, mode).get_decisions(metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py index 04c14aa96..79e2ca0e2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Defaults.py +++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py @@ -9,6 +9,8 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, """Set default attributes on bound entries""" __author__ = 'bcfg-dev@mcs.anl.gov' + options = Bcfg2.Server.Plugin.PrioDir.options + # Rules is a Generator that happens to implement all of the # functionality we want, so we overload it, but Defaults should # _not_ handle any entries; it does its stuff in the structure diff --git a/src/lib/Bcfg2/Server/Plugins/Deps.py b/src/lib/Bcfg2/Server/Plugins/Deps.py index d3a1ee871..1872e68af 100644 --- a/src/lib/Bcfg2/Server/Plugins/Deps.py +++ b/src/lib/Bcfg2/Server/Plugins/Deps.py @@ -1,43 +1,20 @@ """This plugin provides automatic dependency handling.""" import lxml.etree - import Bcfg2.Server.Plugin - - -class DNode(Bcfg2.Server.Plugin.INode): - """DNode provides supports for single predicate types for dependencies.""" - def _load_children(self, data, idict): - for item in data.getchildren(): - if item.tag in self.containers: - self.children.append(self.__class__(item, idict, self)) - else: - data = [(child.tag, child.get('name')) - for child in item.getchildren()] - try: - self.contents[item.tag][item.get('name')] = data - except KeyError: - self.contents[item.tag] = {item.get('name'): data} - - -class DepXMLSrc(Bcfg2.Server.Plugin.XMLSrc): - __node__ = DNode +from Bcfg2.Server.Plugin import PluginExecutionError class Deps(Bcfg2.Server.Plugin.PrioDir, Bcfg2.Server.Plugin.StructureValidator): - name = 'Deps' - __author__ = 'bcfg-dev@mcs.anl.gov' - __child__ = DepXMLSrc - # Override the default sort_order (of 500) so that this plugin # gets handled after others running at the default. In particular, # we want to run after Packages, so we can see the final set of # packages that will be installed on the client. sort_order = 750 - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.PrioDir.__init__(self, core) Bcfg2.Server.Plugin.StructureValidator.__init__(self) self.cache = {} @@ -55,63 +32,59 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, tag = entry.tag if tag.startswith('Bound'): tag = tag[5:] - if (tag, entry.get('name')) not in entries \ - and not isinstance(entry, lxml.etree._Comment): + if ((tag, entry.get('name')) not in entries + and not isinstance(entry, lxml.etree._Comment)): entries.append((tag, entry.get('name'))) entries.sort() entries = tuple(entries) - gdata = list(metadata.groups) - gdata.sort() - gdata = tuple(gdata) + groups = list(metadata.groups) + groups.sort() + groups = tuple(groups) # Check to see if we have cached the prereqs already - if (entries, gdata) in self.cache: - prereqs = self.cache[(entries, gdata)] + if (entries, groups) in self.cache: + prereqs = self.cache[(entries, groups)] else: prereqs = self.calculate_prereqs(metadata, entries) - self.cache[(entries, gdata)] = prereqs + self.cache[(entries, groups)] = prereqs - newstruct = lxml.etree.Element("Independent") + newstruct = lxml.etree.Element("Independent", + name=self.__class__.__name__) for tag, name in prereqs: - try: - lxml.etree.SubElement(newstruct, tag, name=name) - except: - self.logger.error("Failed to add dep entry for %s:%s" % (tag, name)) + lxml.etree.SubElement(newstruct, tag, name=name) structures.append(newstruct) - def calculate_prereqs(self, metadata, entries): """Calculate the prerequisites defined in Deps for the passed set of entries. """ prereqs = [] - [src.Cache(metadata) for src in self.entries.values()] - toexamine = list(entries[:]) while toexamine: entry = toexamine.pop() - matching = [src for src in list(self.entries.values()) - if src.cache and entry[0] in src.cache[1] - and entry[1] in src.cache[1][entry[0]]] + # tuples of (PriorityStructFile, element) for each + # matching element and the structfile that contains it + matching = [] + for deps in self.entries.values(): + el = deps.find("/%s[name='%s']" % (entry.tag, + entry.get("name"))) + if el: + matching.append((deps, el)) if len(matching) > 1: - prio = [int(src.priority) for src in matching] + prio = [int(m[0].priority) for m in matching] if prio.count(max(prio)) > 1: - self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" % - (entry[0].lower(), metadata.hostname, entry[1])) - raise Bcfg2.Server.Plugin.PluginExecutionError + raise PluginExecutionError( + "Deps: Found conflicting dependencies with same " + "priority for %s:%s for %s: %s" % + (entry.tag, entry.get("name"), + metadata.hostname, [m[0].name for m in matching])) index = prio.index(max(prio)) matching = [matching[index]] - elif len(matching) == 1: - for prq in matching[0].cache[1][entry[0]][entry[1]]: - # XML comments seem to show up in the cache as a - # tuple with item 0 being callable. The logic - # below filters them out. Would be better to - # exclude them when we load the cache in the first - # place. - if prq not in prereqs and prq not in entries and not callable(prq[0]): - toexamine.append(prq) - prereqs.append(prq) - else: + if not matching: continue + for prq in matching[0][1].getchildren(): + if prq not in prereqs and prq not in entries: + toexamine.append(prq) + prereqs.append(prq) return prereqs diff --git a/src/lib/Bcfg2/Server/Plugins/Editor.py b/src/lib/Bcfg2/Server/Plugins/Editor.py deleted file mode 100644 index f82e0f1dd..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Editor.py +++ /dev/null @@ -1,80 +0,0 @@ -import Bcfg2.Server.Plugin -import re -import lxml.etree - - -def linesub(pattern, repl, filestring): - """Substitutes instances of pattern with repl in filestring.""" - if filestring == None: - filestring = '' - output = list() - fileread = filestring.split('\n') - for line in fileread: - output.append(re.sub(pattern, repl, filestring)) - return '\n'.join(output) - - -class EditDirectives(Bcfg2.Server.Plugin.SpecificData): - """This object handles the editing directives.""" - def ProcessDirectives(self, input): - """Processes a list of edit directives on input.""" - temp = input - for directive in self.data.split('\n'): - directive = directive.split(',') - temp = linesub(directive[0], directive[1], temp) - return temp - - -class EditEntrySet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, basename, path, entry_type, encoding): - self.ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|%s\.H_.*)$" % path.split('/')[-1]) - Bcfg2.Server.Plugin.EntrySet.__init__(self, - basename, - path, - entry_type, - encoding) - self.inputs = dict() - - def bind_entry(self, entry, metadata): - client = metadata.hostname - filename = entry.get('name') - permdata = {'owner': 'root', - 'group': 'root', - 'mode': '0644'} - [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] - entry.text = self.entries['edits'].ProcessDirectives(self.get_client_data(client)) - if not entry.text: - entry.set('empty', 'true') - try: - f = open('%s/%s.H_%s' % (self.path, filename.split('/')[-1], client), 'w') - f.write(entry.text) - f.close() - except: - pass - - def get_client_data(self, client): - return self.inputs[client] - - -class Editor(Bcfg2.Server.Plugin.GroupSpool, - Bcfg2.Server.Plugin.Probing): - name = 'Editor' - __author__ = 'bcfg2-dev@mcs.anl.gov' - filename_pattern = 'edits' - es_child_cls = EditDirectives - es_cls = EditEntrySet - - def GetProbes(self, _): - '''Return a set of probes for execution on client''' - probelist = list() - for name in list(self.entries.keys()): - probe = lxml.etree.Element('probe') - probe.set('name', name) - probe.set('source', "Editor") - probe.text = "cat %s" % name - probelist.append(probe) - return probelist - - def ReceiveData(self, client, datalist): - for data in datalist: - self.entries[data.get('name')].inputs[client.hostname] = data.text diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index 8e074118f..38f9403f5 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -8,9 +8,9 @@ import os import sys import errno import lxml.etree -import Bcfg2.Options import Bcfg2.Server import Bcfg2.Server.Plugin +import Bcfg2.Server.FileMonitor from Bcfg2.Compat import b64decode #: The probe we send to clients to get the file data. Returns an XML @@ -64,13 +64,12 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, the client """ __author__ = 'chris.a.st.pierre@gmail.com' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Probing.__init__(self) self.config = \ Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, 'config.xml'), - fam=core.fam, should_monitor=True, create=self.name) self.entries = dict() @@ -88,7 +87,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, # for which update is false; we can't possibly do # anything with the data we get from such a probe if (entry.get('update', 'false').lower() == "false" and - not cfg.has_generator(entry, metadata)): + not cfg.has_generator(entry, metadata)): continue self.entries[metadata.hostname][path] = entry probe = lxml.etree.Element('probe', name=path, @@ -148,7 +147,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, self.write_file(fileloc, contents) self.verify_file(filename, contents, metadata) infoxml = os.path.join(cfg.data, filename.lstrip("/"), "info.xml") - self.write_infoxml(infoxml, entry, data) + self.write_infoxml(infoxml, data) elif entrydata == contents: self.debug_log("Existing %s contents match probed contents" % filename) @@ -198,7 +197,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, if tries >= 10: self.logger.error("%s still not registered" % filename) return - self.core.fam.handle_events_in_interval(1) + Bcfg2.Server.FileMonitor.get_fam().handle_events_in_interval(1) try: cfg.entries[filename].bind_entry(entry, metadata) except Bcfg2.Server.Plugin.PluginExecutionError: @@ -214,18 +213,18 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, updated = True tries += 1 - def write_infoxml(self, infoxml, entry, data): + def write_infoxml(self, infoxml, data): """ write an info.xml for the file """ if os.path.exists(infoxml): return self.logger.info("Writing %s for %s" % (infoxml, data.get("name"))) + default_mdata = Bcfg2.Server.Plugin.default_path_metadata() info = lxml.etree.Element( "Info", - owner=data.get("owner", Bcfg2.Options.MDATA_OWNER.value), - group=data.get("group", Bcfg2.Options.MDATA_GROUP.value), - mode=data.get("mode", Bcfg2.Options.MDATA_MODE.value), - encoding=entry.get("encoding", Bcfg2.Options.ENCODING.value)) + owner=data.get("owner", default_mdata['owner']), + group=data.get("group", default_mdata['group']), + mode=data.get("mode", default_mdata['mode'])) root = lxml.etree.Element("FileInfo") root.append(info) diff --git a/src/lib/Bcfg2/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index 6165ac651..c4d9af4a4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -1,7 +1,7 @@ """ The Fossil plugin provides a revision interface for Bcfg2 repos using fossil.""" -from subprocess import Popen, PIPE +from Bcfg2.Utils import Executor import Bcfg2.Server.Plugin @@ -11,24 +11,24 @@ class Fossil(Bcfg2.Server.Plugin.Version): __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = "_FOSSIL_" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) + self.cmd = Executor() self.logger.debug("Initialized Fossil plugin with fossil directory %s" % self.vcs_path) def get_revision(self): """Read fossil revision information for the Bcfg2 repository.""" + result = self.cmd.run(["env LC_ALL=C", "fossil", "info"], + shell=True, cwd=Bcfg2.Options.setup.vcs_root) try: - data = Popen("env LC_ALL=C fossil info", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() - revline = [line.split(': ')[1].strip() for line in data - if line.split(': ')[0].strip() == 'checkout'][-1] - return revline.split(' ')[0] - except IndexError: - msg = "Failed to read fossil info" + revision = None + for line in result.stdout.splitlines(): + ldata = line.split(': ') + if ldata[0].strip() == 'checkout': + revision = line[1].strip().split(' ')[0] + return revision + except (IndexError, AttributeError): + msg = "Failed to read revision from Fossil: %s" % result.error self.logger.error(msg) - self.logger.error('Ran command "fossil info" from directory "%s"' % - self.vcs_root) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 44971aba7..9012fceb0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -2,13 +2,14 @@ git. """ import sys +import Bcfg2.Options from Bcfg2.Server.Plugin import Version, PluginExecutionError -from subprocess import Popen, PIPE try: import git HAS_GITPYTHON = True except ImportError: + from Bcfg2.Utils import Executor HAS_GITPYTHON = False @@ -20,14 +21,16 @@ class Git(Version): if HAS_GITPYTHON: __rmi__ = Version.__rmi__ + ['Update'] - def __init__(self, core, datastore): - Version.__init__(self, core, datastore) + def __init__(self, core): + Version.__init__(self, core) if HAS_GITPYTHON: - self.repo = git.Repo(self.vcs_root) + self.repo = git.Repo(Bcfg2.Options.setup.vcs_root) + self.cmd = None else: self.logger.debug("Git: GitPython not found, using CLI interface " "to Git") self.repo = None + self.cmd = Executor() self.logger.debug("Initialized git plugin with git directory %s" % self.vcs_path) @@ -43,16 +46,16 @@ class Git(Version): return self.repo.head.commit.hexsha else: cmd = ["git", "--git-dir", self.vcs_path, - "--work-tree", self.vcs_root, "rev-parse", "HEAD"] + "--work-tree", Bcfg2.Options.setup.vcs_root, + "rev-parse", "HEAD"] self.debug_log("Git: Running %s" % cmd) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - rv, err = proc.communicate() - if proc.wait(): - raise Exception(err) - return rv + result = self.cmd.run(cmd) + if not result.success: + raise Exception(result.stderr) + return result.stdout except: raise PluginExecutionError("Git: Error getting revision from %s: " - "%s" % (self.vcs_root, + "%s" % (Bcfg2.Options.setup.vcs_root, sys.exc_info()[1])) def Update(self, ref=None): @@ -61,14 +64,15 @@ class Git(Version): """ self.logger.info("Git: Git.Update(ref='%s')" % ref) self.debug_log("Git: Performing garbage collection on repo at %s" % - self.vcs_root) + Bcfg2.Options.setup.vcs_root) try: self._log_git_cmd(self.repo.git.gc('--auto')) except git.GitCommandError: self.logger.warning("Git: Failed to perform garbage collection: %s" % sys.exc_info()[1]) - self.debug_log("Git: Fetching all refs for repo at %s" % self.vcs_root) + self.debug_log("Git: Fetching all refs for repo at %s" % + Bcfg2.Options.setup.vcs_root) try: self._log_git_cmd(self.repo.git.fetch('--all')) except git.GitCommandError: @@ -101,5 +105,5 @@ class Git(Version): "upstream: %s" % sys.exc_info()[1]) self.logger.info("Git: Repo at %s updated to %s" % - (self.vcs_root, self.get_revision())) + (Bcfg2.Options.setup.vcs_root, self.get_revision())) return True diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index 24547949b..b60f60e65 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -6,30 +6,24 @@ import lxml.etree from threading import local import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Metadata import MetadataGroup -try: - from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile -except ImportError: - # BundleTemplateFile missing means that genshi is missing. we - # import genshi to get the _real_ error - import genshi # pylint: disable=W0611 -class GroupLogicConfig(BundleTemplateFile): +class GroupLogicConfig(Bcfg2.Server.Plugin.StructFile): """ Representation of the GroupLogic groups.xml file """ create = lxml.etree.Element("GroupLogic", nsmap=dict(py="http://genshi.edgewall.org/")) - def __init__(self, name, fam): - BundleTemplateFile.__init__(self, name, - Bcfg2.Server.Plugin.Specificity(), None) - self.fam = fam - self.should_monitor = True - self.fam.AddMonitor(self.name, self) + def _match(self, item, metadata, *args): + if item.tag == 'Group' and not len(item.getchildren()): + return [item] + return Bcfg2.Server.Plugin.StructFile._match(self, item, metadata, + *args) - def _match(self, item, metadata): + def _xml_match(self, item, metadata, *args): if item.tag == 'Group' and not len(item.getchildren()): return [item] - return BundleTemplateFile._match(self, item, metadata) + return Bcfg2.Server.Plugin.StructFile._xml_match(self, item, metadata, + *args) class GroupLogic(Bcfg2.Server.Plugin.Plugin, @@ -41,11 +35,11 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, # use groups set by them sort_order = 1000 - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), - core.fam) + should_monitor=True) self._local = local() def get_additional_groups(self, metadata): @@ -66,7 +60,7 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, return [] self._local.building.add(metadata.hostname) rv = [] - for el in self.config.get_xml_value(metadata).xpath("//Group"): + for el in self.config.XMLMatch(metadata).findall("Group"): if el.get("category"): rv.append(MetadataGroup(el.get("name"), category=el.get("category"))) diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 09685d972..7fa95fd05 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -69,11 +69,7 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): create = 'GroupPatterns' def __init__(self, filename, core=None): - try: - fam = core.fam - except AttributeError: - fam = None - Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, should_monitor=True) self.core = core self.patterns = [] @@ -81,7 +77,7 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): def Index(self): Bcfg2.Server.Plugin.XMLFileBacked.Index(self) if (self.core and - self.core.metadata_cache_mode in ['cautious', 'aggressive']): + self.core.metadata_cache_mode in ['cautious', 'aggressive']): self.core.metadata_cache.expire() self.patterns = [] for entry in self.xdata.xpath('//GroupPattern'): @@ -118,48 +114,11 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): """ set group membership based on client hostnames """ - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = PatternFile(os.path.join(self.data, 'config.xml'), core=core) def get_additional_groups(self, metadata): return self.config.process_patterns(metadata.hostname) - - -class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns - <server-plugins-grouping-grouppatterns>` patterns for validity. - This is simply done by trying to create a - :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for - each pattern, and catching exceptions and presenting them as - ``bcfg2-lint`` errors.""" - - def Run(self): - cfg = self.core.plugins['GroupPatterns'].config - for entry in cfg.xdata.xpath('//GroupPattern'): - groups = [g.text for g in entry.findall('Group')] - self.check(entry, groups, ptype='NamePattern') - self.check(entry, groups, ptype='NameRange') - - @classmethod - def Errors(cls): - return {"pattern-fails-to-initialize": "error"} - - def check(self, entry, groups, ptype="NamePattern"): - """ Check a single pattern for validity """ - if ptype == "NamePattern": - pmap = lambda p: PatternMap(p, None, groups) - else: - pmap = lambda p: PatternMap(None, p, groups) - - for el in entry.findall(ptype): - pat = el.text - try: - pmap(pat) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - self.LintError("pattern-fails-to-initialize", - "Failed to initialize %s %s for %s: %s" % - (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Bcfg2/Server/Plugins/Guppy.py b/src/lib/Bcfg2/Server/Plugins/Guppy.py index 3c9b8a459..8427a56c3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Guppy.py +++ b/src/lib/Bcfg2/Server/Plugins/Guppy.py @@ -32,15 +32,12 @@ from guppy.heapy import Remote class Guppy(Bcfg2.Server.Plugin.Plugin): """Guppy is a debugging plugin to help trace memory leaks""" - name = 'Guppy' __author__ = 'bcfg-dev@mcs.anl.gov' - - experimental = True __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable', 'Disable'] __child_rmi__ = __rmi__[:] - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) self.Enable() diff --git a/src/lib/Bcfg2/Server/Plugins/Hg.py b/src/lib/Bcfg2/Server/Plugins/Hg.py index 3fd3918bd..7554b4d52 100644 --- a/src/lib/Bcfg2/Server/Plugins/Hg.py +++ b/src/lib/Bcfg2/Server/Plugins/Hg.py @@ -12,15 +12,15 @@ class Hg(Bcfg2.Server.Plugin.Version): __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".hg" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) self.logger.debug("Initialized hg plugin with hg directory %s" % self.vcs_path) def get_revision(self): """Read hg revision information for the Bcfg2 repository.""" try: - repo_path = self.vcs_root + "/" + repo_path = Bcfg2.Options.setup.vcs_root + "/" repo = hg.repository(ui.ui(), repo_path) tip = repo.changelog.tip() return repo.changelog.rev(tip) diff --git a/src/lib/Bcfg2/Server/Plugins/Hostbase.py b/src/lib/Bcfg2/Server/Plugins/Hostbase.py deleted file mode 100644 index 55757e0b4..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Hostbase.py +++ /dev/null @@ -1,599 +0,0 @@ -""" -This file provides the Hostbase plugin. -It manages dns/dhcp/nis host information -""" - -from lxml.etree import Element, SubElement -import os -import re -from time import strftime -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings' -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugin import PluginExecutionError, PluginInitError -from django.template import Context, loader -from django.db import connection -# Compatibility imports -from Bcfg2.Compat import StringIO - -try: - set -except NameError: - # deprecated since python 2.6 - from sets import Set as set - - -class Hostbase(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.Generator): - """The Hostbase plugin handles host/network info.""" - name = 'Hostbase' - __author__ = 'bcfg-dev@mcs.anl.gov' - filepath = '/my/adm/hostbase/files/bind' - deprecated = True - - def __init__(self, core, datastore): - - self.ready = False - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - Bcfg2.Server.Plugin.Generator.__init__(self) - files = ['zone.tmpl', - 'reversesoa.tmpl', - 'named.tmpl', - 'reverseappend.tmpl', - 'dhcpd.tmpl', - 'hosts.tmpl', - 'hostsappend.tmpl'] - self.filedata = {} - self.dnsservers = [] - self.dhcpservers = [] - self.templates = {'zone': loader.get_template('zone.tmpl'), - 'reversesoa': loader.get_template('reversesoa.tmpl'), - 'named': loader.get_template('named.tmpl'), - 'namedviews': loader.get_template('namedviews.tmpl'), - 'reverseapp': loader.get_template('reverseappend.tmpl'), - 'dhcp': loader.get_template('dhcpd.tmpl'), - 'hosts': loader.get_template('hosts.tmpl'), - 'hostsapp': loader.get_template('hostsappend.tmpl'), - } - self.Entries['ConfigFile'] = {} - self.__rmi__ = ['rebuildState'] - try: - self.rebuildState(None) - except: - raise PluginInitError - - def FetchFile(self, entry, metadata): - """Return prebuilt file data.""" - fname = entry.get('name').split('/')[-1] - if not fname in self.filedata: - raise PluginExecutionError - perms = {'owner': 'root', - 'group': 'root', - 'mode': '644'} - [entry.attrib.__setitem__(key, value) - for (key, value) in list(perms.items())] - entry.text = self.filedata[fname] - - def BuildStructures(self, metadata): - """Build hostbase bundle.""" - if metadata.hostname not in self.dnsservers or metadata.hostname not in self.dhcpservers: - return [] - output = Element("Bundle", name='hostbase') - if metadata.hostname in self.dnsservers: - for configfile in self.Entries['ConfigFile']: - if re.search('/etc/bind/', configfile): - SubElement(output, "ConfigFile", name=configfile) - if metadata.hostname in self.dhcpservers: - SubElement(output, "ConfigFile", name="/etc/dhcp3/dhcpd.conf") - return [output] - - def rebuildState(self, _): - """Pre-cache all state information for hostbase config files - callable as an XMLRPC function. - - """ - self.buildZones() - self.buildDHCP() - self.buildHosts() - self.buildHostsLPD() - self.buildPrinters() - self.buildNetgroups() - return True - - def buildZones(self): - """Pre-build and stash zone files.""" - cursor = connection.cursor() - - cursor.execute("SELECT id, serial FROM hostbase_zone") - zones = cursor.fetchall() - - for zone in zones: - # update the serial number for all zone files - todaydate = (strftime('%Y%m%d')) - try: - if todaydate == str(zone[1])[:8]: - serial = zone[1] + 1 - else: - serial = int(todaydate) * 100 - except (KeyError): - serial = int(todaydate) * 100 - cursor.execute("""UPDATE hostbase_zone SET serial = \'%s\' WHERE id = \'%s\'""" % (str(serial), zone[0])) - - cursor.execute("SELECT * FROM hostbase_zone WHERE zone NOT LIKE \'%%.rev\'") - zones = cursor.fetchall() - - iplist = [] - hosts = {} - - for zone in zones: - zonefile = StringIO() - externalzonefile = StringIO() - cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z - INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - nameservers = cursor.fetchall() - cursor.execute("""SELECT i.ip_addr FROM hostbase_zone_addresses z - INNER JOIN hostbase_zoneaddress i ON z.zoneaddress_id = i.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - addresses = cursor.fetchall() - cursor.execute("""SELECT m.priority, m.mx FROM hostbase_zone_mxs z - INNER JOIN hostbase_mx m ON z.mx_id = m.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - mxs = cursor.fetchall() - context = Context({ - 'zone': zone, - 'nameservers': nameservers, - 'addresses': addresses, - 'mxs': mxs - }) - zonefile.write(self.templates['zone'].render(context)) - externalzonefile.write(self.templates['zone'].render(context)) - - querystring = """SELECT h.hostname, p.ip_addr, - n.name, c.cname, m.priority, m.mx, n.dns_view - FROM (((((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - INNER JOIN hostbase_name_mxs x ON n.id = x.name_id) - INNER JOIN hostbase_mx m ON m.id = x.mx_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE n.name LIKE '%%%%%s' - AND h.status = 'active' - ORDER BY h.hostname, n.name, p.ip_addr - """ % zone[1] - cursor.execute(querystring) - zonehosts = cursor.fetchall() - prevhost = (None, None, None, None) - cnames = StringIO() - cnamesexternal = StringIO() - for host in zonehosts: - if not host[2].split(".", 1)[1] == zone[1]: - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - cnames = StringIO() - cnamesexternal = StringIO() - continue - if not prevhost[1] == host[1] or not prevhost[2] == host[2]: - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - cnames = StringIO() - cnamesexternal = StringIO() - zonefile.write("%-32s%-10s%-32s\n" % - (host[2].split(".", 1)[0], 'A', host[1])) - zonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - if host[6] == 'global': - externalzonefile.write("%-32s%-10s%-32s\n" % - (host[2].split(".", 1)[0], 'A', host[1])) - externalzonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - elif not prevhost[5] == host[5]: - zonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - if host[6] == 'global': - externalzonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - - if host[3]: - try: - if host[3].split(".", 1)[1] == zone[1]: - cnames.write("%-32s%-10s%-32s\n" % - (host[3].split(".", 1)[0], - 'CNAME', host[2].split(".", 1)[0])) - if host[6] == 'global': - cnamesexternal.write("%-32s%-10s%-32s\n" % - (host[3].split(".", 1)[0], - 'CNAME', host[2].split(".", 1)[0])) - else: - cnames.write("%-32s%-10s%-32s\n" % - (host[3] + ".", - 'CNAME', - host[2].split(".", 1)[0])) - if host[6] == 'global': - cnamesexternal.write("%-32s%-10s%-32s\n" % - (host[3] + ".", - 'CNAME', - host[2].split(".", 1)[0])) - - except: - pass - prevhost = host - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - zonefile.write("\n\n%s" % zone[9]) - externalzonefile.write("\n\n%s" % zone[9]) - self.filedata[zone[1]] = zonefile.getvalue() - self.filedata[zone[1] + ".external"] = externalzonefile.getvalue() - zonefile.close() - externalzonefile.close() - self.Entries['ConfigFile']["%s/%s" % (self.filepath, zone[1])] = self.FetchFile - self.Entries['ConfigFile']["%s/%s.external" % (self.filepath, zone[1])] = self.FetchFile - - cursor.execute("SELECT * FROM hostbase_zone WHERE zone LIKE \'%%.rev\' AND zone <> \'.rev\'") - reversezones = cursor.fetchall() - - reversenames = [] - for reversezone in reversezones: - cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z - INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id - WHERE z.zone_id = \'%s\'""" % reversezone[0]) - reverse_nameservers = cursor.fetchall() - - context = Context({ - 'inaddr': reversezone[1].rstrip('.rev'), - 'zone': reversezone, - 'nameservers': reverse_nameservers, - }) - - self.filedata[reversezone[1]] = self.templates['reversesoa'].render(context) - self.filedata[reversezone[1] + '.external'] = self.templates['reversesoa'].render(context) - self.filedata[reversezone[1]] += reversezone[9] - self.filedata[reversezone[1] + '.external'] += reversezone[9] - - subnet = reversezone[1].split(".") - subnet.reverse() - reversenames.append((reversezone[1].rstrip('.rev'), ".".join(subnet[1:]))) - - for filename in reversenames: - cursor.execute(""" - SELECT DISTINCT h.hostname, p.ip_addr, n.dns_view FROM ((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON n.ip_id = p.id - WHERE p.ip_addr LIKE '%s%%%%' AND h.status = 'active' ORDER BY p.ip_addr - """ % filename[1]) - reversehosts = cursor.fetchall() - zonefile = StringIO() - externalzonefile = StringIO() - if len(filename[0].split(".")) == 2: - originlist = [] - [originlist.append((".".join([ip[1].split(".")[2], filename[0]]), - ".".join([filename[1], ip[1].split(".")[2]]))) - for ip in reversehosts - if (".".join([ip[1].split(".")[2], filename[0]]), - ".".join([filename[1], ip[1].split(".")[2]])) not in originlist] - for origin in originlist: - hosts = [(host[1].split("."), host[0]) - for host in reversehosts - if host[1].rstrip('0123456789').rstrip('.') == origin[1]] - hosts_external = [(host[1].split("."), host[0]) - for host in reversehosts - if (host[1].rstrip('0123456789').rstrip('.') == origin[1] - and host[2] == 'global')] - context = Context({ - 'hosts': hosts, - 'inaddr': origin[0], - 'fileorigin': filename[0], - }) - zonefile.write(self.templates['reverseapp'].render(context)) - context = Context({ - 'hosts': hosts_external, - 'inaddr': origin[0], - 'fileorigin': filename[0], - }) - externalzonefile.write(self.templates['reverseapp'].render(context)) - else: - originlist = [filename[0]] - hosts = [(host[1].split("."), host[0]) - for host in reversehosts - if (host[1].split("."), host[0]) not in hosts] - hosts_external = [(host[1].split("."), host[0]) - for host in reversehosts - if ((host[1].split("."), host[0]) not in hosts_external - and host[2] == 'global')] - context = Context({ - 'hosts': hosts, - 'inaddr': filename[0], - 'fileorigin': None, - }) - zonefile.write(self.templates['reverseapp'].render(context)) - context = Context({ - 'hosts': hosts_external, - 'inaddr': filename[0], - 'fileorigin': None, - }) - externalzonefile.write(self.templates['reverseapp'].render(context)) - self.filedata['%s.rev' % filename[0]] += zonefile.getvalue() - self.filedata['%s.rev.external' % filename[0]] += externalzonefile.getvalue() - zonefile.close() - externalzonefile.close() - self.Entries['ConfigFile']['%s/%s.rev' % (self.filepath, filename[0])] = self.FetchFile - self.Entries['ConfigFile']['%s/%s.rev.external' % (self.filepath, filename[0])] = self.FetchFile - - ## here's where the named.conf file gets written - context = Context({ - 'zones': zones, - 'reverses': reversenames, - }) - self.filedata['named.conf'] = self.templates['named'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf'] = self.FetchFile - self.filedata['named.conf.views'] = self.templates['namedviews'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf.views'] = self.FetchFile - - def buildDHCP(self): - """Pre-build dhcpd.conf and stash in the filedata table.""" - - # fetches all the hosts with DHCP == True - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname, mac_addr, ip_addr - FROM (hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip ip ON i.id = ip.interface_id - WHERE i.dhcp=1 AND h.status='active' AND i.mac_addr <> '' - AND i.mac_addr <> 'float' AND i.mac_addr <> 'unknown' - ORDER BY h.hostname, i.mac_addr - """) - - dhcphosts = cursor.fetchall() - count = 0 - hosts = [] - hostdata = [dhcphosts[0][0], dhcphosts[0][1], dhcphosts[0][2]] - if len(dhcphosts) > 1: - for x in range(1, len(dhcphosts)): - # if an interface has 2 or more ip addresses - # adds the ip to the current interface - if hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0] and hostdata[1] == dhcphosts[x][1]: - hostdata[2] = ", ".join([hostdata[2], dhcphosts[x][2]]) - # if a host has 2 or more interfaces - # writes the current one and grabs the next - elif hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0]: - hosts.append(hostdata) - count += 1 - hostdata = ["-".join([dhcphosts[x][0], str(count)]), dhcphosts[x][1], dhcphosts[x][2]] - # new host found, writes current data to the template - else: - hosts.append(hostdata) - count = 0 - hostdata = [dhcphosts[x][0], dhcphosts[x][1], dhcphosts[x][2]] - #makes sure the last of the data gets written out - if hostdata not in hosts: - hosts.append(hostdata) - - context = Context({ - 'hosts': hosts, - 'numips': len(hosts), - }) - - self.filedata['dhcpd.conf'] = self.templates['dhcp'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/dhcpd.conf'] = self.FetchFile - - def buildHosts(self): - """Pre-build and stash /etc/hosts file.""" - - append_data = [] - - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname FROM hostbase_host ORDER BY hostname - """) - hostbase = cursor.fetchall() - domains = [host[0].split(".", 1)[1] for host in hostbase] - domains_set = set(domains) - domain_data = [(domain, domains.count(domain)) for domain in domains_set] - domain_data.sort() - - cursor.execute(""" - SELECT ip_addr FROM hostbase_ip ORDER BY ip_addr - """) - ips = cursor.fetchall() - three_octets = [ip[0].rstrip('0123456789').rstrip('.') \ - for ip in ips] - three_octets_set = set(three_octets) - three_octets_data = [(octet, three_octets.count(octet)) \ - for octet in three_octets_set] - three_octets_data.sort() - - for three_octet in three_octets_data: - querystring = """SELECT h.hostname, h.primary_user, - p.ip_addr, n.name, c.cname - FROM (((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE p.ip_addr LIKE \'%s.%%%%\' AND h.status = 'active'""" % three_octet[0] - cursor.execute(querystring) - tosort = list(cursor.fetchall()) - tosort.sort(lambda x, y: cmp(int(x[2].split(".")[-1]), int(y[2].split(".")[-1]))) - append_data.append((three_octet, tuple(tosort))) - - two_octets = [ip.rstrip('0123456789').rstrip('.') for ip in three_octets] - two_octets_set = set(two_octets) - two_octets_data = [(octet, two_octets.count(octet)) - for octet in two_octets_set] - two_octets_data.sort() - - context = Context({ - 'domain_data': domain_data, - 'three_octets_data': three_octets_data, - 'two_octets_data': two_octets_data, - 'three_octets': three_octets, - 'num_ips': len(three_octets), - }) - - self.filedata['hosts'] = self.templates['hosts'].render(context) - - for subnet in append_data: - ips = [] - simple = True - namelist = [name.split('.', 1)[0] for name in [subnet[1][0][3]]] - cnamelist = [] - if subnet[1][0][4]: - cnamelist.append(subnet[1][0][4].split('.', 1)[0]) - simple = False - appenddata = subnet[1][0] - for ip in subnet[1][1:]: - if appenddata[2] == ip[2]: - namelist.append(ip[3].split('.', 1)[0]) - if ip[4]: - cnamelist.append(ip[4].split('.', 1)[0]) - simple = False - appenddata = ip - else: - if appenddata[0] == ip[0]: - simple = False - ips.append((appenddata[2], appenddata[0], set(namelist), - cnamelist, simple, appenddata[1])) - appenddata = ip - simple = True - namelist = [ip[3].split('.', 1)[0]] - cnamelist = [] - if ip[4]: - cnamelist.append(ip[4].split('.', 1)[0]) - simple = False - ips.append((appenddata[2], appenddata[0], set(namelist), - cnamelist, simple, appenddata[1])) - context = Context({ - 'subnet': subnet[0], - 'ips': ips, - }) - self.filedata['hosts'] += self.templates['hostsapp'].render(context) - self.Entries['ConfigFile']['/mcs/etc/hosts'] = self.FetchFile - - def buildPrinters(self): - """The /mcs/etc/printers.data file""" - header = """# This file is automatically generated. DO NOT EDIT IT! -# -Name Room User Type Notes -============== ========== ============================== ======================== ==================== -""" - - cursor = connection.cursor() - # fetches all the printers from the database - cursor.execute(""" - SELECT printq, location, primary_user, comments - FROM hostbase_host - WHERE whatami='printer' AND printq <> '' AND status = 'active' - ORDER BY printq - """) - printers = cursor.fetchall() - - printersfile = header - for printer in printers: - # splits up the printq line and gets the - # correct description out of the comments section - temp = printer[3].split('\n') - for printq in re.split(',[ ]*', printer[0]): - if len(temp) > 1: - printersfile += ("%-16s%-12s%-32s%-26s%s\n" % - (printq, printer[1], printer[2], temp[1], temp[0])) - else: - printersfile += ("%-16s%-12s%-32s%-26s%s\n" % - (printq, printer[1], printer[2], '', printer[3])) - self.filedata['printers.data'] = printersfile - self.Entries['ConfigFile']['/mcs/etc/printers.data'] = self.FetchFile - - def buildHostsLPD(self): - """Creates the /mcs/etc/hosts.lpd file""" - - # this header needs to be changed to be more generic - header = """+@machines -+@all-machines -achilles.ctd.anl.gov -raven.ops.anl.gov -seagull.hr.anl.gov -parrot.ops.anl.gov -condor.ops.anl.gov -delphi.esh.anl.gov -anlcv1.ctd.anl.gov -anlvms.ctd.anl.gov -olivia.ctd.anl.gov\n\n""" - - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname FROM hostbase_host WHERE netgroup=\"red\" AND status = 'active' - ORDER BY hostname""") - redmachines = list(cursor.fetchall()) - cursor.execute(""" - SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE netgroup=\"red\" AND n.only=1 AND h.status = 'active' - """) - redmachines.extend(list(cursor.fetchall())) - cursor.execute(""" - SELECT hostname FROM hostbase_host WHERE netgroup=\"win\" AND status = 'active' - ORDER BY hostname""") - winmachines = list(cursor.fetchall()) - cursor.execute(""" - SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE netgroup=\"win\" AND n.only=1 AND h.status = 'active' - """) - winmachines.__add__(list(cursor.fetchall())) - hostslpdfile = header - for machine in redmachines: - hostslpdfile += machine[0] + "\n" - hostslpdfile += "\n" - for machine in winmachines: - hostslpdfile += machine[0] + "\n" - self.filedata['hosts.lpd'] = hostslpdfile - self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile - - def buildNetgroups(self): - """Makes the *-machine files""" - header = """################################################################### -# This file lists hosts in the '%s' machine netgroup, it is -# automatically generated. DO NOT EDIT THIS FILE! -# -# Number of hosts in '%s' machine netgroup: %i -#\n\n""" - - cursor = connection.cursor() - # fetches all the hosts that with valid netgroup entries - cursor.execute(""" - SELECT h.hostname, n.name, h.netgroup, n.only FROM ((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE h.netgroup <> '' AND h.netgroup <> 'none' AND h.status = 'active' - ORDER BY h.netgroup, h.hostname - """) - nameslist = cursor.fetchall() - # gets the first host and initializes the hash - hostdata = nameslist[0] - netgroups = {hostdata[2]: [hostdata[0]]} - for row in nameslist: - # if new netgroup, create it - if row[2] not in netgroups: - netgroups.update({row[2]: []}) - # if it belongs in the netgroup and has multiple interfaces, put them in - if hostdata[0] == row[0] and row[3]: - netgroups[row[2]].append(row[1]) - hostdata = row - # if its a new host, write the old one to the hash - elif hostdata[0] != row[0]: - netgroups[row[2]].append(row[0]) - hostdata = row - - for netgroup in netgroups: - fileoutput = StringIO() - fileoutput.write(header % (netgroup, netgroup, len(netgroups[netgroup]))) - for each in netgroups[netgroup]: - fileoutput.write(each + "\n") - self.filedata['%s-machines' % netgroup] = fileoutput.getvalue() - fileoutput.close() - self.Entries['ConfigFile']['/my/adm/hostbase/makenets/machines/%s-machines' % netgroup] = self.FetchFile - - cursor.execute(""" - UPDATE hostbase_host SET dirty=0 - """) diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index f724402d0..553ddbc47 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -3,7 +3,6 @@ import logging import sys import time import traceback -import Bcfg2.Options import Bcfg2.Server.Plugin logger = logging.getLogger('Bcfg2.Plugins.Ldap') @@ -44,10 +43,10 @@ class ConfigFile(Bcfg2.Server.Plugin.FileBacked): The approach implemented here is having the user call a registering decorator that updates a global variable in this module. """ - def __init__(self, filename, fam): + def __init__(self, filename): self.filename = filename Bcfg2.Server.Plugin.FileBacked.__init__(self, self.filename) - fam.AddMonitor(self.filename, self) + self.fam.AddMonitor(self.filename, self) def Index(self): """ @@ -69,12 +68,12 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): experimental = True debug_flag = False - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) - self.config = ConfigFile(self.data + "/config.py", core.fam) + self.config = ConfigFile(self.data + "/config.py") - def debug_log(self, message, flag = None): + def debug_log(self, message, flag=None): if (flag is None) and self.debug_flag or flag: self.logger.error(message) @@ -83,37 +82,39 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): try: data = {} self.debug_log("LdapPlugin debug: found queries " + - str(LDAP_QUERIES)) + str(LDAP_QUERIES)) for QueryClass in LDAP_QUERIES: query = QueryClass() if query.is_applicable(metadata): self.debug_log("LdapPlugin debug: processing query '" + - query.name + "'") + query.name + "'") data[query.name] = query.get_result(metadata) else: self.debug_log("LdapPlugin debug: query '" + query.name + - "' not applicable to host '" + metadata.hostname + "'") + "' not applicable to host '" + + metadata.hostname + "'") return data except Exception: if hasattr(query, "name"): logger.error("LdapPlugin error: " + - "Exception during processing of query named '" + - str(query.name) + - "', query results will be empty" + - " and may cause bind failures") + "Exception during processing of query named '" + + str(query.name) + + "', query results will be empty" + + " and may cause bind failures") for line in traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]): logger.error("LdapPlugin error: " + - line.replace("\n", "")) + line.replace("\n", "")) return {} + class LdapConnection(object): """ Connection to an LDAP server. """ - def __init__(self, host = "localhost", port = 389, - binddn = None, bindpw = None): + def __init__(self, host="localhost", port=389, + binddn=None, bindpw=None): self.host = host self.port = port self.binddn = binddn @@ -134,8 +135,8 @@ class LdapConnection(object): for attempt in range(RETRY_COUNT + 1): if attempt >= 1: logger.error("LdapPlugin error: " + - "LDAP server down (retry " + str(attempt) + "/" + - str(RETRY_COUNT) + ")") + "LDAP server down (retry " + str(attempt) + "/" + + str(RETRY_COUNT) + ")") try: if not self.conn: self.init_conn() @@ -155,6 +156,7 @@ class LdapConnection(object): def url(self): return "ldap://" + self.host + ":" + str(self.port) + class LdapQuery(object): """ Query referencing an LdapConnection and providing several @@ -211,9 +213,10 @@ class LdapQuery(object): return self.result else: logger.error("LdapPlugin error: " + - "No valid connection defined for query " + str(self)) + "No valid connection defined for query " + str(self)) return None + class LdapSubQuery(LdapQuery): """ SubQueries are meant for internal use only and are not added @@ -244,5 +247,5 @@ class LdapSubQuery(LdapQuery): return self.process_result(metadata, **kwargs) else: logger.error("LdapPlugin error: " + - "No valid connection defined for query " + str(self)) + "No valid connection defined for query " + str(self)) return None diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 1e5544c6b..1d15656af 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -12,41 +12,52 @@ import socket import logging import lxml.etree import Bcfg2.Server -import Bcfg2.Server.Lint +import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked +from Bcfg2.Server.Cache import Cache # pylint: disable=W0622 from Bcfg2.Compat import MutableMapping, all, any, wraps # pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo -try: - from django.db import models - HAS_DJANGO = True -except ImportError: - HAS_DJANGO = False +# pylint: disable=C0103 +ClientVersions = None +MetadataClientModel = None +# pylint: enable=C0103 +HAS_DJANGO = False -LOGGER = logging.getLogger(__name__) +def load_django_models(): + """ Load models for Django after option parsing has completed """ + # pylint: disable=W0602 + global MetadataClientModel, ClientVersions, HAS_DJANGO + # pylint: enable=W0602 -if HAS_DJANGO: - class MetadataClientModel(models.Model, + try: + from django.db import models + HAS_DJANGO = True + except ImportError: + HAS_DJANGO = False + return + + class MetadataClientModel(models.Model, # pylint: disable=W0621 Bcfg2.Server.Plugin.PluginDatabaseModel): """ django model for storing clients in the database """ hostname = models.CharField(max_length=255, primary_key=True) version = models.CharField(max_length=31, null=True) - class ClientVersions(MutableMapping, + class ClientVersions(MutableMapping, # pylint: disable=W0621,W0612 Bcfg2.Server.Plugin.DatabaseBacked): """ dict-like object to make it easier to access client bcfg2 versions from the database """ - create = False def __getitem__(self, key): try: - return MetadataClientModel.objects.get(hostname=key).version + return MetadataClientModel.objects.get( + hostname=key).version except MetadataClientModel.DoesNotExist: raise KeyError(key) @@ -80,7 +91,7 @@ if HAS_DJANGO: def keys(self): """ Get keys for the mapping """ - return [c.hostname for c in MetadataClientModel.objects.all()] + return list(iter(self)) def __contains__(self, key): try: @@ -93,25 +104,19 @@ if HAS_DJANGO: class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): """Handles xml config files and all XInclude statements""" - def __init__(self, metadata, watch_clients, basefile): - # we tell XMLFileBacked _not_ to add a monitor for this file, - # because the main Metadata plugin has already added one. - # then we immediately set should_monitor to the proper value, - # so that XInclude'd files get properly watched + def __init__(self, metadata, basefile): fpath = os.path.join(metadata.data, basefile) toptag = os.path.splitext(basefile)[0].title() Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath, - fam=metadata.core.fam, should_monitor=False, create=toptag) - self.should_monitor = watch_clients self.metadata = metadata self.basefile = basefile self.data = None self.basedata = None self.basedir = metadata.data self.logger = metadata.logger - self.pseudo_monitor = isinstance(metadata.core.fam, + self.pseudo_monitor = isinstance(Bcfg2.Server.FileMonitor.get_fam(), Bcfg2.Server.FileMonitor.Pseudo) def _get_xdata(self): @@ -250,8 +255,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): def add_monitor(self, fpath): self.extras.append(fpath) - if self.fam and self.should_monitor: - self.fam.AddMonitor(fpath, self.metadata) + self.fam.AddMonitor(fpath, self.metadata) def HandleEvent(self, event=None): """Handle fam events""" @@ -354,6 +358,8 @@ class MetadataQuery(object): def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): + self.logger = logging.getLogger(self.__class__.__name__) + #: Get :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` #: object for the given hostname. #: @@ -406,8 +412,9 @@ class MetadataQuery(object): @wraps(func) def inner(arg): if isinstance(arg, str): - LOGGER.warning("%s: %s takes a list as argument, not a string" - % (self.__class__.__name__, func.__name__)) + self.logger.warning("%s: %s takes a list as argument, not a " + "string" % (self.__class__.__name__, + func.__name__)) return func(arg) # pylint: enable=C0111 @@ -490,25 +497,33 @@ class MetadataGroup(tuple): # pylint: disable=E0012,R0924 class Metadata(Bcfg2.Server.Plugin.Metadata, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" __author__ = 'bcfg-dev@mcs.anl.gov' sort_order = 500 - def __init__(self, core, datastore, watch_clients=True): + options = Bcfg2.Server.Plugin.DatabaseBacked.options + [ + Bcfg2.Options.Common.password, + Bcfg2.Options.BooleanOption( + cf=('metadata', 'use_database'), dest="metadata_db", + help="Use database capabilities of the Metadata plugin"), + Bcfg2.Options.Option( + cf=('communication', 'authentication'), default='cert+password', + choices=['cert', 'bootstrap', 'cert+password'], + help='Default client authentication method')] + options_parsed_hook = staticmethod(load_django_models) + + def __init__(self, core): Bcfg2.Server.Plugin.Metadata.__init__(self) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) - Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) - self.watch_clients = watch_clients + Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core) self.states = dict() self.extra = dict() self.handlers = dict() self.groups_xml = self._handle_file("groups.xml") if (self._use_db and - os.path.exists(os.path.join(self.data, "clients.xml"))): + os.path.exists(os.path.join(self.data, "clients.xml"))): self.logger.warning("Metadata: database enabled but clients.xml " "found, parsing in compatibility mode") self.clients_xml = self._handle_file("clients.xml") @@ -540,15 +555,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.ordered_groups = [] # mapping of hostname -> version string if self._use_db: - self.versions = ClientVersions(core, datastore) + self.versions = ClientVersions(core) # pylint: disable=E1102 else: self.versions = dict() self.uuid = {} self.session_cache = {} + self.cache = Cache("Metadata") self.default = None self.pdirty = False - self.password = core.setup['password'] + self.password = Bcfg2.Options.setup.password self.query = MetadataQuery(core.build_metadata, self.list_clients, self.get_client_names_by_groups, @@ -576,16 +592,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _handle_file(self, fname): """ set up the necessary magic for handling a metadata file (clients.xml or groups.xml, e.g.) """ - if self.watch_clients: - try: - self.core.fam.AddMonitor(os.path.join(self.data, fname), self) - except: - err = sys.exc_info()[1] - msg = "Unable to add file monitor for %s: %s" % (fname, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - self.states[fname] = False - xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname) + try: + Bcfg2.Server.FileMonitor.get_fam().AddMonitor( + os.path.join(self.data, fname), self) + except: + err = sys.exc_info()[1] + msg = "Unable to add file monitor for %s: %s" % (fname, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) + self.states[fname] = False + xmlcfg = XMLMetadataConfig(self, fname) aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(fname)) self.handlers[xmlcfg.HandleEvent] = getattr(self, "_handle_%s_event" % aname) @@ -600,7 +616,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, elif alias: for child in node: if (child.tag == "Alias" and - child.attrib["name"] == name): + child.attrib["name"] == name): return node return None @@ -666,7 +682,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, try: client = MetadataClientModel.objects.get(hostname=client_name) except MetadataClientModel.DoesNotExist: + # pylint: disable=E1102 client = MetadataClientModel(hostname=client_name) + # pylint: enable=E1102 client.save() self.update_client_list() return client @@ -819,7 +837,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if client.get('secure', 'false').lower() == 'true': self.secure.append(clname) if (client.get('location', 'fixed') == 'floating' or - client.get('floating', 'false').lower() == 'true'): + client.get('floating', 'false').lower() == 'true'): self.floating.append(clname) if 'password' in client.attrib: self.passwords[clname] = client.get('password') @@ -854,7 +872,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, except KeyError: self.clientgroups[clname] = [profile] self.update_client_list() - self.expire_cache() + self.cache.expire() self.states['clients.xml'] = True def _get_condition(self, element): @@ -948,19 +966,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership.setdefault(gname, []) self.group_membership[gname].append( self._aggregate_conditions(conditions)) - self.expire_cache() + self.cache.expire() self.states['groups.xml'] = True - def expire_cache(self, key=None): - self.core.metadata_cache.expire(key) - def HandleEvent(self, event): """Handle update events for data files.""" for handles, event_handler in self.handlers.items(): if handles(event): # clear the entire cache when we get an event for any # metadata file - self.expire_cache() + self.cache.expire() # clear out the list of category suppressions that # have been warned about, since this may change when @@ -1114,7 +1129,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, for p in self.group_membership[grpname]): newgroups.add(grpname) if (grpname in self.groups and - self.groups[grpname].category): + self.groups[grpname].category): categories[self.groups[grpname].category] = grpname groups.update(newgroups) for grpname, predicates in self.negated_groups.items(): @@ -1123,7 +1138,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if any(p(client, groups, categories) for p in predicates): removegroups.add(grpname) if (grpname in self.groups and - self.groups[grpname].category): + self.groups[grpname].category): del categories[self.groups[grpname].category] groups.difference_update(removegroups) return (groups, categories) @@ -1182,8 +1197,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " "been read yet") client = client.lower() - if client in self.core.metadata_cache: - return self.core.metadata_cache[client] + if client in self.cache: + return self.cache[client] if client in self.aliases: client = self.aliases[client] @@ -1280,7 +1295,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, addresses, categories, uuid, password, version, self.query) if self.core.metadata_cache_mode == 'initial': - self.core.metadata_cache[client] = rv + self.cache[client] = rv return rv def get_all_group_names(self): @@ -1373,6 +1388,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return False resolved = self.resolve_client(addresspair) if resolved.lower() == client.lower(): + self.logger.debug("Client %s address validates" % client) return True else: self.logger.error("Got request for %s from incorrect address %s" % @@ -1392,7 +1408,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, client = certinfo['commonName'] self.debug_log("Got cN %s; using as client name" % client) auth_type = self.auth.get(client, - self.core.setup['authentication']) + Bcfg2.Options.setup.authentication) elif user == 'root': id_method = 'address' try: @@ -1416,13 +1432,14 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # next we validate the address if (id_method != 'uuid' and - not self.validate_client_address(client, address)): + not self.validate_client_address(client, address)): return False if id_method == 'cert' and auth_type != 'cert+password': # remember the cert-derived client name for this connection if client in self.floating: self.session_cache[address] = (time.time(), client) + self.logger.debug("Client %s certificate validates" % client) # we are done if cert+password not required return True @@ -1449,6 +1466,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # populate the session cache if user != 'root': self.session_cache[address] = (time.time(), client) + self.logger.debug("Client %s authenticated successfully" % client) return True # pylint: enable=R0911,R0912 @@ -1472,7 +1490,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, (len(removed), removed)) for client in added.union(removed): - self.expire_cache(client) + self.cache.expire(client) def start_client_run(self, metadata): """ Hook to reread client list if the database is in use """ @@ -1481,7 +1499,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, - self.core.setup['authentication']) == 'bootstrap': + Bcfg2.Options.setup.authentication) == 'bootstrap': self.update_client(metadata.hostname, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): @@ -1589,7 +1607,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, for group in egroups: for parent in group.findall('Group'): if (parent.get('name') not in gseen and - include_group(parent.get('name'))): + include_group(parent.get('name'))): rv.append(gfmt % (parent.get('name'), parent.get('name'))) gseen.append(parent.get("name")) @@ -1597,171 +1615,3 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, rv.append('"group-%s" -> "group-%s";' % (group.get('name'), parent.get('name'))) return rv - - -class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin for :ref:`Metadata - <server-plugins-grouping-metadata>`. This checks for several things: - - * ``<Client>`` tags nested inside other ``<Client>`` tags; - * Deprecated options (like ``location="floating"``); - * Profiles that don't exist, or that aren't profile groups; - * Groups or clients that are defined multiple times; - * Multiple default groups or a default group that isn't a profile - group. - """ - - def Run(self): - self.nested_clients() - self.deprecated_options() - self.bogus_profiles() - self.duplicate_groups() - self.duplicate_default_groups() - self.duplicate_clients() - self.default_is_profile() - - @classmethod - def Errors(cls): - return {"nested-client-tags": "warning", - "deprecated-clients-options": "warning", - "nonexistent-profile-group": "error", - "non-profile-set-as-profile": "error", - "duplicate-group": "error", - "duplicate-client": "error", - "multiple-default-groups": "error", - "default-is-not-profile": "error"} - - def deprecated_options(self): - """ Check for the ``location='floating'`` option, which has - been deprecated in favor of ``floating='true'``. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - clientdata = self.metadata.clients_xml.xdata - for el in clientdata.xpath("//Client"): - loc = el.get("location") - if loc: - if loc == "floating": - floating = True - else: - floating = False - self.LintError("deprecated-clients-options", - "The location='%s' option is deprecated. " - "Please use floating='%s' instead:\n%s" % - (loc, floating, self.RenderXML(el))) - - def nested_clients(self): - """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, - which is either redundant or will never match. """ - groupdata = self.metadata.groups_xml.xdata - for el in groupdata.xpath("//Client//Client"): - self.LintError("nested-client-tags", - "Client %s nested within Client tag: %s" % - (el.get("name"), self.RenderXML(el))) - - def bogus_profiles(self): - """ Check for clients that have profiles that are either not - flagged as profile groups in ``groups.xml``, or don't exist. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - for client in self.metadata.clients_xml.xdata.findall('.//Client'): - profile = client.get("profile") - if profile not in self.metadata.groups: - self.LintError("nonexistent-profile-group", - "%s has nonexistent profile group %s:\n%s" % - (client.get("name"), profile, - self.RenderXML(client))) - elif not self.metadata.groups[profile].is_profile: - self.LintError("non-profile-set-as-profile", - "%s is set as profile for %s, but %s is not a " - "profile group:\n%s" % - (profile, client.get("name"), profile, - self.RenderXML(client))) - - def duplicate_default_groups(self): - """ Check for multiple default groups. """ - defaults = [] - for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): - if grp.get("default", "false").lower() == "true": - defaults.append(self.RenderXML(grp)) - if len(defaults) > 1: - self.LintError("multiple-default-groups", - "Multiple default groups defined:\n%s" % - "\n".join(defaults)) - - def duplicate_clients(self): - """ Check for clients that are defined more than once. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - self.duplicate_entries( - self.metadata.clients_xml.xdata.xpath("//Client"), - "client") - - def duplicate_groups(self): - """ Check for groups that are defined more than once. There are two - ways this can happen: - - 1. The group is listed twice with contradictory options. - 2. The group is listed with no options *first*, and then with - options later. - - In this context, 'first' refers to the order in which groups - are parsed; see the loop condition below and - _handle_groups_xml_event above for details. """ - groups = dict() - duplicates = dict() - for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): - grpname = grp.get("name") - if grpname in duplicates: - duplicates[grpname].append(grp) - elif set(grp.attrib.keys()).difference(['negate', 'name']): - # group has options - if grpname in groups: - duplicates[grpname] = [grp, groups[grpname]] - else: - groups[grpname] = grp - else: # group has no options - groups[grpname] = grp - for grpname, grps in duplicates.items(): - self.LintError("duplicate-group", - "Group %s is defined multiple times:\n%s" % - (grpname, - "\n".join(self.RenderXML(g) for g in grps))) - - def duplicate_entries(self, allentries, etype): - """ Generic duplicate entry finder. - - :param allentries: A list of all entries to check for - duplicates. - :type allentries: list of lxml.etree._Element - :param etype: The entry type. This will be used to determine - the error name (``duplicate-<etype>``) and for - display to the end user. - :type etype: string - """ - entries = dict() - for el in allentries: - if el.get("name") in entries: - entries[el.get("name")].append(self.RenderXML(el)) - else: - entries[el.get("name")] = [self.RenderXML(el)] - for ename, els in entries.items(): - if len(els) > 1: - self.LintError("duplicate-%s" % etype, - "%s %s is defined multiple times:\n%s" % - (etype.title(), ename, "\n".join(els))) - - def default_is_profile(self): - """ Ensure that the default group is a profile group. """ - if (self.metadata.default and - not self.metadata.groups[self.metadata.default].is_profile): - xdata = \ - self.metadata.groups_xml.xdata.xpath("//Group[@name='%s']" % - self.metadata.default)[0] - self.LintError("default-is-not-profile", - "Default group is not a profile group:\n%s" % - self.RenderXML(xdata)) diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index 8f1d03586..045e46350 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -5,28 +5,25 @@ import re import sys import glob import socket -import Bcfg2.Server -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import Plugin, Generator, StructFile, \ + PluginExecutionError -class NagiosGen(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Generator): +class NagiosGen(Plugin, Generator): """ NagiosGen is a Bcfg2 plugin that dynamically generates Nagios configuration file based on Bcfg2 data. """ __author__ = 'bcfg-dev@mcs.anl.gov' line_fmt = '\t%-32s %s' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Generator.__init__(self) + def __init__(self, core): + Plugin.__init__(self, core) + Generator.__init__(self) self.config = \ - Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, - 'config.xml'), - core.fam, should_monitor=True, - create=self.name) + StructFile(os.path.join(self.data, 'config.xml'), + should_monitor=True, create=self.name) self.Entries = { 'Path': {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} + '/etc/nagios/conf.d/bcfg2.cfg': self.createserverconfig}} self.client_attrib = {'encoding': 'ascii', 'owner': 'root', @@ -42,11 +39,11 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, def createhostconfig(self, entry, metadata): """Build host specific configuration file.""" try: - host_address = socket.gethostbyname(metadata.hostname) - except socket.gaierror: - self.logger.error("Failed to find IP address for %s" % - metadata.hostname) - raise Bcfg2.Server.Plugin.PluginExecutionError + host_address = socket.getaddrinfo(metadata.hostname, None)[0][4][0] + except socket.error: + self.logger.error() + raise PluginExecutionError("Failed to find IP address for %s" % + metadata.hostname) host_groups = [grp for grp in metadata.groups if os.path.isfile('%s/%s-group.cfg' % (self.data, grp))] host_config = ['define host {', @@ -56,7 +53,7 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, if host_groups: host_config.append(self.line_fmt % ("hostgroups", - ",".join(host_groups))) + ",".join(sorted(host_groups)))) # read the config xtra = dict() diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index 0853ea993..c5fb46c97 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -80,11 +80,9 @@ class Ohai(Bcfg2.Server.Plugin.Plugin, """The Ohai plugin is used to detect information about the client operating system. """ - name = 'Ohai' - experimental = True - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Probing.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) self.probe = lxml.etree.Element('probe', name='Ohai', source='Ohai', diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 71128d64c..cf53c5866 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -11,8 +11,8 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, create = False - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.GoalValidator.__init__(self) def validate_goals(self, metadata, goals): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 4a78f846f..cfabd8457 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -13,8 +13,7 @@ class AptCollection(Collection): overrides nothing, and defers all operations to :class:`PacSource` """ - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): + def __init__(self, metadata, sources, cachepath, basepath, debug=False): # we define an __init__ that just calls the parent __init__, # so that we can set the docstring on __init__ to something # different from the parent __init__ -- namely, the parent @@ -22,7 +21,7 @@ class AptCollection(Collection): # which we use to delineate the actual docs from the # .. autoattribute hacks we have to do to get private # attributes included in sphinx 1.0 """ - Collection.__init__(self, metadata, sources, cachepath, basepath, fam, + Collection.__init__(self, metadata, sources, cachepath, basepath, debug=debug) __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] @@ -53,10 +52,6 @@ class AptCollection(Collection): class AptSource(Source): """ Handle APT sources """ - #: :ref:`server-plugins-generators-packages-magic-groups` for - #: ``AptSource`` are "apt", "debian", "ubuntu", and "nexenta" - basegroups = ['apt', 'debian', 'ubuntu', 'nexenta'] - #: AptSource sets the ``type`` on Package entries to "deb" ptype = 'deb' @@ -74,13 +69,11 @@ class AptSource(Source): else: return ["%sPackages.gz" % self.rawurl] - def read_files(self): + def read_files(self): # pylint: disable=R0912 bdeps = dict() + brecs = dict() bprov = dict() self.essentialpkgs = set() - depfnames = ['Depends', 'Pre-Depends'] - if self.recommended: - depfnames.append('Recommends') for fname in self.files: if not self.rawurl: barch = [x @@ -92,6 +85,7 @@ class AptSource(Source): barch = self.arches[0] if barch not in bdeps: bdeps[barch] = dict() + brecs[barch] = dict() bprov[barch] = dict() try: reader = gzip.GzipFile(fname) @@ -106,10 +100,11 @@ class AptSource(Source): pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: if words[1].strip() == 'yes': self.essentialpkgs.add(pkgname) - elif words[0] in depfnames: + elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): if '|' in dep: @@ -120,17 +115,24 @@ class AptSource(Source): barch, vindex) vindex += 1 - bdeps[barch][pkgname].append(dyn_dname) + + if words[0] == 'Recommends': + brecs[barch][pkgname].append(dyn_dname) + else: + bdeps[barch][pkgname].append(dyn_dname) bprov[barch][dyn_dname] = set(cdeps) else: raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() - bdeps[barch][pkgname].append(raw_dep) + if words[0] == 'Recommends': + brecs[barch][pkgname].append(raw_dep) + else: + bdeps[barch][pkgname].append(raw_dep) elif words[0] == 'Provides': for pkg in words[1].split(','): dname = pkg.rstrip().strip() if dname not in bprov[barch]: bprov[barch][dname] = set() bprov[barch][dname].add(pkgname) - self.process_files(bdeps, bprov) + self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 39c51f351..004e27874 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -73,17 +73,17 @@ The Collection Module --------------------- """ -import sys import copy -import logging import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Plugin +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import any, md5 # pylint: disable=W0622 +from Bcfg2.Server.FileMonitor import get_fam +from Bcfg2.Server.Statistics import track_statistics -LOGGER = logging.getLogger(__name__) - -class Collection(list, Bcfg2.Server.Plugin.Debuggable): +class Collection(list, Debuggable): """ ``Collection`` objects represent the set of :class:`Bcfg2.Server.Plugins.Packages.Source` objects that apply to a given client, and can be used to query all software @@ -93,8 +93,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): #: Whether or not this Packages backend supports package groups __package_groups__ = False - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): + def __init__(self, metadata, sources, cachepath, basepath, debug=False): """ :param metadata: The client metadata for this collection :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata @@ -111,29 +110,24 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): directory, where more permanent data can be stored :type basepath: string - :param fam: A file monitor object to use if this Collection - needs to monitor for file activity - :type fam: Bcfg2.Server.FileMonitor.FileMonitor :param debug: Enable debugging output :type debug: bool .. ----- .. autoattribute:: __package_groups__ """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) + Debuggable.__init__(self) list.__init__(self, sources) - self.debug_flag = debug + self.debug_flag = self.debug_flag or debug self.metadata = metadata self.basepath = basepath self.cachepath = cachepath self.virt_pkgs = dict() - self.fam = fam + self.fam = get_fam() try: - self.setup = sources[0].setup self.ptype = sources[0].ptype except IndexError: - self.setup = None self.ptype = "unknown" @property @@ -204,19 +198,6 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): return sorted(list(set(groups))) @property - def basegroups(self): - """ Get a list of group names used by this Collection type in - resolution of - :ref:`server-plugins-generators-packages-magic-groups`. - - The base implementation simply aggregates the results of - :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.basegroups`.""" - groups = set() - for source in self: - groups.update(source.basegroups) - return list(groups) - - @property def cachefiles(self): """ A list of the full path to all cachefiles used by this collection. @@ -229,7 +210,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): cachefiles.add(source.cachefile) return list(cachefiles) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_groups(self, grouplist): """ Given a list of package group names, return a dict of ``<group name>: <list of packages>``. This method is provided @@ -250,7 +231,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): rv[group] = self.get_group(group, ptype) return rv - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_group(self, group, ptype=None): """ Get the list of packages of the given type in a package group. @@ -308,7 +289,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): return any(source.is_virtual_package(self.metadata, package) for source in self) - def get_deps(self, package): + def get_deps(self, package, recs=None): """ Get a list of the dependencies of the given package. The base implementation simply aggregates the results of @@ -318,9 +299,14 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): :type package: string :returns: list of strings, but see :ref:`pkg-objects` """ + recommended = None + if recs and package in recs: + recommended = recs[package] + for source in self: if source.is_package(self.metadata, package): - return source.get_deps(self.metadata, package) + return source.get_deps(self.metadata, package, recommended) + return [] def get_essential(self): @@ -386,20 +372,6 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): for source in self: source.filter_unknown(unknown) - def magic_groups_match(self): - """ Returns True if the client's - :ref:`server-plugins-generators-packages-magic-groups` match - the magic groups for any of the sources contained in this - Collection. - - The base implementation returns True if any source - :func:`Bcfg2.Server.Plugins.Packages.Source.Source.magic_groups_match` - returns True. - - :returns: bool - """ - return any(s.magic_groups_match(self.metadata) for s in self) - def build_extra_structures(self, independent): """ Add additional entries to the ``<Independent/>`` section of the final configuration. This can be used to handle, e.g., @@ -476,9 +448,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ for pkg in pkglist: lxml.etree.SubElement(entry, 'BoundPackage', name=pkg, - version=self.setup.cfp.get("packages", - "version", - default="auto"), + version=Bcfg2.Options.setup.packages_version, type=self.ptype, origin='Packages') def get_new_packages(self, initial, complete): @@ -499,8 +469,9 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ return list(complete.difference(initial)) - @Bcfg2.Server.Plugin.track_statistics() - def complete(self, packagelist): # pylint: disable=R0912,R0914 + @track_statistics() + def complete(self, packagelist, # pylint: disable=R0912,R0914 + recommended=None): """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the @@ -564,7 +535,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): self.debug_log("Packages: handling package requirement %s" % (current,)) packages.add(current) - deps = self.get_deps(current) + deps = self.get_deps(current, recommended) newdeps = set(deps).difference(examined) if newdeps: self.debug_log("Packages: Package %s added requirements %s" @@ -630,22 +601,8 @@ def get_collection_class(source_type): :type source_type: string :returns: type - the Collection subclass that should be used to instantiate an object to contain sources of the given type. """ - modname = "Bcfg2.Server.Plugins.Packages.%s" % source_type.title() - try: - module = sys.modules[modname] - except KeyError: - try: - module = __import__(modname).Server.Plugins.Packages - except ImportError: - msg = "Packages: Unknown source type %s" % source_type - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - try: - cclass = getattr(module, source_type.title() + "Collection") - except AttributeError: - msg = "Packages: No collection class found for %s sources" % \ - source_type - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - return cclass + for mod in Bcfg2.Options.setup.packages_backends: + if mod.__name__.endswith(".%s" % source_type.title()): + return getattr(mod, "%sCollection" % source_type.title()) + raise Bcfg2.Server.Plugin.PluginExecutionError( + "Packages: No collection class found for %s sources" % source_type) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 99aed5ce5..5f4d2ea41 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -12,8 +12,7 @@ class PacCollection(Collection): overrides nothing, and defers all operations to :class:`PacSource` """ - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): + def __init__(self, metadata, sources, cachepath, basepath, debug=False): # we define an __init__ that just calls the parent __init__, # so that we can set the docstring on __init__ to something # different from the parent __init__ -- namely, the parent @@ -21,7 +20,7 @@ class PacCollection(Collection): # which we use to delineate the actual docs from the # .. autoattribute hacks we have to do to get private # attributes included in sphinx 1.0 """ - Collection.__init__(self, metadata, sources, cachepath, basepath, fam, + Collection.__init__(self, metadata, sources, cachepath, basepath, debug=debug) __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] @@ -29,10 +28,6 @@ class PacCollection(Collection): class PacSource(Source): """ Handle Pacman sources """ - #: :ref:`server-plugins-generators-packages-magic-groups` for - #: ``PacSource`` are "arch" and "parabola" - basegroups = ['arch', 'parabola'] - #: PacSource sets the ``type`` on Package entries to "pacman" ptype = 'pacman' diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index c47e18201..1af046ec0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -4,12 +4,12 @@ import os import sys import Bcfg2.Server.Plugin +from Bcfg2.Server.Statistics import track_statistics from Bcfg2.Server.Plugins.Packages.Source import SourceInitError # pylint: disable=E0012,R0924 -class PackagesSources(Bcfg2.Server.Plugin.StructFile, - Bcfg2.Server.Plugin.Debuggable): +class PackagesSources(Bcfg2.Server.Plugin.StructFile): """ PackagesSources handles parsing of the :mod:`Bcfg2.Server.Plugins.Packages` ``sources.xml`` file, and the creation of the appropriate @@ -19,7 +19,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, __identifier__ = None create = "Sources" - def __init__(self, filename, cachepath, fam, packages, setup): + def __init__(self, filename, cachepath, packages): """ :param filename: The full path to ``sources.xml`` :type filename: string @@ -27,21 +27,15 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, :class:`Bcfg2.Server.Plugins.Packages.Source.Source` data will be cached :type cachepath: string - :param fam: The file access monitor to use to create watches - on ``sources.xml`` and any XIncluded files. - :type fam: Bcfg2.Server.FileMonitor.FileMonitor :param packages: The Packages plugin object ``sources.xml`` is being parsed on behalf of (i.e., the calling object) :type packages: Bcfg2.Server.Plugins.Packages.Packages - :param setup: A Bcfg2 options dict - :type setup: dict :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` - If ``sources.xml`` cannot be read """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, + Bcfg2.Server.Plugin.StructFile.__init__(self, filename, should_monitor=True) #: The full path to the directory where @@ -57,8 +51,6 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, err = sys.exc_info()[1] self.logger.error("Could not create Packages cache at %s: %s" % (self.cachepath, err)) - #: The Bcfg2 options dict - self.setup = setup #: The :class:`Bcfg2.Server.Plugins.Packages.Packages` that #: instantiated this ``PackagesSources`` object @@ -72,10 +64,9 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, self.parsed = set() def set_debug(self, debug): - Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) + Bcfg2.Server.Plugin.StructFile.set_debug(self, debug) for source in self.entries: source.set_debug(debug) - set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ def HandleEvent(self, event=None): """ HandleEvent is called whenever the FAM registers an event. @@ -106,7 +97,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, load its data. """ return sorted(list(self.parsed)) == sorted(self.extras) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.entries = [] @@ -120,7 +111,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, ``Index`` is responsible for calling :func:`source_from_xml` for each ``Source`` tag in each file. """ - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def source_from_xml(self, xsource): """ Create a :class:`Bcfg2.Server.Plugins.Packages.Source.Source` subclass @@ -141,19 +132,17 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, xsource.get("url")))) return None - try: - module = getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cls = getattr(module, "%sSource" % stype.title()) - except (ImportError, AttributeError): - err = sys.exc_info()[1] - self.logger.error("Packages: Unknown source type %s (%s)" % (stype, - err)) + cls = None + for mod in Bcfg2.Options.setup.packages_backends: + if mod.__name__.endswith(".%s" % stype.title()): + cls = getattr(mod, "%sSource" % stype.title()) + break + else: + self.logger.error("Packages: Unknown source type %s" % stype) return None try: - source = cls(self.cachepath, xsource, self.setup) + source = cls(self.cachepath, xsource) except SourceInitError: err = sys.exc_info()[1] self.logger.error("Packages: %s" % err) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py new file mode 100644 index 000000000..e393cabfe --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py @@ -0,0 +1,86 @@ +""" pkgng backend for :mod:`Bcfg2.Server.Plugins.Packages` """ + +import lzma +import tarfile + +try: + import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): + import simplejson as json + +from Bcfg2.Server.Plugins.Packages.Collection import Collection +from Bcfg2.Server.Plugins.Packages.Source import Source + + +class PkgngCollection(Collection): + """ Handle collections of pkgng sources. This is a no-op object + that simply inherits from + :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`, + overrides nothing, and defers all operations to :class:`PacSource` + """ + + def __init__(self, metadata, sources, cachepath, basepath, debug=False): + # we define an __init__ that just calls the parent __init__, + # so that we can set the docstring on __init__ to something + # different from the parent __init__ -- namely, the parent + # __init__ docstring, minus everything after ``.. -----``, + # which we use to delineate the actual docs from the + # .. autoattribute hacks we have to do to get private + # attributes included in sphinx 1.0 """ + Collection.__init__(self, metadata, sources, cachepath, basepath, + debug=debug) + __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] + + +class PkgngSource(Source): + """ Handle pkgng sources """ + + #: PkgngSource sets the ``type`` on Package entries to "pkgng" + ptype = 'pkgng' + + @property + def urls(self): + """ A list of URLs to the base metadata file for each + repository described by this source. """ + if not self.rawurl: + rv = [] + for part in self.components: + for arch in self.arches: + rv.append("%s/freebsd:%s:%s/%s/packagesite.txz" % + (self.url, self.version, arch, part)) + return rv + else: + return ["%s/packagesite.txz" % self.rawurl] + + def read_files(self): + bdeps = dict() + for fname in self.files: + if not self.rawurl: + abi = [x + for x in fname.split('@') + if x.startswith('freebsd:')][0][8:] + barch = ':'.join(abi.split(':')[1:]) + else: + # RawURL entries assume that they only have one <Arch></Arch> + # element and that it is the architecture of the source. + barch = self.arches[0] + if barch not in bdeps: + bdeps[barch] = dict() + try: + tar = tarfile.open(fileobj=lzma.LZMAFile(fname)) + reader = tar.extractfile('packagesite.yaml') + except: + self.logger.error("Packages: Failed to read file %s" % fname) + raise + for line in reader.readlines(): + if not isinstance(line, str): + line = line.decode('utf-8') + pkg = json.loads(line) + pkgname = pkg['name'] + self.pkgnames.add(pkgname) + if 'deps' in pkg: + bdeps[barch][pkgname] = pkg['deps'].keys() + self.process_files(bdeps, dict()) + read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 30cdd543f..67ada2399 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -27,7 +27,6 @@ in your ``Source`` subclass: * :func:`Source.urls` * :func:`Source.read_files` -* :attr:`Source.basegroups` Additionally, you may want to consider overriding the following methods and attributes: @@ -50,10 +49,11 @@ in your ``Source`` subclass. For an example of this kind of import os import re import sys -import Bcfg2.Server.Plugin +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ cPickle, md5 +from Bcfg2.Server.Statistics import track_statistics def fetch_url(url): @@ -92,7 +92,7 @@ class SourceInitError(Exception): REPO_RE = re.compile(r'(?:pulp/repos/|/RPMS\.|/)([^/]+)/?$') -class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 +class Source(Debuggable): # pylint: disable=R0902 """ ``Source`` objects represent a single <Source> tag in ``sources.xml``. Note that a single Source tag can itself describe multiple repositories (if it uses the "url" attribute @@ -106,28 +106,21 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 those features. """ - #: The list of - #: :ref:`server-plugins-generators-packages-magic-groups` that - #: make sources of this type available to clients. - basegroups = [] - #: The Package type handled by this Source class. The ``type`` #: attribute of Package entries will be set to the value ``ptype`` #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`. ptype = None - def __init__(self, basepath, xsource, setup): # pylint: disable=R0912 + def __init__(self, basepath, xsource): # pylint: disable=R0912 """ :param basepath: The base filesystem path under which cache data for this source should be stored :type basepath: string :param xsource: The XML tag that describes this source :type source: lxml.etree._Element - :param setup: A Bcfg2 options dict - :type setup: dict :raises: :class:`Bcfg2.Server.Plugins.Packages.Source.SourceInitError` """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) + Debuggable.__init__(self) #: The base filesystem path under which cache data for this #: source should be stored @@ -136,9 +129,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: The XML tag that describes this source self.xsource = xsource - #: A Bcfg2 options dict - self.setup = setup - #: A set of package names that are deemed "essential" by this #: source self.essentialpkgs = set() @@ -259,6 +249,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` self.provides = dict() + #: A dict of ``<package name>`` -> ``<list of recommended + #: symbols>``. This will not necessarily be populated. + self.recommends = dict() + #: The file (or directory) used for this source's cache data self.cachefile = os.path.join(self.basepath, "cache-%s" % self.cachekey) @@ -312,8 +306,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :return: list of strings - group names """ return sorted(list(set([g for g in metadata.groups - if (g in self.basegroups or - g in self.groups or + if (g in self.groups or g in self.arches)]))) def load_state(self): @@ -325,7 +318,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :raises: cPickle.UnpicklingError - If the saved data is corrupt """ data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, - self.essentialpkgs) = cPickle.load(data) + self.essentialpkgs, self.recommends) = cPickle.load(data) def save_state(self): """ Save state to :attr:`cachefile`. If caching and @@ -333,10 +326,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 does not need to be implemented. """ cache = open(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides, - self.essentialpkgs), cache, 2) + self.essentialpkgs, self.recommends), cache, 2) cache.close() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def setup_data(self, force_update=False): """ Perform all data fetching and setup tasks. For most backends, this involves downloading all metadata from the @@ -533,13 +526,14 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 as its final step.""" pass - def process_files(self, dependencies, provides): + def process_files(self, dependencies, # pylint: disable=R0912,W0102 + provides, recommends=dict()): """ Given dicts of depends and provides generated by :func:`read_files`, this generates :attr:`deps` and :attr:`provides` and calls :func:`save_state` to save the cached data to disk. - Both arguments are dicts of dicts of lists. Keys are the + All arguments are dicts of dicts of lists. Keys are the arches of packages contained in this source; values are dicts whose keys are package names and values are lists of either dependencies for each package the symbols provided by each @@ -551,14 +545,20 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :param provides: A dict of symbols provided by packages in this repository. :type provides: dict; see above. + :param recommends: A dict of recommended dependencies + found for this source. + :type recommends: dict; see above. """ self.deps['global'] = dict() + self.recommends['global'] = dict() self.provides['global'] = dict() for barch in dependencies: self.deps[barch] = dict() + self.recommends[barch] = dict() self.provides[barch] = dict() for pkgname in self.pkgnames: pset = set() + rset = set() for barch in dependencies: if pkgname not in dependencies[barch]: dependencies[barch][pkgname] = [] @@ -568,6 +568,18 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 else: for barch in dependencies: self.deps[barch][pkgname] = dependencies[barch][pkgname] + + for barch in recommends: + if pkgname not in recommends[barch]: + recommends[barch][pkgname] = [] + rset.add(tuple(recommends[barch][pkgname])) + if len(rset) == 1: + self.recommends['global'][pkgname] = rset.pop() + else: + for barch in recommends: + self.recommends[barch][pkgname] = \ + recommends[barch][pkgname] + provided = set() for bprovided in list(provides.values()): provided.update(set(bprovided)) @@ -645,16 +657,15 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 def applies(self, metadata): """ Return true if this source applies to the given client, - i.e., the client is in all necessary groups and - :ref:`server-plugins-generators-packages-magic-groups`. + i.e., the client is in all necessary groups. :param metadata: The client metadata to check to see if this source applies :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: bool """ - # check base groups - if not self.magic_groups_match(metadata): + # check arch groups + if not self.arch_groups_match(metadata): return False # check Group/Client tags from sources.xml @@ -676,17 +687,24 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 """ return ['global'] + [a for a in self.arches if a in metadata.groups] - def get_deps(self, metadata, package): + def get_deps(self, metadata, package, recommended=None): """ Get a list of the dependencies of the given package. :param package: The name of the symbol :type package: string :returns: list of strings """ + recs = [] + if ((recommended is None and self.recommended) or + (recommended and recommended.lower() == 'true')): + for arch in self.get_arches(metadata): + if package in self.recommends[arch]: + recs.extend(self.recommends[arch][package]) + for arch in self.get_arches(metadata): if package in self.deps[arch]: - return self.deps[arch][package] - return [] + recs.extend(self.deps[arch][package]) + return recs def get_provides(self, metadata, package): """ Get a list of all symbols provided by the given package. @@ -725,29 +743,13 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 """ return [] - def magic_groups_match(self, metadata): - """ Returns True if the client's - :ref:`server-plugins-generators-packages-magic-groups` match - the magic groups this source. Also returns True if magic - groups are off in the configuration and the client's - architecture matches (i.e., architecture groups are *always* - checked). + def arch_groups_match(self, metadata): + """ Returns True if the client is in an arch group that + matches the arch of this source. :returns: bool """ - found_arch = False for arch in self.arches: if arch in metadata.groups: - found_arch = True - break - if not found_arch: - return False - - if not self.setup.cfp.getboolean("packages", "magic_groups", - default=False): - return True - else: - for group in self.basegroups: - if group in metadata.groups: - return True - return False + return True + return False diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 6139a28b5..f26ded4c5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -60,8 +60,10 @@ import socket import logging import lxml.etree import Bcfg2.Server.Plugin +import Bcfg2.Server.FileMonitor from lockfile import FileLock from Bcfg2.Utils import Executor +from distutils.spawn import find_executable # pylint: disable=E0611 # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -69,6 +71,7 @@ from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url +from Bcfg2.Server.Statistics import track_statistics LOGGER = logging.getLogger(__name__) @@ -107,13 +110,36 @@ PULPSERVER = None PULPCONFIG = None -def _setup_pulp(setup): +options = [ # pylint: disable=C0103 + Bcfg2.Options.Common.client_timeout, + Bcfg2.Options.PathOption( + cf=("packages:yum", "helper"), dest="yum_helper", + help="Path to the bcfg2-yum-helper executable"), + Bcfg2.Options.BooleanOption( + cf=("packages:yum", "use_yum_libraries"), + help="Use Python yum libraries"), + Bcfg2.Options.PathOption( + cf=("packages:yum", "gpg_keypath"), default="/etc/pki/rpm-gpg", + help="GPG key path on the client"), + Bcfg2.Options.Option( + cf=("packages:yum", "*"), dest="yum_options", + help="Other yum options to include in generated yum configs")] +if HAS_PULP: + options.append( + Bcfg2.Options.Option( + cf=("packages:pulp", "username"), dest="pulp_username", + help="Username for Pulp authentication")) + options.append( + Bcfg2.Options.Option( + cf=("packages:pulp", "password"), dest="pulp_password", + help="Password for Pulp authentication")) + + +def _setup_pulp(): """ Connect to a Pulp server and pass authentication credentials. This only needs to be called once, but multiple calls won't hurt anything. - :param setup: A Bcfg2 options dict - :type setup: dict :returns: :class:`pulp.client.api.server.PulpServer` """ global PULPSERVER, PULPCONFIG @@ -124,19 +150,6 @@ def _setup_pulp(setup): raise Bcfg2.Server.Plugin.PluginInitError(msg) if PULPSERVER is None: - try: - username = setup.cfp.get("packages:pulp", "username") - password = setup.cfp.get("packages:pulp", "password") - except ConfigParser.NoSectionError: - msg = "Packages: No [pulp] section found in bcfg2.conf" - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - except ConfigParser.NoOptionError: - msg = "Packages: Required option not found in bcfg2.conf: %s" % \ - sys.exc_info()[1] - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - PULPCONFIG = ConsumerConfig() serveropts = PULPCONFIG.server @@ -144,7 +157,9 @@ def _setup_pulp(setup): int(serveropts['port']), serveropts['scheme'], serveropts['path']) - PULPSERVER.set_basic_auth_credentials(username, password) + PULPSERVER.set_basic_auth_credentials( + Bcfg2.Options.setup.pulp_username, + Bcfg2.Options.setup.pulp_password) server.set_active_server(PULPSERVER) return PULPSERVER @@ -175,7 +190,7 @@ class PulpCertificateSet(Bcfg2.Server.Plugin.EntrySet): #: The path to certificates on consumer machines certpath = "/etc/pki/consumer/cert.pem" - def __init__(self, path, fam): + def __init__(self, path): """ :param path: The path to the directory where Pulp consumer certificates will be stored @@ -193,7 +208,7 @@ class PulpCertificateSet(Bcfg2.Server.Plugin.EntrySet): important='true', sensitive='true', paranoid=self.metadata['paranoid']) - self.fam = fam + self.fam = Bcfg2.Server.FileMonitor.get_fam() self.fam.AddMonitor(path, self) def HandleEvent(self, event): @@ -274,9 +289,8 @@ class YumCollection(Collection): #: :class:`PulpCertificateSet` object used to handle Pulp certs pulp_cert_set = None - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): - Collection.__init__(self, metadata, sources, cachepath, basepath, fam, + def __init__(self, metadata, sources, cachepath, basepath, debug=False): + Collection.__init__(self, metadata, sources, cachepath, basepath, debug=debug) self.keypath = os.path.join(self.cachepath, "keys") @@ -297,13 +311,15 @@ class YumCollection(Collection): if not os.path.exists(self.cachefile): self.debug_log("Creating common cache %s" % self.cachefile) os.mkdir(self.cachefile) - if not self.disableMetaData: + if Bcfg2.Options.setup.packages_metadata: self.setup_data() + self.cmd = Executor() else: self.cachefile = None + self.cmd = None if HAS_PULP and self.has_pulp_sources: - _setup_pulp(self.setup) + _setup_pulp() if self.pulp_cert_set is None: certdir = os.path.join( self.basepath, @@ -319,28 +335,7 @@ class YumCollection(Collection): self.logger.error("Could not create Pulp consumer " "cert directory at %s: %s" % (certdir, err)) - self.__class__.pulp_cert_set = PulpCertificateSet(certdir, - self.fam) - - @property - def disableMetaData(self): # pylint: disable=C0103 - """ Report whether or not metadata processing is enabled. - This duplicates code in Packages/__init__.py, and can probably - be removed in Bcfg2 1.4 when we have a module-level setup - object. """ - if self.setup is None: - return True - try: - return not self.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled" - return self.setup.cfp.get( - "packages", - "metadata", - default="enabled").lower() == "disabled" + self.__class__.pulp_cert_set = PulpCertificateSet(certdir) @property def __package_groups__(self): @@ -348,24 +343,18 @@ class YumCollection(Collection): @property def helper(self): - """ The full path to :file:`bcfg2-yum-helper`. First, we - check in the config file to see if it has been explicitly - specified; next we see if it's in $PATH (which we do by making - a call to it; I wish there was a way to do this without - forking, but apparently not); finally we check in /usr/sbin, - the default location. """ + """The full path to :file:`bcfg2-yum-helper`. First, we check in the + config file to see if it has been explicitly specified; next + we see if it's in $PATH; finally we default to /usr/sbin, the + default location. """ # pylint: disable=W0212 - if not self.__class__._helper: - try: - self.__class__._helper = self.setup.cfp.get("packages:yum", - "helper") - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + if not self._helper: + self.__class__._helper = Bcfg2.Options.setup.yum_helper + if not self.__class__._helper: # first see if bcfg2-yum-helper is in PATH - try: - self.debug_log("Checking for bcfg2-yum-helper in $PATH") - self.cmd.run(['bcfg2-yum-helper']) - self.__class__._helper = 'bcfg2-yum-helper' - except OSError: + self.debug_log("Checking for bcfg2-yum-helper in $PATH") + self.__class__._helper = find_executable('bcfg2-yum-helper') + if not self.__class__._helper: self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" return self.__class__._helper # pylint: enable=W0212 @@ -374,9 +363,7 @@ class YumCollection(Collection): def use_yum(self): """ True if we should use the yum Python libraries, False otherwise """ - return HAS_YUM and self.setup.cfp.getboolean("packages:yum", - "use_yum_libraries", - default=False) + return HAS_YUM and Bcfg2.Options.setup.use_yum_libraries @property def has_pulp_sources(self): @@ -393,7 +380,7 @@ class YumCollection(Collection): cachefiles.add(self.cachefile) return list(cachefiles) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def write_config(self): """ Write the server-side config file to :attr:`cfgfile` based on the data from :func:`get_config`""" @@ -412,15 +399,15 @@ class YumCollection(Collection): debuglevel="0", sslverify="0", reposdir="/dev/null") - if self.setup['debug']: + if Bcfg2.Options.setup.debug: mainopts['debuglevel'] = "5" - elif self.setup['verbose']: + elif Bcfg2.Options.setup.verbose: mainopts['debuglevel'] = "2" try: - for opt in self.setup.cfp.options("packages:yum"): + for opt, val in Bcfg2.Options.setup.yum_options.items(): if opt not in self.option_blacklist: - mainopts[opt] = self.setup.cfp.get("packages:yum", opt) + mainopts[opt] = val except ConfigParser.NoSectionError: pass @@ -515,7 +502,7 @@ class YumCollection(Collection): return "# This config was generated automatically by the Bcfg2 " \ "Packages plugin\n\n" + buf.getvalue() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def build_extra_structures(self, independent): """ Add additional entries to the ``<Independent/>`` section of the final configuration. This adds several kinds of @@ -561,8 +548,7 @@ class YumCollection(Collection): for key in needkeys: # figure out the path of the key on the client - keydir = self.setup.cfp.get("global", "gpg_keypath", - default="/etc/pki/rpm-gpg") + keydir = Bcfg2.Options.setup.gpg_keypath remotekey = os.path.join(keydir, os.path.basename(key)) localkey = os.path.join(self.keypath, os.path.basename(key)) kdata = open(localkey).read() @@ -607,7 +593,7 @@ class YumCollection(Collection): # each pulp source can only have one arch, so we don't # have to check the arch in url_map if (source.pulp_id and - source.pulp_id not in consumer['repoids']): + source.pulp_id not in consumer['repoids']): try: consumerapi.bind(self.metadata.hostname, source.pulp_id) @@ -622,7 +608,7 @@ class YumCollection(Collection): name=self.pulp_cert_set.certpath) self.pulp_cert_set.bind_entry(crt, self.metadata) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _get_pulp_consumer(self, consumerapi=None): """ Get a Pulp consumer object for the client. @@ -651,7 +637,7 @@ class YumCollection(Collection): "%s" % err) return consumer - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _add_gpg_instances(self, keyentry, localkey, remotekey, keydata=None): """ Add GPG keys instances to a ``Package`` entry. This is called from :func:`build_extra_structures` to add GPG keys to @@ -694,7 +680,7 @@ class YumCollection(Collection): self.logger.error("Packages: Could not read GPG key %s: %s" % (localkey, err)) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_groups(self, grouplist): """ If using the yum libraries, given a list of package group names, return a dict of ``<group name>: <list of packages>``. @@ -781,8 +767,7 @@ class YumCollection(Collection): """ Given a package tuple, return a dict of attributes suitable for applying to either a Package or an Instance tag """ - attrs = dict(version=self.setup.cfp.get("packages", "version", - default="auto")) + attrs = dict(version=Bcfg2.Options.setup.packages_version) if attrs['version'] == 'any' or not isinstance(pkgtup, tuple): return attrs @@ -871,8 +856,8 @@ class YumCollection(Collection): new.append(pkg) return new - @Bcfg2.Server.Plugin.track_statistics() - def complete(self, packagelist): + @track_statistics() + def complete(self, packagelist, recommended=None): """ Build a complete list of all packages and their dependencies. When using the Python yum libraries, this defers to the @@ -890,7 +875,7 @@ class YumCollection(Collection): resolved. """ if not self.use_yum: - return Collection.complete(self, packagelist) + return Collection.complete(self, packagelist, recommended) lock = FileLock(os.path.join(self.cachefile, "lock")) slept = 0 @@ -925,7 +910,7 @@ class YumCollection(Collection): else: return set(), set() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def call_helper(self, command, inputdata=None): """ Make a call to :ref:`bcfg2-yum-helper`. The yum libs have horrific memory leaks, so apparently the right way to get @@ -943,22 +928,20 @@ class YumCollection(Collection): ``bcfg2-yum-helper`` command. """ cmd = [self.helper, "-c", self.cfgfile] - if self.setup['verbose']: + if Bcfg2.Options.setup.verbose: cmd.append("-v") if self.debug_flag: - if not self.setup['verbose']: - # ensure that running in debug gets -vv, even if - # verbose is not enabled - cmd.append("-v") - cmd.append("-v") + cmd.append("-d") cmd.append(command) self.debug_log("Packages: running %s" % " ".join(cmd)) if inputdata: - result = self.cmd.run(cmd, timeout=self.setup['client_timeout'], + result = self.cmd.run(cmd, + timeout=Bcfg2.Options.setup.client_timeout, inputdata=json.dumps(inputdata)) else: - result = self.cmd.run(cmd, timeout=self.setup['client_timeout']) + result = self.cmd.run(cmd, + timeout=Bcfg2.Options.setup.client_timeout) if not result.success: self.logger.error("Packages: error running bcfg2-yum-helper: %s" % result.error) @@ -1021,20 +1004,16 @@ class YumCollection(Collection): class YumSource(Source): """ Handle yum sources """ - #: :ref:`server-plugins-generators-packages-magic-groups` for - #: ``YumSource`` are "yum", "redhat", "centos", and "fedora" - basegroups = ['yum', 'redhat', 'centos', 'fedora'] - #: YumSource sets the ``type`` on Package entries to "yum" ptype = 'yum' - def __init__(self, basepath, xsource, setup): - Source.__init__(self, basepath, xsource, setup) + def __init__(self, basepath, xsource): + Source.__init__(self, basepath, xsource) self.pulp_id = None if HAS_PULP and xsource.get("pulp_id"): self.pulp_id = xsource.get("pulp_id") - _setup_pulp(self.setup) + _setup_pulp() repoapi = RepositoryAPI() try: self.repo = repoapi.repository(self.pulp_id) @@ -1077,9 +1056,7 @@ class YumSource(Source): def use_yum(self): """ True if we should use the yum Python libraries, False otherwise """ - return HAS_YUM and self.setup.cfp.getboolean("packages:yum", - "use_yum_libraries", - default=False) + return HAS_YUM and Bcfg2.Options.setup.use_yum_libraries def save_state(self): """ If using the builtin yum parser, save state to @@ -1161,7 +1138,7 @@ class YumSource(Source): self.file_to_arch[self.escape_url(fullurl)] = arch return urls - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def read_files(self): """ When using the builtin yum parser, read and parse locally downloaded metadata files. This diverges from the stock @@ -1209,7 +1186,7 @@ class YumSource(Source): self.packages[key].difference(self.packages['global']) self.save_state() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def parse_filelist(self, data, arch): """ parse filelists.xml.gz data """ if arch not in self.filemap: @@ -1223,7 +1200,7 @@ class YumSource(Source): self.filemap[arch][fentry.text] = \ set([pkg.get('name')]) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def parse_primary(self, data, arch): """ parse primary.xml.gz data """ if arch not in self.packages: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py new file mode 100644 index 000000000..48304d26e --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py @@ -0,0 +1,399 @@ +""" Libraries for bcfg2-yum-helper plugin, used if yum library support +is enabled. The yum libs have horrific memory leaks, so apparently +the right way to get around that in long-running processes it to have +a short-lived helper. No, seriously -- check out the yum-updatesd +code. It's pure madness. """ + +import os +import sys +import yum +import logging +import Bcfg2.Options +import Bcfg2.Logger +from Bcfg2.Compat import wraps +from lockfile import FileLock, LockTimeout +try: + import json +except ImportError: + import simplejson as json + + +def pkg_to_tuple(package): + """ json doesn't distinguish between tuples and lists, but yum + does, so we convert a package in list format to one in tuple + format """ + if isinstance(package, list): + return tuple(package) + else: + return package + + +def pkgtup_to_string(package): + """ given a package tuple, return a human-readable string + describing the package """ + if package[3] in ['auto', 'any']: + return package[0] + + rv = [package[0], "-"] + if package[2]: + rv.extend([package[2], ':']) + rv.extend([package[3], '-', package[4]]) + if package[1]: + rv.extend(['.', package[1]]) + return ''.join(str(e) for e in rv) + + +class YumHelper(object): + """ Yum helper base object """ + + def __init__(self, cfgfile, verbose=1): + self.cfgfile = cfgfile + self.yumbase = yum.YumBase() + # pylint: disable=E1121,W0212 + try: + self.yumbase.preconf.debuglevel = verbose + self.yumbase.preconf.fn = cfgfile + self.yumbase._getConfig() + except AttributeError: + self.yumbase._getConfig(cfgfile, debuglevel=verbose) + # pylint: enable=E1121,W0212 + self.logger = logging.getLogger(self.__class__.__name__) + + +class DepSolver(YumHelper): + """ Yum dependency solver. This is used for operations that only + read from the yum cache, and thus operates in cacheonly mode. """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + # internally, yum uses an integer, not a boolean, for conf.cache + self.yumbase.conf.cache = 1 + self._groups = None + + def get_groups(self): + """ getter for the groups property """ + if self._groups is not None: + return self._groups + else: + return ["noarch"] + + def set_groups(self, groups): + """ setter for the groups property """ + self._groups = set(groups).union(["noarch"]) + + groups = property(get_groups, set_groups) + + def get_package_object(self, pkgtup, silent=False): + """ given a package tuple, get a yum package object """ + try: + matches = yum.packageSack.packagesNewestByName( + self.yumbase.pkgSack.searchPkgTuple(pkgtup)) + except yum.Errors.PackageSackError: + if not silent: + self.logger.warning("Package '%s' not found" % + self.get_package_name(pkgtup)) + matches = [] + except yum.Errors.RepoError: + err = sys.exc_info()[1] + self.logger.error("Temporary failure loading metadata for %s: %s" % + (self.get_package_name(pkgtup), err)) + matches = [] + + pkgs = self._filter_arch(matches) + if pkgs: + return pkgs[0] + else: + return None + + def get_group(self, group, ptype="default"): + """ Resolve a package group name into a list of packages """ + if group.startswith("@"): + group = group[1:] + + try: + if self.yumbase.comps.has_group(group): + group = self.yumbase.comps.return_group(group) + else: + self.logger.error("%s is not a valid group" % group) + return [] + except yum.Errors.GroupsError: + err = sys.exc_info()[1] + self.logger.warning(err) + return [] + + if ptype == "default": + return [p + for p, d in list(group.default_packages.items()) + if d] + elif ptype == "mandatory": + return [p + for p, m in list(group.mandatory_packages.items()) + if m] + elif ptype == "optional" or ptype == "all": + return group.packages + else: + self.logger.warning("Unknown group package type '%s'" % ptype) + return [] + + def _filter_arch(self, packages): + """ filter packages in the given list that do not have an + architecture in the list of groups for this client """ + matching = [] + for pkg in packages: + if pkg.arch in self.groups: + matching.append(pkg) + else: + self.logger.debug("%s has non-matching architecture (%s)" % + (pkg, pkg.arch)) + if matching: + return matching + else: + # no packages match architecture; we'll assume that the + # user knows what s/he is doing and this is a multiarch + # box. + return packages + + def get_package_name(self, package): + """ get the name of a package or virtual package from the + internal representation used by this Collection class """ + if isinstance(package, tuple): + if len(package) == 3: + return yum.misc.prco_tuple_to_string(package) + else: + return pkgtup_to_string(package) + else: + return str(package) + + def complete(self, packagelist): + """ resolve dependencies and generate a complete package list + from the given list of initial packages """ + packages = set() + unknown = set() + for pkg in packagelist: + if isinstance(pkg, tuple): + pkgtup = pkg + else: + pkgtup = (pkg, None, None, None, None) + pkgobj = self.get_package_object(pkgtup) + if not pkgobj: + self.logger.debug("Unknown package %s" % + self.get_package_name(pkg)) + unknown.add(pkg) + else: + if self.yumbase.tsInfo.exists(pkgtup=pkgobj.pkgtup): + self.logger.debug("%s added to transaction multiple times" + % pkgobj) + else: + self.logger.debug("Adding %s to transaction" % pkgobj) + self.yumbase.tsInfo.addInstall(pkgobj) + self.yumbase.resolveDeps() + + for txmbr in self.yumbase.tsInfo: + packages.add(txmbr.pkgtup) + return list(packages), list(unknown) + + +def acquire_lock(func): + """ decorator for CacheManager methods that gets and release a + lock while the method runs """ + @wraps(func) + def inner(self, *args, **kwargs): + """ Get and release a lock while running the function this + wraps. """ + self.logger.debug("Acquiring lock at %s" % self.lockfile) + while not self.lock.i_am_locking(): + try: + self.lock.acquire(timeout=60) # wait up to 60 seconds + except LockTimeout: + self.lock.break_lock() + self.lock.acquire() + try: + func(self, *args, **kwargs) + finally: + self.lock.release() + self.logger.debug("Released lock at %s" % self.lockfile) + + return inner + + +class CacheManager(YumHelper): + """ Yum cache manager. Unlike :class:`DepSolver`, this can write + to the yum cache, and so is used for operations that muck with the + cache. (Technically, :func:`CacheManager.clean_cache` could be in + either DepSolver or CacheManager, but for consistency I've put it + here.) """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + self.lockfile = \ + os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), + "lock") + self.lock = FileLock(self.lockfile) + + @acquire_lock + def clean_cache(self): + """ clean the yum cache """ + for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", + "ExpireCache"]: + # for reasons that are entirely obvious, all of the yum + # API clean* methods return a tuple of 0 (zero, always + # zero) and a list containing a single message about how + # many files were deleted. so useful. thanks, yum. + msg = getattr(self.yumbase, "clean%s" % mdtype)()[1][0] + if not msg.startswith("0 "): + self.logger.info(msg) + + @acquire_lock + def populate_cache(self): + """ populate the yum cache """ + for repo in self.yumbase.repos.findRepos('*'): + repo.metadata_expire = 0 + repo.mdpolicy = "group:all" + self.yumbase.doRepoSetup() + self.yumbase.repos.doSetup() + for repo in self.yumbase.repos.listEnabled(): + # this populates the cache as a side effect + repo.repoXML # pylint: disable=W0104 + try: + repo.getGroups() + except yum.Errors.RepoMDError: + pass # this repo has no groups + self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) + # this does something with the groups cache as a side effect + self.yumbase.comps # pylint: disable=W0104 + + +class HelperSubcommand(Bcfg2.Options.Subcommand): + """ Base class for all yum helper subcommands """ + + # the value to JSON encode and print out if the command fails + fallback = None + + # whether or not this command accepts input on stdin + accept_input = True + + def __init__(self): + Bcfg2.Options.Subcommand.__init__(self) + self.verbosity = 0 + if Bcfg2.Options.setup.debug: + self.verbosity = 5 + elif Bcfg2.Options.setup.verbose: + self.verbosity = 1 + + def run(self, setup): + try: + data = json.loads(sys.stdin.read()) + except: # pylint: disable=W0702 + self.logger.error("Unexpected error decoding JSON input: %s" % + sys.exc_info()[1]) + print(json.dumps(self.fallback)) + return 2 + + try: + print(json.dumps(self._run(setup, data))) + except: # pylint: disable=W0702 + self.logger.error("Unexpected error running %s: %s" % + self.__class__.__name__.lower(), + sys.exc_info()[1], exc_info=1) + print(json.dumps(self.fallback)) + return 2 + return 0 + + def _run(self, setup, data): + """ Actually run the command """ + raise NotImplementedError + + +class DepSolverSubcommand(HelperSubcommand): # pylint: disable=W0223 + """ Base class for helper commands that use the depsolver (i.e., + only resolve dependencies, don't modify the cache) """ + + def __init__(self): + HelperSubcommand.__init__(self) + self.depsolver = DepSolver(Bcfg2.Options.setup.yum_config, + self.verbosity) + + +class CacheManagerSubcommand(HelperSubcommand): # pylint: disable=W0223 + """ Base class for helper commands that use the cachemanager + (i.e., modify the cache) """ + fallback = False + accept_input = False + + def __init__(self): + HelperSubcommand.__init__(self) + self.cachemgr = CacheManager(Bcfg2.Options.setup.yum_config, + self.verbosity) + + +class Clean(CacheManagerSubcommand): + """ Clean the cache """ + def _run(self, setup, data): # pylint: disable=W0613 + self.cachemgr.clean_cache() + return True + + +class MakeCache(CacheManagerSubcommand): + """ Update the on-disk cache """ + def _run(self, setup, data): # pylint: disable=W0613 + self.cachemgr.populate_cache() + return True + + +class Complete(DepSolverSubcommand): + """ Given an initial set of packages, get a complete set of + packages with all dependencies resolved """ + fallback = dict(packages=[], unknown=[]) + + def _run(self, _, data): + self.depsolver.groups = data['groups'] + self.fallback['unknown'] = data['packages'] + (packages, unknown) = self.depsolver.complete( + [pkg_to_tuple(p) for p in data['packages']]) + return dict(packages=list(packages), unknown=list(unknown)) + + +class GetGroups(DepSolverSubcommand): + """ Resolve the given package groups """ + def _run(self, _, data): + rv = dict() + for gdata in data: + if "type" in gdata: + packages = self.depsolver.get_group(gdata['group'], + ptype=gdata['type']) + else: + packages = self.depsolver.get_group(gdata['group']) + rv[gdata['group']] = list(packages) + return rv + + +Get_Groups = GetGroups # pylint: disable=C0103 + + +class CLI(Bcfg2.Options.CommandRegistry): + """ The bcfg2-yum-helper CLI """ + options = [ + Bcfg2.Options.PathOption( + "-c", "--yum-config", help="Yum config file"), + Bcfg2.Options.PositionalArgument( + "command", help="Yum helper command", + choices=['clean', 'complete', 'get_groups'])] + + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, globals().values(), + parent=HelperSubcommand) + parser = Bcfg2.Options.get_parser("Bcfg2 yum helper", + components=[self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + def run(self): + """ Run bcfg2-yum-helper """ + if not os.path.exists(Bcfg2.Options.setup.yum_config): + self.logger.error("Config file %s not found" % + Bcfg2.Options.setup.yum_config) + return 1 + return self.runcommand() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 4b58c0fdb..d11ac60fe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -7,20 +7,32 @@ import sys import glob import shutil import lxml.etree -import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ - MutableMapping +from Bcfg2.Compat import urlopen, HTTPError, URLError, MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources +from Bcfg2.Server.Statistics import track_statistics -#: The default path for generated yum configs -YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" -#: The default path for generated apt configs -APT_CONFIG_DEFAULT = \ - "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +def packages_boolean(value): + """ For historical reasons, the Packages booleans 'resolver' and + 'metadata' both accept "enabled" in addition to the normal boolean + values. """ + if value == 'disabled': + return False + elif value == 'enabled': + return True + else: + return value + + +class PackagesBackendAction(Bcfg2.Options.ComponentAction): + """ ComponentAction to load Packages backends """ + bases = ['Bcfg2.Server.Plugins.Packages'] + module = True class OnDemandDict(MutableMapping): @@ -70,7 +82,6 @@ class OnDemandDict(MutableMapping): class Packages(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, @@ -85,6 +96,39 @@ class Packages(Bcfg2.Server.Plugin.Plugin, .. private-include: _build_packages""" + options = [ + Bcfg2.Options.Option( + cf=("packages", "backends"), dest="packages_backends", + help="Packages backends to load", + type=Bcfg2.Options.Types.comma_list, + action=PackagesBackendAction, + default=['Yum', 'Apt', 'Pac', 'Pkgng']), + Bcfg2.Options.PathOption( + cf=("packages", "cache"), dest="packages_cache", + help="Path to the Packages cache", + default='<repository>/Packages/cache'), + Bcfg2.Options.Option( + cf=("packages", "resolver"), dest="packages_resolver", + help="Disable the Packages resolver", + type=packages_boolean, default=True), + Bcfg2.Options.Option( + cf=("packages", "metadata"), dest="packages_metadata", + help="Disable all Packages metadata processing", + type=packages_boolean, default=True), + Bcfg2.Options.Option( + cf=("packages", "version"), dest="packages_version", + help="Set default Package entry version", default="auto", + choices=["auto", "any"]), + Bcfg2.Options.PathOption( + cf=("packages", "yum_config"), + help="The default path for generated yum configs", + default="/etc/yum.repos.d/bcfg2.repo"), + Bcfg2.Options.PathOption( + cf=("packages", "apt_config"), + help="The default path for generated apt configs", + default= + "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list")] + #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. conflicts = ['Pkgmgr'] @@ -93,12 +137,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] - __child_rmi__ = Bcfg2.Server.Plugin.Plugin.__child_rmi__ + \ - [('Refresh', 'expire_cache'), ('Reload', 'expire_cache')] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) @@ -107,9 +147,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: Packages does a potentially tremendous amount of on-disk #: caching. ``cachepath`` holds the base directory to where #: data should be cached. - self.cachepath = \ - self.core.setup.cfp.get("packages", "cache", - default=os.path.join(self.data, 'cache')) + self.cachepath = Bcfg2.Options.setup.packages_cache #: Where Packages should store downloaded GPG key files self.keypath = os.path.join(self.cachepath, 'keys') @@ -117,14 +155,6 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # create key directory if needed os.makedirs(self.keypath) - # warn about deprecated magic groups - if self.core.setup.cfp.getboolean("packages", "magic_groups", - default=False): - self.logger.warning("Packages: Magic groups are deprecated and " - "will be removed in a future release") - self.logger.warning("You can disable magic groups by setting " - "magic_groups=0 in [packages] in bcfg2.conf") - # pylint: disable=C0301 #: The #: :class:`Bcfg2.Server.Plugins.Packages.PackagesSources.PackagesSources` @@ -132,8 +162,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` objects for #: this plugin. self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), - self.cachepath, core.fam, self, - self.core.setup) + self.cachepath, self) #: We cache #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` @@ -153,7 +182,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`, #: a unique key identifying the collection by its *config*, #: which could be shared among multiple clients. - self.collections = dict() + self.collections = Bcfg2.Server.Cache.Cache("Packages", "collections") #: clients is a cache mapping of hostname -> #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` @@ -161,21 +190,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. - self.clients = dict() - - #: groupcache caches group lookups. It maps Collections (via - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) - #: to sets of package groups, and thence to the packages - #: indicated by those groups. - self.groupcache = dict() - - #: pkgcache caches complete package sets. It maps Collections - #: (via - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) - #: to sets of initial packages, and thence to the final - #: (complete) package selections resolved from the initial - #: packages - self.pkgcache = dict() + self.clients = Bcfg2.Server.Cache.Cache("Packages", "cache") + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ @@ -187,48 +203,6 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return rv set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ - @property - def disableResolver(self): # pylint: disable=C0103 - """ Report the state of the resolver. This can be disabled in - the configuration. Note that disabling metadata (see - :attr:`disableMetaData`) implies disabling the resolver. - - This property cannot be set. """ - if self.disableMetaData: - # disabling metadata without disabling the resolver Breaks - # Things - return True - try: - return not self.core.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled", which are not handled according to the - # Python docs but appear to be handled properly by - # ConfigParser in at least some versions - return self.core.setup.cfp.get( - "packages", - "resolver", - default="enabled").lower() == "disabled" - - @property - def disableMetaData(self): # pylint: disable=C0103 - """ Report whether or not metadata processing is enabled. - - This property cannot be set. """ - try: - return not self.core.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled" - return self.core.setup.cfp.get( - "packages", - "metadata", - default="enabled").lower() == "disabled" - def create_config(self, entry, metadata): """ Create yum/apt config for the specified client. @@ -276,9 +250,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ if entry.tag == 'Package': collection = self.get_collection(metadata) - entry.set('version', self.core.setup.cfp.get("packages", - "version", - default="auto")) + entry.set('version', Bcfg2.Options.setup.packages_version) entry.set('type', collection.ptype) elif entry.tag == 'Path': self.create_config(entry, metadata) @@ -304,27 +276,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` """ if entry.tag == 'Package': - if self.core.setup.cfp.getboolean("packages", "magic_groups", - default=False): - collection = self.get_collection(metadata) - if collection.magic_groups_match(): - return True - else: - return True + return True elif entry.tag == 'Path': # managed entries for yum/apt configs - if (entry.get("name") == - self.core.setup.cfp.get("packages", - "yum_config", - default=YUM_CONFIG_DEFAULT) or - entry.get("name") == - self.core.setup.cfp.get("packages", - "apt_config", - default=APT_CONFIG_DEFAULT)): + if entry.get("name") in [Bcfg2.Options.setup.apt_config, + Bcfg2.Options.setup.yum_config]: return True return False - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def validate_structures(self, metadata, structures): """ Do the real work of Packages. This does two things: @@ -353,15 +313,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :returns: None """ collection = self.get_collection(metadata) - indep = lxml.etree.Element('Independent') + indep = lxml.etree.Element('Independent', name=self.__class__.__name__) self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) structures.append(indep) - @Bcfg2.Server.Plugin.track_statistics() - def _build_packages(self, metadata, independent, structures, - collection=None): + @track_statistics() + def _build_packages(self, metadata, independent, # pylint: disable=R0914 + structures, collection=None): """ Perform dependency resolution and build the complete list of packages that need to be included in the specification by :func:`validate_structures`, based on the initial list of @@ -382,8 +342,10 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :func:`get_collection` :type collection: Bcfg2.Server.Plugins.Packages.Collection.Collection """ - if self.disableResolver: - # Config requests no resolver + if (not Bcfg2.Options.setup.packages_metadata or + not Bcfg2.Options.setup.packages_resolver): + # Config requests no resolver. Note that disabling + # metadata implies disabling the resolver. for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("group"): @@ -396,10 +358,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, initial = set() to_remove = [] groups = [] + recommended = dict() + for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.update(collection.packages_from_entry(pkg)) + + if pkg.get("recommended"): + recommended[pkg.get("name")] = pkg.get("recommended") elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -422,11 +389,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, groups.sort() # check for this set of groups in the group cache + gcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_groups", + collection.cachekey) gkey = hash(tuple(groups)) - if gkey not in self.groupcache[collection.cachekey]: - self.groupcache[collection.cachekey][gkey] = \ - collection.get_groups(groups) - for pkgs in self.groupcache[collection.cachekey][gkey].values(): + if gkey not in gcache: + gcache[gkey] = collection.get_groups(groups) + for pkgs in gcache[gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution @@ -434,10 +402,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # check for this set of packages in the package cache pkey = hash(tuple(base)) - if pkey not in self.pkgcache[collection.cachekey]: - self.pkgcache[collection.cachekey][pkey] = \ - collection.complete(base) - packages, unknown = self.pkgcache[collection.cachekey][pkey] + pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", + collection.cachekey) + if pkey not in pcache: + pcache[pkey] = collection.complete(base, recommended) + packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) @@ -447,7 +416,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, newpkgs.sort() collection.packages_to_entry(newpkgs, independent) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Refresh(self): """ Packages.Refresh() => True|False @@ -455,7 +424,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config(force_update=True) return True - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Reload(self): """ Packages.Refresh() => True|False @@ -463,7 +432,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config() return True - def expire_cache(self, _=None): + def child_reload(self, _=None): + """ Reload the Packages configuration on a child process. """ self.Reload() def _load_config(self, force_update=False): @@ -490,18 +460,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for collection in list(self.collections.values()): cachefiles.update(collection.cachefiles) - if not self.disableMetaData: + if Bcfg2.Options.setup.packages_metadata: collection.setup_data(force_update) # clear Collection and package caches - self.clients = dict() - self.collections = dict() - self.groupcache = dict() - self.pkgcache = dict() + Bcfg2.Server.Cache.expire("Packages") for source in self.sources.entries: cachefiles.add(source.cachefile) - if not self.disableMetaData: + if Bcfg2.Options.setup.packages_metadata: source.setup_data(force_update) for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): @@ -533,7 +500,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if localfile not in keyfiles: keyfiles.append(localfile) if ((force_update and key not in keys) or - not os.path.exists(localfile)): + not os.path.exists(localfile)): self.logger.info("Packages: Downloading and parsing %s" % key) try: @@ -556,7 +523,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if kfile not in keyfiles: os.unlink(kfile) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_collection(self, metadata): """ Get a :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` @@ -573,12 +540,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object - collection = Collection(metadata, [], self.cachepath, self.data, - self.core.fam) - ckey = collection.cachekey - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) - return collection + return Collection(metadata, [], self.cachepath, self.data) if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] @@ -610,13 +572,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, "for %s" % (cclass.__name__, metadata.hostname)) collection = cclass(metadata, relevant, self.cachepath, self.data, - self.core.fam, debug=self.debug_flag) + debug=self.debug_flag) ckey = collection.cachekey if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) return collection def get_additional_data(self, metadata): @@ -665,8 +625,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ - if metadata.hostname in self.clients: - del self.clients[metadata.hostname] + self.clients.expire(metadata.hostname) def end_statistics(self, metadata): """ Hook to clear the cache for this client in :attr:`clients` diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index a1dcb575f..c7d8986ed 100644 --- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -1,24 +1,18 @@ '''This module implements a package management scheme for all images''' -import os import re -import glob +import sys import logging import lxml.etree import Bcfg2.Server.Plugin -import Bcfg2.Server.Lint +from Bcfg2.Server.Plugin import PluginExecutionError -try: - set -except NameError: - # deprecated since python 2.6 - from sets import Set as set logger = logging.getLogger('Bcfg2.Plugins.Pkgmgr') class FuzzyDict(dict): - fuzzy = re.compile('(?P<name>.*):(?P<alist>\S+(,\S+)*)') + fuzzy = re.compile(r'(?P<name>.*):(?P<alist>\S+(,\S+)*)') def __getitem__(self, key): if isinstance(key, str): @@ -47,95 +41,217 @@ class FuzzyDict(dict): raise -class PNode(Bcfg2.Server.Plugin.INode): +class PNode(object): """PNode has a list of packages available at a particular group intersection. """ - splitters = {'rpm': re.compile('^(.*/)?(?P<name>[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ - '(?P<version>[\w\d\.]+-([\w\d\.]+))\.(?P<arch>\S+)\.rpm$'), - 'encap': re.compile('^(?P<name>[\w-]+)-(?P<version>[\w\d\.+-]+).encap.*$')} + splitters = dict( + rpm=re.compile( + r'^(.*/)?(?P<name>[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + + r'(?P<version>[\w\d\.]+-([\w\d\.]+))\.(?P<arch>\S+)\.rpm$'), + encap=re.compile( + r'^(?P<name>[\w-]+)-(?P<version>[\w\d\.+-]+).encap.*$')) + raw = dict( + Client="lambda m, e:'%(name)s' == m.hostname and predicate(m, e)", + Group="lambda m, e:'%(name)s' in m.groups and predicate(m, e)") + nraw = dict( + Client="lambda m, e:'%(name)s' != m.hostname and predicate(m, e)", + Group="lambda m, e:'%(name)s' not in m.groups and predicate(m, e)") + containers = ['Group', 'Client'] ignore = ['Package'] - def Match(self, metadata, data, entry=lxml.etree.Element("None")): - """Return a dictionary of package mappings.""" - if self.predicate(metadata, entry): - for key in self.contents: - try: - data[key].update(self.contents[key]) - except: - data[key] = FuzzyDict() - data[key].update(self.contents[key]) - for child in self.children: - child.Match(metadata, data) - def __init__(self, data, pdict, parent=None): # copy local attributes to all child nodes if no local attribute exists if 'Package' not in pdict: pdict['Package'] = set() for child in data.getchildren(): - attrs = set(data.attrib.keys()).difference(child.attrib.keys() + ['name']) + attrs = set(data.attrib.keys()).difference( + child.attrib.keys() + ['name']) for attr in attrs: try: child.set(attr, data.get(attr)) except: - # don't fail on things like comments and other immutable elements + # don't fail on things like comments and other + # immutable elements pass - Bcfg2.Server.Plugin.INode.__init__(self, data, pdict, parent) + self.data = data + self.contents = {} + if parent is None: + self.predicate = lambda m, e: True + else: + predicate = parent.predicate + if data.get('negate', 'false').lower() == 'true': + psrc = self.nraw + else: + psrc = self.raw + if data.tag in list(psrc.keys()): + self.predicate = eval(psrc[data.tag] % + {'name': data.get('name')}, + {'predicate': predicate}) + else: + raise PluginExecutionError("Unknown tag: %s" % data.tag) + self.children = [] + self._load_children(data, pdict) + if 'Package' not in self.contents: self.contents['Package'] = FuzzyDict() for pkg in data.findall('./Package'): - if 'name' in pkg.attrib and pkg.get('name') not in pdict['Package']: + if ('name' in pkg.attrib and + pkg.get('name') not in pdict['Package']): pdict['Package'].add(pkg.get('name')) - if pkg.get('name') != None: + if pkg.get('name') is not None: self.contents['Package'][pkg.get('name')] = {} if pkg.getchildren(): self.contents['Package'][pkg.get('name')]['__children__'] \ - = pkg.getchildren() + = pkg.getchildren() if 'simplefile' in pkg.attrib: - pkg.set('url', "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) + pkg.set('url', + "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) self.contents['Package'][pkg.get('name')].update(pkg.attrib) else: if 'file' in pkg.attrib: if 'multiarch' in pkg.attrib: archs = pkg.get('multiarch').split() srcs = pkg.get('srcs', pkg.get('multiarch')).split() - url = ' '.join(["%s/%s" % (pkg.get('uri'), - pkg.get('file') % {'src':srcs[idx], - 'arch':archs[idx]}) - for idx in range(len(archs))]) + url = ' '.join( + ["%s/%s" % (pkg.get('uri'), + pkg.get('file') % {'src': srcs[idx], + 'arch': archs[idx]}) + for idx in range(len(archs))]) pkg.set('url', url) else: pkg.set('url', '%s/%s' % (pkg.get('uri'), pkg.get('file'))) - if pkg.get('type') in self.splitters and pkg.get('file') != None: - mdata = self.splitters[pkg.get('type')].match(pkg.get('file')) + if (pkg.get('type') in self.splitters and + pkg.get('file') is not None): + mdata = \ + self.splitters[pkg.get('type')].match(pkg.get('file')) if not mdata: - logger.error("Failed to match pkg %s" % pkg.get('file')) + logger.error("Failed to match pkg %s" % + pkg.get('file')) continue pkgname = mdata.group('name') self.contents['Package'][pkgname] = mdata.groupdict() self.contents['Package'][pkgname].update(pkg.attrib) if pkg.attrib.get('file'): - self.contents['Package'][pkgname]['url'] = pkg.get('url') - self.contents['Package'][pkgname]['type'] = pkg.get('type') + self.contents['Package'][pkgname]['url'] = \ + pkg.get('url') + self.contents['Package'][pkgname]['type'] = \ + pkg.get('type') if pkg.get('verify'): - self.contents['Package'][pkgname]['verify'] = pkg.get('verify') + self.contents['Package'][pkgname]['verify'] = \ + pkg.get('verify') if pkg.get('multiarch'): - self.contents['Package'][pkgname]['multiarch'] = pkg.get('multiarch') + self.contents['Package'][pkgname]['multiarch'] = \ + pkg.get('multiarch') if pkgname not in pdict['Package']: pdict['Package'].add(pkgname) if pkg.getchildren(): - self.contents['Package'][pkgname]['__children__'] = pkg.getchildren() + self.contents['Package'][pkgname]['__children__'] = \ + pkg.getchildren() else: - self.contents['Package'][pkg.get('name')].update(pkg.attrib) + self.contents['Package'][pkg.get('name')].update( + pkg.attrib) + def _load_children(self, data, idict): + """ load children """ + for item in data.getchildren(): + if item.tag in self.ignore: + continue + elif item.tag in self.containers: + self.children.append(self.__class__(item, idict, self)) + else: + try: + self.contents[item.tag][item.get('name')] = \ + dict(item.attrib) + except KeyError: + self.contents[item.tag] = \ + {item.get('name'): dict(item.attrib)} + if item.text: + self.contents[item.tag][item.get('name')]['__text__'] = \ + item.text + if item.getchildren(): + self.contents[item.tag][item.get('name')]['__children__'] \ + = item.getchildren() + try: + idict[item.tag].append(item.get('name')) + except KeyError: + idict[item.tag] = [item.get('name')] -class PkgSrc(Bcfg2.Server.Plugin.XMLSrc): - """PkgSrc files contain a PNode hierarchy that - returns matching package entries. - """ + def Match(self, metadata, data, entry=lxml.etree.Element("None")): + """Return a dictionary of package mappings.""" + if self.predicate(metadata, entry): + for key in self.contents: + try: + data[key].update(self.contents[key]) + except: # pylint: disable=W0702 + data[key] = FuzzyDict() + data[key].update(self.contents[key]) + for child in self.children: + child.Match(metadata, data) + + +class PkgSrc(Bcfg2.Server.Plugin.XMLFileBacked): + """ XMLSrc files contain a + :class:`Bcfg2.Server.Plugin.helpers.INode` hierarchy that returns + matching entries. XMLSrc objects are deprecated and + :class:`Bcfg2.Server.Plugin.helpers.StructFile` should be + preferred where possible.""" __node__ = PNode __cacheobj__ = FuzzyDict + __priority_required__ = True + + def __init__(self, filename, should_monitor=False): + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, + should_monitor) + self.items = {} + self.cache = None + self.pnode = None + self.priority = -1 + + def HandleEvent(self, _=None): + """Read file upon update.""" + try: + data = open(self.name).read() + except IOError: + msg = "Failed to read file %s: %s" % (self.name, sys.exc_info()[1]) + logger.error(msg) + raise PluginExecutionError(msg) + self.items = {} + try: + xdata = lxml.etree.XML(data, parser=Bcfg2.Server.XMLParser) + except lxml.etree.XMLSyntaxError: + msg = "Failed to parse file %s: %s" % (self.name, + sys.exc_info()[1]) + logger.error(msg) + raise PluginExecutionError(msg) + self.pnode = self.__node__(xdata, self.items) + self.cache = None + try: + self.priority = int(xdata.get('priority')) + except (ValueError, TypeError): + if self.__priority_required__: + msg = "Got bogus priority %s for file %s" % \ + (xdata.get('priority'), self.name) + logger.error(msg) + raise PluginExecutionError(msg) + + del xdata, data + + def Cache(self, metadata): + """Build a package dict for a given host.""" + if self.cache is None or self.cache[0] != metadata: + cache = (metadata, self.__cacheobj__()) + if self.pnode is None: + logger.error("Cache method called early for %s; " + "forcing data load" % self.name) + self.HandleEvent() + return + self.pnode.Match(metadata, cache[1]) + self.cache = cache + + def __str__(self): + return str(self.items) class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): @@ -165,53 +281,14 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): mdata = FuzzyDict.fuzzy.match(pname) if mdata: arches = mdata.group('alist').split(',') - [entry.remove(inst) for inst in \ - entry.findall('Instance') \ - if inst.get('arch') not in arches] + for inst in entry.findall('Instance'): + if inst.get('arch') not in arches: + entry.remove(inst) def HandlesEntry(self, entry, metadata): - return entry.tag == 'Package' and entry.get('name').split(':')[0] in list(self.Entries['Package'].keys()) + return ( + entry.tag == 'Package' and + entry.get('name').split(':')[0] in self.Entries['Package'].keys()) def HandleEntry(self, entry, metadata): self.BindEntry(entry, metadata) - - -class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): - """ Find duplicate :ref:`Pkgmgr - <server-plugins-generators-pkgmgr>` entries with the same - priority. """ - - def Run(self): - pset = set() - for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr', - '*.xml')): - if self.HandlesFile(pfile): - xdata = lxml.etree.parse(pfile).getroot() - # get priority, type, group - priority = xdata.get('priority') - ptype = xdata.get('type') - for pkg in xdata.xpath("//Package"): - if pkg.getparent().tag == 'Group': - grp = pkg.getparent().get('name') - if (type(grp) is not str and - grp.getparent().tag == 'Group'): - pgrp = grp.getparent().get('name') - else: - pgrp = 'none' - else: - grp = 'none' - pgrp = 'none' - ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) - # check if package is already listed with same - # priority, type, grp - if ptuple in pset: - self.LintError( - "duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) - else: - pset.add(ptuple) - - @classmethod - def Errors(cls): - return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 5d846b4bb..21d50ace6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -8,15 +8,33 @@ import copy import operator import lxml.etree import Bcfg2.Server +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin -from Bcfg2.Compat import any, unicode # pylint: disable=W0622 - -try: - from django.db import models - from django.core.exceptions import MultipleObjectsReturned - HAS_DJANGO = True +from Bcfg2.Compat import unicode, any # pylint: disable=W0622 +import Bcfg2.Server.FileMonitor +from Bcfg2.Logger import Debuggable +from Bcfg2.Server.Statistics import track_statistics + +HAS_DJANGO = False +# pylint: disable=C0103 +ProbesDataModel = None +ProbesGroupsModel = None +# pylint: enable=C0103 + + +def load_django_models(): + """ Load models for Django after option parsing has completed """ + # pylint: disable=W0602 + global ProbesDataModel, ProbesGroupsModel, HAS_DJANGO + # pylint: enable=W0602 + try: + from django.db import models + HAS_DJANGO = True + except ImportError: + HAS_DJANGO = False + return - class ProbesDataModel(models.Model, + class ProbesDataModel(models.Model, # pylint: disable=W0621,W0612 Bcfg2.Server.Plugin.PluginDatabaseModel): """ The database model for storing probe data """ hostname = models.CharField(max_length=255) @@ -24,13 +42,12 @@ try: timestamp = models.DateTimeField(auto_now=True) data = models.TextField(null=True) - class ProbesGroupsModel(models.Model, + class ProbesGroupsModel(models.Model, # pylint: disable=W0621,W0612 Bcfg2.Server.Plugin.PluginDatabaseModel): """ The database model for storing probe groups """ hostname = models.CharField(max_length=255) group = models.CharField(max_length=255) -except ImportError: - HAS_DJANGO = False + try: import json @@ -51,6 +68,226 @@ except ImportError: HAS_YAML = False +class ProbeStore(Debuggable): + """ Caching abstraction layer between persistent probe data + storage and the Probes plugin.""" + + def __init__(self, core, datadir): # pylint: disable=W0613 + Debuggable.__init__(self) + self._groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + self._datacache = Bcfg2.Server.Cache.Cache("Probes", "probedata") + + def get_groups(self, hostname): + """ Get the list of groups for the given host """ + if hostname not in self._groupcache: + self._load_groups(hostname) + return self._groupcache.get(hostname, []) + + def set_groups(self, hostname, groups): + """ Set the list of groups for the given host """ + raise NotImplementedError + + def get_data(self, hostname): + """ Get a dict of probe data for the given host """ + if hostname not in self._datacache: + self._load_data(hostname) + return self._datacache.get(hostname, dict()) + + def set_data(self, hostname, data): + """ Set probe data for the given host """ + raise NotImplementedError + + def _load_groups(self, hostname): + """ When probe groups are not found in the cache, this + function is called to load them from the backend (XML or + database). """ + raise NotImplementedError + + def _load_data(self, hostname): + """ When probe groups are not found in the cache, this + function is called to load them from the backend (XML or + database). """ + raise NotImplementedError + + def commit(self): + """ Commit the current data in the cache to the persistent + backend store. This is not used with the + :class:`Bcfg2.Server.Plugins.Probes.DBProbeStore`, because it + commits on every change. """ + pass + + +class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): + """ Caching abstraction layer between the database and the Probes + plugin. """ + create = False + + def __init__(self, core, datadir): + Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core) + ProbeStore.__init__(self, core, datadir) + + @property + def _use_db(self): + return True + + def _load_groups(self, hostname): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + groupdata = ProbesGroupsModel.objects.filter(hostname=hostname) + self._groupcache[hostname] = list(set(r.group for r in groupdata)) + Bcfg2.Server.Cache.expire("Metadata", hostname) + + @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock + def set_groups(self, hostname, groups): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + olddata = self._groupcache.get(hostname, []) + self._groupcache[hostname] = groups + for group in groups: + try: + ProbesGroupsModel.objects.get_or_create( + hostname=hostname, + group=group) + except ProbesGroupsModel.MultipleObjectsReturned: + ProbesGroupsModel.objects.filter(hostname=hostname, + group=group).delete() + ProbesGroupsModel.objects.get_or_create( + hostname=hostname, + group=group) + ProbesGroupsModel.objects.filter( + hostname=hostname).exclude(group__in=groups).delete() + if olddata != groups: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + def _load_data(self, hostname): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + ts_set = False + for pdata in ProbesDataModel.objects.filter(hostname=hostname): + if not ts_set: + self._datacache[hostname].timestamp = \ + time.mktime(pdata.timestamp.timetuple()) + ts_set = True + self._datacache[hostname][pdata.probe] = ProbeData(pdata.data) + Bcfg2.Server.Cache.expire("Metadata", hostname) + + @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock + def set_data(self, hostname, data): + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + expire_metadata = False + for probe, pdata in data.items(): + self._datacache[hostname][probe] = pdata + try: + record, created = ProbesDataModel.objects.get_or_create( + hostname=hostname, + probe=probe) + except ProbesDataModel.MultipleObjectsReturned: + ProbesDataModel.objects.filter(hostname=hostname, + probe=probe).delete() + record, created = ProbesDataModel.objects.get_or_create( + hostname=hostname, + probe=probe) + expire_metadata |= created + if record.data != pdata: + record.data = pdata + record.save() + expire_metadata = True + qset = ProbesDataModel.objects.filter( + hostname=hostname).exclude(probe__in=data.keys()) + if len(qset): + qset.delete() + expire_metadata = True + if expire_metadata: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + +class XMLProbeStore(ProbeStore): + """ Caching abstraction layer between ``probed.xml`` and the + Probes plugin.""" + def __init__(self, core, datadir): + ProbeStore.__init__(self, core, datadir) + self._fname = os.path.join(datadir, 'probed.xml') + self._load_data() + + def _load_data(self, _=None): + """ Load probe data from probed.xml """ + Bcfg2.Server.Cache.expire("Probes", "probegroups") + Bcfg2.Server.Cache.expire("Probes", "probedata") + if not os.path.exists(self._fname): + self.commit() + try: + data = lxml.etree.parse(self._fname, + parser=Bcfg2.Server.XMLParser).getroot() + except (IOError, lxml.etree.XMLSyntaxError): + err = sys.exc_info()[1] + self.logger.error("Failed to read file probed.xml: %s" % err) + return + for client in data.getchildren(): + self._datacache[client.get('name')] = \ + ClientProbeDataSet(timestamp=client.get("timestamp")) + self._groupcache[client.get('name')] = [] + for pdata in client: + if pdata.tag == 'Probe': + self._datacache[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get("value")) + elif pdata.tag == 'Group': + self._groupcache[client.get('name')].append( + pdata.get('name')) + + Bcfg2.Server.Cache.expire("Metadata") + + def _load_groups(self, hostname): + self._load_data(hostname) + + def commit(self): + """ Write received probe data to probed.xml """ + top = lxml.etree.Element("Probed") + for client, probed in sorted(self._datacache.items()): + # make a copy of probe data for this client in case it + # submits probe data while we're trying to write + # probed.xml + probedata = copy.copy(probed) + ctag = \ + lxml.etree.SubElement(top, 'Client', name=client, + timestamp=str(int(probedata.timestamp))) + for probe in sorted(probedata): + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self._datacache[client][probe].decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self._datacache[client][probe]) + for group in sorted(self._groupcache[client]): + lxml.etree.SubElement(ctag, "Group", name=group) + try: + top.getroottree().write(self._fname, + xml_declaration=False, + pretty_print='true') + except IOError: + err = sys.exc_info()[1] + self.logger.error("Failed to write %s: %s" % (self._fname, err)) + + def set_groups(self, hostname, groups): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + olddata = self._groupcache.get(hostname, []) + self._groupcache[hostname] = groups + if olddata != groups: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + def set_data(self, hostname, data): + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + expire_metadata = False + for probe, pdata in data.items(): + olddata = self._datacache[hostname].get(probe, ProbeData('')) + self._datacache[hostname][probe] = pdata + expire_metadata |= olddata != data + if expire_metadata: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + class ClientProbeDataSet(dict): """ dict of probe => [probe data] that records a timestamp for each host """ @@ -124,17 +361,16 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): bangline = re.compile(r'^#!\s*(?P<interpreter>.*)$') basename_is_regex = True - def __init__(self, path, fam, encoding, plugin_name): + def __init__(self, path, plugin_name): self.plugin_name = plugin_name Bcfg2.Server.Plugin.EntrySet.__init__(self, r'[0-9A-Za-z_\-]+', path, - Bcfg2.Server.Plugin.SpecificData, - encoding) - fam.AddMonitor(path, self) + Bcfg2.Server.Plugin.SpecificData) + Bcfg2.Server.FileMonitor.get_fam().AddMonitor(path, self) def HandleEvent(self, event): """ handle events on everything but probed.xml """ if (event.filename != self.path and - not event.filename.endswith("probed.xml")): + not event.filename.endswith("probed.xml")): return self.handle_event(event) def get_probe_data(self, metadata): @@ -161,7 +397,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): probe.set('name', os.path.basename(name)) probe.set('source', self.plugin_name) if (metadata.version_info and - metadata.version_info > (1, 3, 1, '', 0)): + metadata.version_info > (1, 3, 1, '', 0)): try: probe.text = entry.data.decode('utf-8') except AttributeError: @@ -187,235 +423,90 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, core, datastore): + groupline_re = re.compile(r'^group:\s*(?P<groupname>\S+)\s*') + + options = [ + Bcfg2.Options.BooleanOption( + cf=('probes', 'use_database'), dest="probes_db", + help="Use database capabilities of the Probes plugin"), + Bcfg2.Options.Option( + cf=('probes', 'allowed_groups'), dest="probes_allowed_groups", + help="Whitespace-separated list of group name regexps to which " + "probes can assign a client", + default=[re.compile('.*')], + type=Bcfg2.Options.Types.anchored_regex_list)] + options_parsed_hook = staticmethod(load_django_models) + + def __init__(self, core): Bcfg2.Server.Plugin.Probing.__init__(self) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) + Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core) try: - self.probes = ProbeSet(self.data, core.fam, core.setup['encoding'], - self.name) + self.probes = ProbeSet(self.data, self.name) except: err = sys.exc_info()[1] raise Bcfg2.Server.Plugin.PluginInitError(err) - self.allowed_cgroups = core.setup['probe_allowed_groups'] - self.probedata = dict() - self.cgroups = dict() - self.load_data() - __init__.__doc__ = Bcfg2.Server.Plugin.DatabaseBacked.__init__.__doc__ - - @Bcfg2.Server.Plugin.track_statistics() - def write_data(self, client): - """ Write probe data out for use with bcfg2-info """ if self._use_db: - return self._write_data_db(client) + self.probestore = DBProbeStore(core, self.data) else: - return self._write_data_xml(client) - - def _write_data_xml(self, _): - """ Write received probe data to probed.xml """ - top = lxml.etree.Element("Probed") - for client, probed in sorted(self.probedata.items()): - # make a copy of probe data for this client in case it - # submits probe data while we're trying to write - # probed.xml - probedata = copy.copy(probed) - ctag = \ - lxml.etree.SubElement(top, 'Client', name=client, - timestamp=str(int(probedata.timestamp))) - for probe in sorted(probedata): - try: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str( - self.probedata[client][probe]).decode('utf-8')) - except AttributeError: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) - for group in sorted(self.cgroups[client]): - lxml.etree.SubElement(ctag, "Group", name=group) - try: - top.getroottree().write(os.path.join(self.data, 'probed.xml'), - xml_declaration=False, - pretty_print='true') - except IOError: - err = sys.exc_info()[1] - self.logger.error("Failed to write probed.xml: %s" % err) + self.probestore = XMLProbeStore(core, self.data) - @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock - def _write_data_db(self, client): - """ Write received probe data to the database """ - for probe, data in self.probedata[client.hostname].items(): - try: - pdata = ProbesDataModel.objects.get_or_create( - hostname=client.hostname, - probe=probe)[0] - except MultipleObjectsReturned: - ProbesDataModel.objects.filter(hostname=client.hostname, - probe=probe).delete() - ProbesDataModel.objects.get_or_create( - hostname=client.hostname, - probe=probe) - if pdata.data != data: - pdata.data = data - pdata.save() + @track_statistics() + def GetProbes(self, metadata): + return self.probes.get_probe_data(metadata) - ProbesDataModel.objects.filter( - hostname=client.hostname).exclude( - probe__in=self.probedata[client.hostname]).delete() - - for group in self.cgroups[client.hostname]: - try: - ProbesGroupsModel.objects.get_or_create( - hostname=client.hostname, - group=group) - except MultipleObjectsReturned: - ProbesGroupsModel.objects.filter(hostname=client.hostname, - group=group).delete() - ProbesGroupsModel.objects.get_or_create( - hostname=client.hostname, - group=group) - ProbesGroupsModel.objects.filter( - hostname=client.hostname).exclude( - group__in=self.cgroups[client.hostname]).delete() - - def expire_cache(self, key=None): - self.load_data(client=key) - - def load_data(self, client=None): - """ Load probe data from the appropriate backend (probed.xml - or the database) """ - if self._use_db: - return self._load_data_db(client=client) - else: - # the XML backend doesn't support loading data for single - # clients, so it reloads all data - return self._load_data_xml() - - def _load_data_xml(self): - """ Load probe data from probed.xml """ - try: - data = lxml.etree.parse(os.path.join(self.data, 'probed.xml'), - parser=Bcfg2.Server.XMLParser).getroot() - except (IOError, lxml.etree.XMLSyntaxError): - err = sys.exc_info()[1] - self.logger.error("Failed to read file probed.xml: %s" % err) - return - self.probedata = {} - self.cgroups = {} - for client in data.getchildren(): - self.probedata[client.get('name')] = \ - ClientProbeDataSet(timestamp=client.get("timestamp")) - self.cgroups[client.get('name')] = [] - for pdata in client: - if pdata.tag == 'Probe': - self.probedata[client.get('name')][pdata.get('name')] = \ - ProbeData(pdata.get("value")) - elif pdata.tag == 'Group': - self.cgroups[client.get('name')].append(pdata.get('name')) - - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) - - def _load_data_db(self, client=None): - """ Load probe data from the database """ - if client is None: - self.probedata = {} - self.cgroups = {} - probedata = ProbesDataModel.objects.all() - groupdata = ProbesGroupsModel.objects.all() - else: - self.probedata.pop(client, None) - self.cgroups.pop(client, None) - probedata = ProbesDataModel.objects.filter(hostname=client) - groupdata = ProbesGroupsModel.objects.filter(hostname=client) - - for pdata in probedata: - if pdata.hostname not in self.probedata: - self.probedata[pdata.hostname] = ClientProbeDataSet( - timestamp=time.mktime(pdata.timestamp.timetuple())) - self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) - for pgroup in groupdata: - if pgroup.hostname not in self.cgroups: - self.cgroups[pgroup.hostname] = [] - self.cgroups[pgroup.hostname].append(pgroup.group) - - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, - key=client) - - @Bcfg2.Server.Plugin.track_statistics() - def GetProbes(self, meta): - return self.probes.get_probe_data(meta) - GetProbes.__doc__ = Bcfg2.Server.Plugin.Probing.GetProbes.__doc__ - - @Bcfg2.Server.Plugin.track_statistics() def ReceiveData(self, client, datalist): - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - if client.hostname in self.cgroups: - olddata = copy.copy(self.cgroups[client.hostname]) - else: - olddata = [] - - cgroups = [] - cprobedata = ClientProbeDataSet() + cgroups = set() + cdata = dict() for data in datalist: - self.ReceiveDataItem(client, data, cgroups, cprobedata) - self.cgroups[client.hostname] = cgroups - self.probedata[client.hostname] = cprobedata - - if (self.core.metadata_cache_mode in ['cautious', 'aggressive'] and - olddata != self.cgroups[client.hostname]): - self.core.metadata_cache.expire(client.hostname) - self.write_data(client) - ReceiveData.__doc__ = Bcfg2.Server.Plugin.Probing.ReceiveData.__doc__ - - def ReceiveDataItem(self, client, data, cgroups, cprobedata): - """Receive probe results pertaining to client.""" + groups, cdata[data.get("name")] = \ + self.ReceiveDataItem(client, data) + cgroups.update(groups) + self.probestore.set_groups(client.hostname, list(cgroups)) + self.probestore.set_data(client.hostname, cdata) + self.probestore.commit() + + def ReceiveDataItem(self, client, data): + """ Receive probe results pertaining to client. Returns a + tuple of (<probe groups>, <probe data>). """ if data.text is None: self.logger.info("Got null response to probe %s from %s" % (data.get('name'), client.hostname)) - cprobedata[data.get('name')] = ProbeData('') - return + return [], '' dlines = data.text.split('\n') self.logger.debug("Processing probe from %s: %s:%s" % (client.hostname, data.get('name'), [line.strip() for line in dlines])) + groups = [] for line in dlines[:]: - if line.split(':')[0] == 'group': - newgroup = line.split(':')[1].strip() - if newgroup not in cgroups: - if self._group_allowed(newgroup): - cgroups.append(newgroup) - else: - self.logger.info( - "Disallowed group assignment %s from %s" % - (newgroup, client.hostname)) + match = self.groupline_re.match(line) + if match: + newgroup = match.group("groupname") + if self._group_allowed(newgroup): + groups.append(newgroup) + else: + self.logger.warning( + "Disallowed group assignment %s from %s" % + (newgroup, client.hostname)) dlines.remove(line) - dobj = ProbeData("\n".join(dlines)) - cprobedata[data.get('name')] = dobj + return (groups, ProbeData("\n".join(dlines))) + + def get_additional_groups(self, metadata): + return self.probestore.get_groups(metadata.hostname) + + def get_additional_data(self, metadata): + return self.probestore.get_data(metadata.hostname) def _group_allowed(self, group): """ Determine if the named group can be set as a probe group by checking the regexes listed in the [probes] groups_allowed setting """ - return any(r.match(group) for r in self.allowed_cgroups) - - def get_additional_groups(self, meta): - return self.cgroups.get(meta.hostname, list()) - get_additional_groups.__doc__ = \ - Bcfg2.Server.Plugin.Connector.get_additional_groups.__doc__ - - def get_additional_data(self, meta): - return self.probedata.get(meta.hostname, ClientProbeDataSet()) - get_additional_data.__doc__ = \ - Bcfg2.Server.Plugin.Connector.get_additional_data.__doc__ + return any(r.match(group) + for r in Bcfg2.Options.setup.probes_allowed_groups) diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index bbca01ead..28400f6d2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -7,13 +7,9 @@ import sys import copy import logging import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError -try: - import Bcfg2.Encryption - HAS_CRYPTO = True -except ImportError: - HAS_CRYPTO = False try: import json @@ -35,8 +31,6 @@ except ImportError: LOGGER = logging.getLogger(__name__) -SETUP = None - class PropertyFile(object): """ Base Properties file handler """ @@ -53,12 +47,9 @@ class PropertyFile(object): """ Write the data in this data structure back to the property file. This public method performs checking to ensure that writing is possible and then calls :func:`_write`. """ - if not SETUP.cfp.getboolean("properties", "writes_enabled", - default=True): - msg = "Properties files write-back is disabled in the " + \ - "configuration" - LOGGER.error(msg) - raise PluginExecutionError(msg) + if not Bcfg2.Options.setup.writes_enabled: + raise PluginExecutionError("Properties files write-back is " + "disabled in the configuration") try: self.validate_data() except PluginExecutionError: @@ -90,8 +81,8 @@ class PropertyFile(object): class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): """ Handle JSON Properties files. """ - def __init__(self, name, fam=None): - Bcfg2.Server.Plugin.FileBacked.__init__(self, name, fam=fam) + def __init__(self, name): + Bcfg2.Server.Plugin.FileBacked.__init__(self, name) PropertyFile.__init__(self, name) self.json = None __init__.__doc__ = Bcfg2.Server.Plugin.FileBacked.__init__.__doc__ @@ -129,8 +120,8 @@ class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): class YAMLPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): """ Handle YAML Properties files. """ - def __init__(self, name, fam=None): - Bcfg2.Server.Plugin.FileBacked.__init__(self, name, fam=fam) + def __init__(self, name): + Bcfg2.Server.Plugin.FileBacked.__init__(self, name) PropertyFile.__init__(self, name) self.yaml = None __init__.__doc__ = Bcfg2.Server.Plugin.FileBacked.__init__.__doc__ @@ -168,8 +159,8 @@ class YAMLPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): """ Handle XML Properties files. """ - def __init__(self, name, fam=None, should_monitor=False): - Bcfg2.Server.Plugin.StructFile.__init__(self, name, fam=fam, + def __init__(self, name, should_monitor=False): + Bcfg2.Server.Plugin.StructFile.__init__(self, name, should_monitor=should_monitor) PropertyFile.__init__(self, name) @@ -202,45 +193,8 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): else: return True - def Index(self): - Bcfg2.Server.Plugin.StructFile.Index(self) - if HAS_CRYPTO: - for el in self.xdata.xpath("//*[@encrypted]"): - try: - el.text = self._decrypt(el).encode('ascii', - 'xmlcharrefreplace') - except UnicodeDecodeError: - self.logger.info("Properties: Decrypted %s to gibberish, " - "skipping" % el.tag) - except (TypeError, Bcfg2.Encryption.EVPError): - strict = self.xdata.get( - "decrypt", - SETUP.cfp.get(Bcfg2.Encryption.CFG_SECTION, "decrypt", - default="strict")) == "strict" - msg = "Properties: Failed to decrypt %s element in %s" % \ - (el.tag, self.name) - if strict: - raise PluginExecutionError(msg) - else: - self.logger.debug(msg) - - def _decrypt(self, element): - """ Decrypt a single encrypted properties file element """ - if not element.text or not element.text.strip(): - return - passes = Bcfg2.Encryption.get_passphrases(SETUP) - try: - passphrase = passes[element.get("encrypted")] - return Bcfg2.Encryption.ssl_decrypt( - element.text, passphrase, - algorithm=Bcfg2.Encryption.get_algorithm(SETUP)) - except KeyError: - raise Bcfg2.Encryption.EVPError("No passphrase named '%s'" % - element.get("encrypted")) - raise Bcfg2.Encryption.EVPError("Failed to decrypt") - def get_additional_data(self, metadata): - if SETUP.cfp.getboolean("properties", "automatch", default=False): + if Bcfg2.Options.setup.automatch: default_automatch = "true" else: default_automatch = "false" @@ -262,6 +216,13 @@ class Properties(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): """ The properties plugin maps property files into client metadata instances. """ + options = [ + Bcfg2.Options.BooleanOption( + cf=("properties", "writes_enabled"), default=True, + help="Enable or disable Properties write-back"), + Bcfg2.Options.BooleanOption( + cf=("properties", "automatch"), + help="Enable Properties automatch")] #: Extensions that are understood by Properties. extensions = ["xml"] @@ -280,12 +241,10 @@ class Properties(Bcfg2.Server.Plugin.Plugin, #: Ignore XML schema (``.xsd``) files ignore = re.compile(r'.*\.xsd$') - def __init__(self, core, datastore): - global SETUP # pylint: disable=W0603 - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, core.fam) - SETUP = core.setup + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data) #: Instead of creating children of this object with a static #: object, we use :func:`property_dispatcher` to create a @@ -293,23 +252,21 @@ class Properties(Bcfg2.Server.Plugin.Plugin, self.__child__ = self.property_dispatcher __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ - def property_dispatcher(self, fname, fam): + def property_dispatcher(self, fname): """ Dispatch an event on a Properties file to the appropriate object. :param fname: The name of the file that received the event :type fname: string - :param fam: The file monitor the event was received by - :type fam: Bcfg2.Server.FileMonitor.FileMonitor :returns: An object of the appropriate subclass of :class:`PropertyFile` """ if fname.endswith(".xml"): - return XMLPropertyFile(fname, fam) + return XMLPropertyFile(fname) elif HAS_JSON and fname.endswith(".json"): - return JSONPropertyFile(fname, fam) + return JSONPropertyFile(fname) elif HAS_YAML and (fname.endswith(".yaml") or fname.endswith(".yml")): - return YAMLPropertyFile(fname, fam) + return YAMLPropertyFile(fname) else: raise Bcfg2.Server.Plugin.PluginExecutionError( "Properties: Unknown extension %s" % fname) diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 072f3f7e7..59fbe6f03 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -4,7 +4,7 @@ import os import sys import Bcfg2.Server import Bcfg2.Server.Plugin -from subprocess import Popen, PIPE +from Bcfg2.Utils import Executor try: from syck import load as yaml_load, error as yaml_error @@ -28,16 +28,15 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): """ A plugin to run Puppet external node classifiers (http://docs.puppetlabs.com/guides/external_nodes.html) """ - experimental = True __child__ = PuppetENCFile - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) - Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, - self.core.fam) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data) self.cache = dict() + self.cmd = Executor() def _run_encs(self, metadata): """ Run all Puppet ENCs """ @@ -46,20 +45,17 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, epath = os.path.join(self.data, enc) self.debug_log("PuppetENC: Running ENC %s for %s" % (enc, metadata.hostname)) - proc = Popen([epath, metadata.hostname], stdin=PIPE, stdout=PIPE, - stderr=PIPE) - (out, err) = proc.communicate() - rv = proc.wait() - if rv != 0: - msg = "PuppetENC: Error running ENC %s for %s (%s): %s" % \ - (enc, metadata.hostname, rv, err) + result = self.cmd.run([epath, metadata.hostname]) + if not result.success: + msg = "PuppetENC: Error running ENC %s for %s: %s" % \ + (enc, metadata.hostname, result.error) self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - if err: - self.debug_log("ENC Error: %s" % err) + if result.stderr: + self.debug_log("ENC Error: %s" % result.stderr) try: - yaml = yaml_load(out) + yaml = yaml_load(result.stdout) self.debug_log("Loaded data from %s for %s: %s" % (enc, metadata.hostname, yaml)) except yaml_error: @@ -69,13 +65,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - groups = dict() - if "classes" in yaml: - # stock Puppet ENC output format - groups = yaml['classes'] - elif "groups" in yaml: - # more Bcfg2-ish output format - groups = yaml['groups'] + groups = yaml.get("classes", yaml.get("groups", dict())) if groups: if isinstance(groups, list): self.debug_log("ENC %s adding groups to %s: %s" % diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index fa11d9250..282de8247 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -5,11 +5,10 @@ import time import platform import traceback import lxml.etree -from Bcfg2.Reporting.Transport import load_transport_from_config, \ - TransportError -from Bcfg2.Options import REPORTING_COMMON_OPTIONS +import Bcfg2.Options +from Bcfg2.Reporting.Transport.base import TransportError from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \ - Debuggable, PluginInitError, PluginExecutionError + PluginInitError, PluginExecutionError # required for reporting try: @@ -33,25 +32,23 @@ def _rpc_call(method): # pylint: disable=W0223 -class Reporting(Statistics, Threaded, PullSource, Debuggable): +class Reporting(Statistics, Threaded, PullSource): """ Unified statistics and reporting plugin """ - __rmi__ = Debuggable.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry'] + __rmi__ = Statistics.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry'] + + options = [Bcfg2.Options.Common.reporting_transport] CLIENT_METADATA_FIELDS = ('profile', 'bundles', 'aliases', 'addresses', 'groups', 'categories', 'uuid', 'version') - def __init__(self, core, datastore): - Statistics.__init__(self, core, datastore) + def __init__(self, core): + Statistics.__init__(self, core) PullSource.__init__(self) Threaded.__init__(self) - Debuggable.__init__(self) self.whoami = platform.node() self.transport = None - core.setup.update(REPORTING_COMMON_OPTIONS) - core.setup.reparse() - if not HAS_SOUTH: msg = "Django south is required for Reporting" self.logger.error(msg) @@ -59,20 +56,19 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): # This must be loaded here for bcfg2-admin try: - self.transport = load_transport_from_config(self.core.setup) + self.transport = Bcfg2.Options.setup.reporting_transport() except TransportError: - msg = "%s: Failed to load transport: %s" % \ - (self.name, traceback.format_exc().splitlines()[-1]) - self.logger.error(msg) - raise PluginInitError(msg) + raise PluginInitError("%s: Failed to instantiate transport: %s" % + (self.name, sys.exc_info()[1])) if self.debug_flag: self.transport.set_debug(self.debug_flag) def start_threads(self): + """Nothing to do here""" pass def set_debug(self, debug): - rv = Debuggable.set_debug(self, debug) + rv = Statistics.set_debug(self, debug) if self.transport is not None: self.transport.set_debug(debug) return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index 21862c5db..a3f682ed6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -1,6 +1,7 @@ """This generator provides rule-based entry mappings.""" import re +import Bcfg2.Options import Bcfg2.Server.Plugin @@ -8,45 +9,44 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles service assignments.""" __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) + options = Bcfg2.Server.Plugin.PrioDir.options + [ + Bcfg2.Options.BooleanOption( + cf=("rules", "regex"), dest="rules_regex", + help="Allow regular expressions in Rules")] + + def __init__(self, core): + Bcfg2.Server.Plugin.PrioDir.__init__(self, core) self._regex_cache = dict() def HandlesEntry(self, entry, metadata): - if entry.tag in self.Entries: - return self._matches(entry, metadata, - self.Entries[entry.tag].keys()) + for src in self.entries.values(): + for candidate in src.XMLMatch(metadata).xpath("//%s" % entry.tag): + if self._matches(entry, metadata, candidate): + return True return False - def BindEntry(self, entry, metadata): - attrs = self.get_attrs(entry, metadata) - for key, val in list(attrs.items()): - if key not in entry.attrib: - entry.attrib[key] = val + HandleEntry = Bcfg2.Server.Plugin.PrioDir.BindEntry - HandleEntry = BindEntry - - def _matches(self, entry, metadata, rules): - if Bcfg2.Server.Plugin.PrioDir._matches(self, entry, metadata, rules): + def _matches(self, entry, metadata, candidate): + if Bcfg2.Server.Plugin.PrioDir._matches(self, entry, metadata, + candidate): return True elif (entry.tag == "Path" and - ((entry.get('name').endswith("/") and - entry.get('name').rstrip("/") in rules) or - (not entry.get('name').endswith("/") and - entry.get('name') + '/' in rules))): + entry.get('name').rstrip("/") == + candidate.get("name").rstrip("/")): # special case for Path tags: # http://trac.mcs.anl.gov/projects/bcfg2/ticket/967 return True elif self._regex_enabled: # attempt regular expression matching - for rule in rules: - if rule not in self._regex_cache: - self._regex_cache[rule] = re.compile("%s$" % rule) - if self._regex_cache[rule].match(entry.get('name')): - return True + rule = candidate.get("name") + if rule not in self._regex_cache: + self._regex_cache[rule] = re.compile("%s$" % rule) + if self._regex_cache[rule].match(entry.get('name')): + return True return False @property def _regex_enabled(self): """ Return True if rules regexes are enabled, False otherwise """ - return self.core.setup.cfp.getboolean("rules", "regex", default=False) + return Bcfg2.Options.setup.rules_regex diff --git a/src/lib/Bcfg2/Server/Plugins/SEModules.py b/src/lib/Bcfg2/Server/Plugins/SEModules.py index fa47f9496..248b662f9 100644 --- a/src/lib/Bcfg2/Server/Plugins/SEModules.py +++ b/src/lib/Bcfg2/Server/Plugins/SEModules.py @@ -43,9 +43,6 @@ class SEModules(Bcfg2.Server.Plugin.GroupSpool): #: SEModules manages ``SEModule`` entries entry_type = 'SEModule' - #: The SEModules plugin is experimental - experimental = True - def _get_module_filename(self, entry): """ GroupSpool stores entries as /foo.pp, but we want people to be able to specify module entries as name='foo' or diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 2deea5f07..89c7107aa 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -5,27 +5,25 @@ import os import sys import socket import shutil -import logging import tempfile -from itertools import chain -from subprocess import Popen, PIPE +import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Plugin +from itertools import chain +from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Compat import any, u_str, b64encode # pylint: disable=W0622 - -LOGGER = logging.getLogger(__name__) +try: + from Bcfg2.Server.Encryption import ssl_encrypt, bruteforce_decrypt, \ + EVPError + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False class KeyData(Bcfg2.Server.Plugin.SpecificData): """ class to handle key data for HostKeyEntrySet """ - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, - name, - specific, - encoding) - self.encoding = encoding - def __lt__(self, other): return self.name < other.name @@ -42,49 +40,62 @@ class KeyData(Bcfg2.Server.Plugin.SpecificData): entry.text = b64encode(self.data) else: try: - entry.text = u_str(self.data, self.encoding) + entry.text = u_str(self.data, Bcfg2.Options.setup.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) - LOGGER.error(msg) - LOGGER.error("Please verify you are using the proper encoding") + self.logger.error(msg) + self.logger.error("Please verify you are using the proper " + "encoding") raise Bcfg2.Server.Plugin.PluginExecutionError(msg) except ValueError: msg = "Error in specification for %s: %s" % (entry.get('name'), sys.exc_info()[1]) - LOGGER.error(msg) - LOGGER.error("You need to specify base64 encoding for %s" % - entry.get('name')) + self.logger.error(msg) + self.logger.error("You need to specify base64 encoding for %s" + % entry.get('name')) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) if entry.text in ['', None]: entry.set('empty', 'true') + def handle_event(self, event): + Bcfg2.Server.Plugin.SpecificData.handle_event(self, event) + if event.filename.endswith(".crypt"): + if self.data is None: + return + # todo: let the user specify a passphrase by name + try: + self.data = bruteforce_decrypt(self.data) + except EVPError: + raise PluginExecutionError("Failed to decrypt %s" % self.name) + class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet): """ EntrySet to handle all kinds of host keys """ def __init__(self, basename, path): - if basename.startswith("ssh_host_key"): - encoding = "base64" - else: - encoding = None - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData, - encoding) + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData) self.metadata = {'owner': 'root', 'group': 'root', 'type': 'file'} - if encoding is not None: - self.metadata['encoding'] = encoding + if basename.startswith("ssh_host_key"): + self.metadata['encoding'] = "base64" if basename.endswith('.pub'): self.metadata['mode'] = '0644' else: self.metadata['mode'] = '0600' + def specificity_from_filename(self, fname, specific=None): + if fname.endswith(".crypt"): + fname = fname[0:-6] + return Bcfg2.Server.Plugin.EntrySet.specificity_from_filename( + self, fname, specific=specific) + class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): """ EntrySet to handle the ssh_known_hosts file """ def __init__(self, path): Bcfg2.Server.Plugin.EntrySet.__init__(self, "ssh_known_hosts", path, - KeyData, None) + KeyData) self.metadata = {'owner': 'root', 'group': 'root', 'type': 'file', @@ -92,7 +103,6 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): class SSHbase(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.PullTarget): """ @@ -124,9 +134,13 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, "ssh_host_rsa_key.pub", "ssh_host_key.pub"] - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) + options = [ + Bcfg2.Options.Option( + cf=("sshbase", "passphrase"), dest="sshbase_passphrase", + help="Passphrase used to encrypt generated private SSH host keys")] + + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.PullTarget.__init__(self) self.ipcache = {} @@ -137,7 +151,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, # do so once self.badnames = dict() - core.fam.AddMonitor(self.data, self) + self.fam = Bcfg2.Server.FileMonitor.get_fam() + self.fam.AddMonitor(self.data, self) self.static = dict() self.entries = dict() @@ -150,9 +165,15 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.entries["/etc/ssh/" + keypattern] = \ HostKeyEntrySet(keypattern, self.data) self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk + self.cmd = Executor() - def expire_cache(self, key=None): - self.__skn = False + @property + def passphrase(self): + """ The passphrase used to encrypt private keys """ + if HAS_CRYPTO and Bcfg2.Options.setup.sshbase_passphrase: + return Bcfg2.Options.setup.passphrases[ + Bcfg2.Options.setup.sshbase_passphrase] + return None def get_skn(self): """Build memory cache of the ssh known hosts file.""" @@ -252,7 +273,11 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, return for entry in list(self.entries.values()): - if entry.specific.match(event.filename): + if event.filename.endswith(".crypt"): + fname = event.filename[0:-6] + else: + fname = event.filename + if entry.specific.match(fname): entry.handle_event(event) if any(event.filename.startswith(kp) for kp in self.keypatterns @@ -262,7 +287,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.skn = False return - if event.filename in ['info', 'info.xml', ':info']: + if event.filename == 'info.xml': for entry in list(self.entries.values()): entry.handle_event(event) return @@ -284,12 +309,13 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, (event.filename, action)) def get_ipcache_entry(self, client): - """Build a cache of dns results.""" + """ Build a cache of dns results. """ if client in self.ipcache: if self.ipcache[client]: return self.ipcache[client] else: - raise socket.gaierror + raise PluginExecutionError("No cached IP address for %s" % + client) else: # need to add entry try: @@ -298,14 +324,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: - ipaddr = Popen(["getent", "hosts", client], - stdout=PIPE).stdout.read().strip().split() - if ipaddr: - self.ipcache[client] = (ipaddr, client) - return (ipaddr, client) + result = self.cmd.run(["getent", "hosts", client]) + if result.success: + ipaddr = result.stdout.strip().split() + if ipaddr: + self.ipcache[client] = (ipaddr, client) + return (ipaddr, client) self.ipcache[client] = False - self.logger.error("Failed to find IP address for %s" % client) - raise socket.gaierror + msg = "Failed to find IP address for %s: %s" % (client, + result.error) + self.logger(msg) + raise PluginExecutionError(msg) def get_namecache_entry(self, cip): """Build a cache of name lookups from client IP addresses.""" @@ -375,13 +404,15 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, msg = "%s still not registered" % filename self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - self.core.fam.handle_events_in_interval(1) + self.fam.handle_events_in_interval(1) tries += 1 try: self.entries[entry.get('name')].bind_entry(entry, metadata) is_bound = True except Bcfg2.Server.Plugin.PluginExecutionError: - pass + print("Failed to bind %s: %s") % ( + lxml.etree.tostring(entry), + sys.exc_info()[1]) def GenerateHostKeyPair(self, client, filename): """Generate new host key pair for client.""" @@ -404,19 +435,34 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, cmd = ["ssh-keygen", "-q", "-f", temploc, "-N", "", "-t", keytype, "-C", "root@%s" % client] self.debug_log("SSHbase: Running: %s" % " ".join(cmd)) - proc = Popen(cmd, stdout=PIPE, stdin=PIPE) - err = proc.communicate()[1] - if proc.wait(): + result = self.cmd.run(cmd) + if not result.success: raise PluginExecutionError("SSHbase: Error running ssh-keygen: %s" - % err) + % result.error) + + if self.passphrase: + self.debug_log("SSHbase: Encrypting private key for %s" % fileloc) + try: + data = ssl_encrypt(open(temploc).read(), self.passphrase) + except IOError: + raise PluginExecutionError("Unable to read temporary SSH key: " + "%s" % sys.exc_info()[1]) + except EVPError: + raise PluginExecutionError("Unable to encrypt SSH key: %s" % + sys.exc_info()[1]) + try: + open("%s.crypt" % fileloc, "wb").write(data) + except IOError: + raise PluginExecutionError("Unable to write encrypted SSH " + "key: %s" % sys.exc_info()[1]) try: - shutil.copy(temploc, fileloc) + if not self.passphrase: + shutil.copy(temploc, fileloc) shutil.copy("%s.pub" % temploc, publoc) except IOError: - err = sys.exc_info()[1] - raise PluginExecutionError("Temporary SSH keys not found: %s" % - err) + raise PluginExecutionError("Unable to copy temporary SSH key: %s" % + sys.exc_info()[1]) try: os.unlink(temploc) diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py deleted file mode 100644 index f111ffc60..000000000 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ /dev/null @@ -1,371 +0,0 @@ -""" The SSLCA generator handles the creation and management of ssl -certificates and their keys. """ - -import os -import sys -import logging -import tempfile -import lxml.etree -from subprocess import Popen, PIPE, STDOUT -import Bcfg2.Options -import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser -from Bcfg2.Server.Plugin import PluginExecutionError - -LOGGER = logging.getLogger(__name__) - - -class SSLCAXMLSpec(Bcfg2.Server.Plugin.StructFile): - """ Base class to handle key.xml and cert.xml """ - attrs = dict() - tag = None - - def get_spec(self, metadata): - """ Get a specification for the type of object described by - this SSLCA XML file for the given client metadata object """ - entries = [e for e in self.Match(metadata) if e.tag == self.tag] - if len(entries) == 0: - raise PluginExecutionError("No matching %s entry found for %s " - "in %s" % (self.tag, - metadata.hostname, - self.name)) - elif len(entries) > 1: - LOGGER.warning("More than one matching %s entry found for %s in " - "%s; using first match" % (self.tag, - metadata.hostname, - self.name)) - rv = dict() - for attr, default in self.attrs.items(): - val = entries[0].get(attr.lower(), default) - if default in ['true', 'false']: - rv[attr] = val == 'true' - else: - rv[attr] = val - return rv - - -class SSLCAKeySpec(SSLCAXMLSpec): - """ Handle key.xml files """ - attrs = dict(bits='2048', type='rsa') - tag = 'Key' - - -class SSLCACertSpec(SSLCAXMLSpec): - """ Handle cert.xml files """ - attrs = dict(ca='default', - format='pem', - key=None, - days='365', - C=None, - L=None, - ST=None, - OU=None, - O=None, - emailAddress=None, - append_chain='false') - tag = 'Cert' - - def get_spec(self, metadata): - rv = SSLCAXMLSpec.get_spec(self, metadata) - rv['subjectaltname'] = [e.text for e in self.Match(metadata) - if e.tag == "subjectAltName"] - return rv - - -class SSLCADataFile(Bcfg2.Server.Plugin.SpecificData): - """ Handle key and cert files """ - def bind_entry(self, entry, _): - """ Bind the data in the file to the given abstract entry """ - entry.text = self.data - entry.set("type", "file") - return entry - - -class SSLCAEntrySet(Bcfg2.Server.Plugin.EntrySet): - """ Entry set to handle SSLCA entries and XML files """ - def __init__(self, _, path, entry_type, encoding, parent=None): - Bcfg2.Server.Plugin.EntrySet.__init__(self, os.path.basename(path), - path, entry_type, encoding) - self.parent = parent - self.key = None - self.cert = None - - def handle_event(self, event): - action = event.code2str() - fpath = os.path.join(self.path, event.filename) - - if event.filename == 'key.xml': - if action in ['exists', 'created', 'changed']: - self.key = SSLCAKeySpec(fpath) - self.key.HandleEvent(event) - elif event.filename == 'cert.xml': - if action in ['exists', 'created', 'changed']: - self.cert = SSLCACertSpec(fpath) - self.cert.HandleEvent(event) - else: - Bcfg2.Server.Plugin.EntrySet.handle_event(self, event) - - def build_key(self, entry, metadata): - """ - either grabs a prexisting key hostfile, or triggers the generation - of a new key if one doesn't exist. - """ - # TODO: verify key fits the specs - filename = "%s.H_%s" % (os.path.basename(entry.get('name')), - metadata.hostname) - self.logger.info("SSLCA: Generating new key %s" % filename) - key_spec = self.key.get_spec(metadata) - ktype = key_spec['type'] - bits = key_spec['bits'] - if ktype == 'rsa': - cmd = ["openssl", "genrsa", bits] - elif ktype == 'dsa': - cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] - self.debug_log("SSLCA: Generating new key: %s" % " ".join(cmd)) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - key, err = proc.communicate() - if proc.wait(): - raise PluginExecutionError("SSLCA: Failed to generate key %s for " - "%s: %s" % (entry.get("name"), - metadata.hostname, err)) - open(os.path.join(self.path, filename), 'w').write(key) - return key - - def build_cert(self, entry, metadata, keyfile): - """ generate a new cert """ - filename = "%s.H_%s" % (os.path.basename(entry.get('name')), - metadata.hostname) - self.logger.info("SSLCA: Generating new cert %s" % filename) - cert_spec = self.cert.get_spec(metadata) - ca = self.parent.get_ca(cert_spec['ca']) - req_config = None - req = None - try: - req_config = self.build_req_config(metadata) - req = self.build_request(keyfile, req_config, metadata) - days = cert_spec['days'] - cmd = ["openssl", "ca", "-config", ca['config'], "-in", req, - "-days", days, "-batch"] - passphrase = ca.get('passphrase') - if passphrase: - cmd.extend(["-passin", "pass:%s" % passphrase]) - - def _scrub_pass(arg): - """ helper to scrub the passphrase from the - argument list """ - if arg.startswith("pass:"): - return "pass:******" - else: - return arg - else: - _scrub_pass = lambda a: a - - self.debug_log("SSLCA: Generating new certificate: %s" % - " ".join(_scrub_pass(a) for a in cmd)) - proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - (cert, err) = proc.communicate() - if proc.wait(): - # pylint: disable=E1103 - raise PluginExecutionError("SSLCA: Failed to generate cert: %s" - % err.splitlines()[-1]) - # pylint: enable=E1103 - finally: - try: - if req_config and os.path.exists(req_config): - os.unlink(req_config) - if req and os.path.exists(req): - os.unlink(req) - except OSError: - self.logger.error("SSLCA: Failed to unlink temporary files: %s" - % sys.exc_info()[1]) - if cert_spec['append_chain'] and 'chaincert' in ca: - cert += open(ca['chaincert']).read() - - open(os.path.join(self.path, filename), 'w').write(cert) - return cert - - def build_req_config(self, metadata): - """ - generates a temporary openssl configuration file that is - used to generate the required certificate request - """ - # create temp request config file - fd, fname = tempfile.mkstemp() - cfp = ConfigParser.ConfigParser({}) - cfp.optionxform = str - defaults = { - 'req': { - 'default_md': 'sha1', - 'distinguished_name': 'req_distinguished_name', - 'req_extensions': 'v3_req', - 'x509_extensions': 'v3_req', - 'prompt': 'no' - }, - 'req_distinguished_name': {}, - 'v3_req': { - 'subjectAltName': '@alt_names' - }, - 'alt_names': {} - } - for section in list(defaults.keys()): - cfp.add_section(section) - for key in defaults[section]: - cfp.set(section, key, defaults[section][key]) - cert_spec = self.cert.get_spec(metadata) - altnamenum = 1 - altnames = cert_spec['subjectaltname'] - altnames.extend(list(metadata.aliases)) - altnames.append(metadata.hostname) - for altname in altnames: - cfp.set('alt_names', 'DNS.' + str(altnamenum), altname) - altnamenum += 1 - for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: - if cert_spec[item]: - cfp.set('req_distinguished_name', item, cert_spec[item]) - cfp.set('req_distinguished_name', 'CN', metadata.hostname) - self.debug_log("SSLCA: Writing temporary request config to %s" % fname) - try: - cfp.write(os.fdopen(fd, 'w')) - except IOError: - raise PluginExecutionError("SSLCA: Failed to write temporary CSR " - "config file: %s" % sys.exc_info()[1]) - return fname - - def build_request(self, keyfile, req_config, metadata): - """ - creates the certificate request - """ - fd, req = tempfile.mkstemp() - os.close(fd) - days = self.cert.get_spec(metadata)['days'] - cmd = ["openssl", "req", "-new", "-config", req_config, - "-days", days, "-key", keyfile, "-text", "-out", req] - self.debug_log("SSLCA: Generating new CSR: %s" % " ".join(cmd)) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - err = proc.communicate()[1] - if proc.wait(): - raise PluginExecutionError("SSLCA: Failed to generate CSR: %s" % - err) - return req - - def verify_cert(self, filename, keyfile, entry, metadata): - """ Perform certification verification against the CA and - against the key """ - ca = self.parent.get_ca(self.cert.get_spec(metadata)['ca']) - do_verify = ca.get('chaincert') - if do_verify: - return (self.verify_cert_against_ca(filename, entry, metadata) and - self.verify_cert_against_key(filename, keyfile)) - return True - - def verify_cert_against_ca(self, filename, entry, metadata): - """ - check that a certificate validates against the ca cert, - and that it has not expired. - """ - ca = self.parent.get_ca(self.cert.get_spec(metadata)['ca']) - chaincert = ca.get('chaincert') - cert = os.path.join(self.path, filename) - cmd = ["openssl", "verify"] - is_root = ca.get('root_ca', "false").lower() == 'true' - if is_root: - cmd.append("-CAfile") - else: - # verifying based on an intermediate cert - cmd.extend(["-purpose", "sslserver", "-untrusted"]) - cmd.extend([chaincert, cert]) - self.debug_log("SSLCA: Verifying %s against CA: %s" % - (entry.get("name"), " ".join(cmd))) - res = Popen(cmd, stdout=PIPE, stderr=STDOUT).stdout.read() - if res == cert + ": OK\n": - self.debug_log("SSLCA: %s verified successfully against CA" % - entry.get("name")) - return True - self.logger.warning("SSLCA: %s failed verification against CA: %s" % - (entry.get("name"), res)) - return False - - def verify_cert_against_key(self, filename, keyfile): - """ - check that a certificate validates against its private key. - """ - def _modulus(fname, ftype="x509"): - """ get the modulus from the given file """ - cmd = ["openssl", ftype, "-noout", "-modulus", "-in", fname] - self.debug_log("SSLCA: Getting modulus of %s for verification: %s" - % (fname, " ".join(cmd))) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - rv, err = proc.communicate() - if proc.wait(): - self.logger.warning("SSLCA: Failed to get modulus of %s: %s" % - (fname, err)) - return rv.strip() # pylint: disable=E1103 - - certfile = os.path.join(self.path, filename) - cert = _modulus(certfile) - key = _modulus(keyfile, ftype="rsa") - if cert == key: - self.debug_log("SSLCA: %s verified successfully against key %s" % - (filename, keyfile)) - return True - self.logger.warning("SSLCA: %s failed verification against key %s" % - (filename, keyfile)) - return False - - def bind_entry(self, entry, metadata): - if self.key: - self.bind_info_to_entry(entry, metadata) - try: - return self.best_matching(metadata).bind_entry(entry, metadata) - except PluginExecutionError: - entry.text = self.build_key(entry, metadata) - entry.set("type", "file") - return entry - elif self.cert: - key = self.cert.get_spec(metadata)['key'] - cleanup_keyfile = False - try: - keyfile = self.parent.entries[key].best_matching(metadata).name - except PluginExecutionError: - cleanup_keyfile = True - # create a temp file with the key in it - fd, keyfile = tempfile.mkstemp() - os.chmod(keyfile, 384) # 0600 - el = lxml.etree.Element('Path', name=key) - self.parent.core.Bind(el, metadata) - os.fdopen(fd, 'w').write(el.text) - - try: - self.bind_info_to_entry(entry, metadata) - try: - best = self.best_matching(metadata) - if self.verify_cert(best.name, keyfile, entry, metadata): - return best.bind_entry(entry, metadata) - except PluginExecutionError: - pass - # if we get here, it's because either a) there was no best - # matching entry; or b) the existing cert did not verify - entry.text = self.build_cert(entry, metadata, keyfile) - entry.set("type", "file") - return entry - finally: - if cleanup_keyfile: - try: - os.unlink(keyfile) - except OSError: - err = sys.exc_info()[1] - self.logger.error("SSLCA: Failed to unlink temporary " - "key %s: %s" % (keyfile, err)) - - -class SSLCA(Bcfg2.Server.Plugin.GroupSpool): - """ The SSLCA generator handles the creation and management of ssl - certificates and their keys. """ - __author__ = 'g.hagger@gmail.com' - # python 2.5 doesn't support mixing *magic and keyword arguments - es_cls = lambda self, *args: SSLCAEntrySet(*args, **dict(parent=self)) - es_child_cls = SSLCADataFile - - def get_ca(self, name): - """ get a dict describing a CA from the config file """ - return dict(self.core.setup.cfp.items("sslca_%s" % name)) diff --git a/src/lib/Bcfg2/Server/Plugins/Snapshots.py b/src/lib/Bcfg2/Server/Plugins/Snapshots.py deleted file mode 100644 index cc5946bb2..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Snapshots.py +++ /dev/null @@ -1,129 +0,0 @@ -import logging -import difflib -import Bcfg2.Server.Plugin -import Bcfg2.Server.Snapshots -import Bcfg2.Logger -from Bcfg2.Server.Snapshots.model import Snapshot -import sys -import time -import threading - -# Compatibility import -from Bcfg2.Compat import Queue, u_str, b64decode - -logger = logging.getLogger('Snapshots') - -ftypes = ['ConfigFile', 'SymLink', 'Directory'] -datafields = { - 'Package': ['version'], - 'Path': ['type'], - 'Service': ['status'], - 'ConfigFile': ['owner', 'group', 'mode'], - 'Directory': ['owner', 'group', 'mode'], - 'SymLink': ['to'], - } - - -def build_snap_ent(entry): - basefields = [] - if entry.tag in ['Package', 'Service']: - basefields += ['type'] - desired = dict([(key, u_str(entry.get(key))) for key in basefields]) - state = dict([(key, u_str(entry.get(key))) for key in basefields]) - desired.update([(key, u_str(entry.get(key))) for key in \ - datafields[entry.tag]]) - if entry.tag == 'ConfigFile' or \ - ((entry.tag == 'Path') and (entry.get('type') == 'file')): - if entry.text == None: - desired['contents'] = None - else: - if entry.get('encoding', 'ascii') == 'ascii': - desired['contents'] = u_str(entry.text) - else: - desired['contents'] = u_str(b64decode(entry.text)) - - if 'current_bfile' in entry.attrib: - state['contents'] = u_str(b64decode(entry.get('current_bfile'))) - elif 'current_bdiff' in entry.attrib: - diff = b64decode(entry.get('current_bdiff')) - state['contents'] = u_str( \ - '\n'.join(difflib.restore(diff.split('\n'), 1))) - - state.update([(key, u_str(entry.get('current_' + key, entry.get(key)))) \ - for key in datafields[entry.tag]]) - if entry.tag in ['ConfigFile', 'Path'] and entry.get('exists', 'true') == 'false': - state = None - return [desired, state] - - -class Snapshots(Bcfg2.Server.Plugin.Statistics): - name = 'Snapshots' - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Statistics.__init__(self, core, datastore) - self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile) - self.work_queue = Queue() - self.loader = threading.Thread(target=self.load_snapshot) - - def start_threads(self): - self.loader.start() - - def load_snapshot(self): - while self.running: - try: - (metadata, data) = self.work_queue.get(block=True, timeout=5) - except: - continue - self.statistics_from_old_stats(metadata, data) - - def process_statistics(self, metadata, data): - return self.work_queue.put((metadata, data)) - - def statistics_from_old_stats(self, metadata, xdata): - # entries are name -> (modified, correct, start, desired, end) - # not sure we can get all of this from old format stats - t1 = time.time() - entries = dict([('Package', dict()), - ('Service', dict()), ('Path', dict())]) - extra = dict([('Package', dict()), ('Service', dict()), - ('Path', dict())]) - bad = [] - state = xdata.find('.//Statistics') - correct = state.get('state') == 'clean' - revision = u_str(state.get('revision', '-1')) - for entry in state.find('.//Bad'): - data = [False, False, u_str(entry.get('name'))] \ - + build_snap_ent(entry) - if entry.tag in ftypes: - etag = 'Path' - else: - etag = entry.tag - entries[etag][entry.get('name')] = data - for entry in state.find('.//Modified'): - if entry.tag in ftypes: - etag = 'Path' - else: - etag = entry.tag - if entry.get('name') in entries[etag]: - data = [True, False, u_str(entry.get('name'))] + \ - build_snap_ent(entry) - else: - data = [True, False, u_str(entry.get('name'))] + \ - build_snap_ent(entry) - for entry in state.find('.//Extra'): - if entry.tag in datafields: - data = build_snap_ent(entry)[1] - ename = u_str(entry.get('name')) - data['name'] = ename - extra[entry.tag][ename] = data - else: - print("extra", entry.tag, entry.get('name')) - t2 = time.time() - snap = Snapshot.from_data(self.session, correct, revision, - metadata, entries, extra) - self.session.add(snap) - self.session.commit() - t3 = time.time() - logger.info("Snapshot storage took %fs" % (t3 - t2)) - return True diff --git a/src/lib/Bcfg2/Server/Plugins/Statistics.py b/src/lib/Bcfg2/Server/Plugins/Statistics.py deleted file mode 100644 index 7fae445d0..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Statistics.py +++ /dev/null @@ -1,160 +0,0 @@ -'''This file manages the statistics collected by the BCFG2 Server''' - -import copy -import difflib -import logging -import lxml.etree -import os -import sys -from time import asctime, localtime, time, strptime, mktime -import threading -from Bcfg2.Compat import b64decode -import Bcfg2.Server.Plugin - - -class StatisticsStore(object): - """Manages the memory and file copy of statistics collected about client runs.""" - __min_write_delay__ = 0 - - def __init__(self, filename): - self.filename = filename - self.element = lxml.etree.Element('Dummy') - self.dirty = 0 - self.lastwrite = 0 - self.logger = logging.getLogger('Bcfg2.Server.Statistics') - self.ReadFromFile() - - def WriteBack(self, force=0): - """Write statistics changes back to persistent store.""" - if (self.dirty and (self.lastwrite + self.__min_write_delay__ <= time())) \ - or force: - try: - fout = open(self.filename + '.new', 'w') - except IOError: - ioerr = sys.exc_info()[1] - self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr)) - else: - fout.write(lxml.etree.tostring(self.element, - xml_declaration=False).decode('UTF-8')) - fout.close() - os.rename(self.filename + '.new', self.filename) - self.dirty = 0 - self.lastwrite = time() - - def ReadFromFile(self): - """Reads current state regarding statistics.""" - try: - fin = open(self.filename, 'r') - data = fin.read() - fin.close() - self.element = lxml.etree.XML(data) - self.dirty = 0 - except (IOError, lxml.etree.XMLSyntaxError): - self.logger.error("Creating new statistics file %s"%(self.filename)) - self.element = lxml.etree.Element('ConfigStatistics') - self.WriteBack() - self.dirty = 0 - - def updateStats(self, xml, client): - """Updates the statistics of a current node with new data.""" - - # Current policy: - # - Keep anything less than 24 hours old - # - Keep latest clean run for clean nodes - # - Keep latest clean and dirty run for dirty nodes - newstat = xml.find('Statistics') - - if newstat.get('state') == 'clean': - node_dirty = 0 - else: - node_dirty = 1 - - # Find correct node entry in stats data - # The following list comprehension should be guarenteed to return at - # most one result - nodes = [elem for elem in self.element.findall('Node') \ - if elem.get('name') == client] - nummatch = len(nodes) - if nummatch == 0: - # Create an entry for this node - node = lxml.etree.SubElement(self.element, 'Node', name=client) - elif nummatch == 1 and not node_dirty: - # Delete old instance - node = nodes[0] - [node.remove(elem) for elem in node.findall('Statistics') \ - if self.isOlderThan24h(elem.get('time'))] - elif nummatch == 1 and node_dirty: - # Delete old dirty statistics entry - node = nodes[0] - [node.remove(elem) for elem in node.findall('Statistics') \ - if (elem.get('state') == 'dirty' \ - and self.isOlderThan24h(elem.get('time')))] - else: - # Shouldn't be reached - self.logger.error("Duplicate node entry for %s"%(client)) - - # Set current time for stats - newstat.set('time', asctime(localtime())) - - # Add statistic - node.append(copy.copy(newstat)) - - # Set dirty - self.dirty = 1 - self.WriteBack(force=1) - - def isOlderThan24h(self, testTime): - """Helper function to determine if <time> string is older than 24 hours.""" - now = time() - utime = mktime(strptime(testTime)) - secondsPerDay = 60*60*24 - - return (now-utime) > secondsPerDay - - -class Statistics(Bcfg2.Server.Plugin.ThreadedStatistics, - Bcfg2.Server.Plugin.PullSource): - name = 'Statistics' - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore) - Bcfg2.Server.Plugin.PullSource.__init__(self) - fpath = "%s/etc/statistics.xml" % datastore - self.data_file = StatisticsStore(fpath) - - def handle_statistic(self, metadata, data): - self.data_file.updateStats(data, metadata.hostname) - - def FindCurrent(self, client): - rt = self.data_file.element.xpath('//Node[@name="%s"]' % client)[0] - maxtime = max([strptime(stat.get('time')) for stat \ - in rt.findall('Statistics')]) - return [stat for stat in rt.findall('Statistics') \ - if strptime(stat.get('time')) == maxtime][0] - - def GetExtra(self, client): - return [(entry.tag, entry.get('name')) for entry \ - in self.FindCurrent(client).xpath('.//Extra/*')] - - def GetCurrentEntry(self, client, e_type, e_name): - curr = self.FindCurrent(client) - entry = curr.xpath('.//Bad/%s[@name="%s"]' % (e_type, e_name)) - if not entry: - raise Bcfg2.Server.Plugin.PluginExecutionError - cfentry = entry[-1] - - owner = cfentry.get('current_owner', cfentry.get('owner')) - group = cfentry.get('current_group', cfentry.get('group')) - mode = cfentry.get('current_mode', cfentry.get('mode')) - if cfentry.get('sensitive') in ['true', 'True']: - raise Bcfg2.Server.Plugin.PluginExecutionError - elif 'current_bfile' in cfentry.attrib: - contents = b64decode(cfentry.get('current_bfile')) - elif 'current_bdiff' in cfentry.attrib: - diff = b64decode(cfentry.get('current_bdiff')) - contents = '\n'.join(difflib.restore(diff.split('\n'), 1)) - else: - contents = None - - return (owner, group, mode, contents) diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index 240fd7f89..b752650f0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -4,19 +4,33 @@ additional XML-RPC methods for committing data to the repository and updating the repository. """ import sys +import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser try: import pysvn HAS_SVN = True except ImportError: - import pipes - from subprocess import Popen, PIPE + from Bcfg2.Utils import Executor HAS_SVN = False class Svn(Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" + options = Bcfg2.Server.Plugin.Version.options + [ + Bcfg2.Options.Option( + cf=("svn", "conflict_resolution"), dest="svn_conflict_resolution", + type=lambda v: v.replace("-", "_"), + choices=dir(pysvn.wc_conflict_choice), # pylint: disable=E1101 + default=pysvn.wc_conflict_choice.postpone, # pylint: disable=E1101 + help="SVN conflict resolution method"), + Bcfg2.Options.Option( + cf=("svn", "user"), dest="svn_user", help="SVN username"), + Bcfg2.Options.Option( + cf=("svn", "password"), dest="svn_password", help="SVN password"), + Bcfg2.Options.BooleanOption( + cf=("svn", "always_trust"), dest="svn_trust_ssl", + help="Always trust SSL certs from SVN server")] + __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".svn" if HAS_SVN: @@ -24,73 +38,42 @@ class Svn(Bcfg2.Server.Plugin.Version): else: __vcs_metadata_path__ = ".svn" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Version.__init__(self, core) self.revision = None self.svn_root = None + self.client = None + self.cmd = None if not HAS_SVN: self.logger.debug("Svn: PySvn not found, using CLI interface to " "SVN") - self.client = None + self.cmd = Executor() else: self.client = pysvn.Client() - # pylint: disable=E1101 - choice = pysvn.wc_conflict_choice.postpone - try: - resolution = self.core.setup.cfp.get( - "svn", - "conflict_resolution").replace('-', '_') - if resolution in ["edit", "launch", "working"]: - self.logger.warning("Svn: Conflict resolver %s requires " - "manual intervention, using %s" % - choice) - else: - choice = getattr(pysvn.wc_conflict_choice, resolution) - except AttributeError: - self.logger.warning("Svn: Conflict resolver %s does not " - "exist, using %s" % choice) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.info("Svn: No conflict resolution method " - "selected, using %s" % choice) - # pylint: enable=E1101 self.debug_log("Svn: Conflicts will be resolved with %s" % - choice) - self.client.callback_conflict_resolver = \ - self.get_conflict_resolver(choice) + Bcfg2.Options.setup.svn_conflict_resolution) + self.client.callback_conflict_resolver = self.conflict_resolver - try: - if self.core.setup.cfp.get( - "svn", - "always_trust").lower() == "true": - self.client.callback_ssl_server_trust_prompt = \ - self.ssl_server_trust_prompt - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.debug("Svn: Using subversion cache for SSL " - "certificate trust") + if Bcfg2.Options.setup.svn_trust_ssl: + self.client.callback_ssl_server_trust_prompt = \ + self.ssl_server_trust_prompt - try: - if (self.core.setup.cfp.get("svn", "user") and - self.core.setup.cfp.get("svn", "password")): - self.client.callback_get_login = \ - self.get_login - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.info("Svn: Using subversion cache for " - "password-based authetication") + if (Bcfg2.Options.setup.svn_user and + Bcfg2.Options.setup.svn_password): + self.client.callback_get_login = self.get_login self.logger.debug("Svn: Initialized svn plugin with SVN directory %s" % self.vcs_path) - # pylint: disable=W0613 - def get_login(self, realm, username, may_save): + def get_login(self, realm, username, may_save): # pylint: disable=W0613 """ PySvn callback to get credentials for HTTP basic authentication """ self.logger.debug("Svn: Logging in with username: %s" % - self.core.setup.cfp.get("svn", "user")) - return True, \ - self.core.setup.cfp.get("svn", "user"), \ - self.core.setup.cfp.get("svn", "password"), \ - False - # pylint: enable=W0613 + Bcfg2.Options.setup.svn_user) + return (True, + Bcfg2.Options.setup.svn_user, + Bcfg2.Options.setup.svn_password, + False) def ssl_server_trust_prompt(self, trust_dict): """ PySvn callback to always trust SSL certificates from SVN server """ @@ -101,37 +84,35 @@ class Svn(Bcfg2.Server.Plugin.Version): trust_dict['realm'])) return True, trust_dict['failures'], False - def get_conflict_resolver(self, choice): - """ Get a PySvn conflict resolution callback """ - def callback(conflict_description): - """ PySvn callback function to resolve conflicts """ - self.logger.info("Svn: Resolving conflict for %s with %s" % - (conflict_description['path'], choice)) - return choice, None, False - - return callback + def conflict_resolver(self, conflict_description): + """ PySvn callback function to resolve conflicts """ + self.logger.info("Svn: Resolving conflict for %s with %s" % + (conflict_description['path'], + Bcfg2.Options.setup.svn_conflict_resolution)) + return Bcfg2.Options.setup.svn_conflict_resolution, None, False def get_revision(self): """Read svn revision information for the Bcfg2 repository.""" msg = None if HAS_SVN: try: - info = self.client.info(self.vcs_root) + info = self.client.info(Bcfg2.Options.setup.vcs_root) self.revision = info.revision self.svn_root = info.url return str(self.revision.number) except pysvn.ClientError: # pylint: disable=E1101 msg = "Svn: Failed to get revision: %s" % sys.exc_info()[1] else: - try: - data = Popen("env LC_ALL=C svn info %s" % - pipes.quote(self.vcs_root), shell=True, - stdout=PIPE).communicate()[0].split('\n') - return [line.split(': ')[1] for line in data - if line[:9] == 'Revision:'][-1] - except IndexError: - msg = "Failed to read svn info" - self.logger.error('Ran command "svn info %s"' % self.vcs_root) + result = self.cmd.run(["env LC_ALL=C", "svn", "info", + Bcfg2.Options.setup.vcs_root], + shell=True) + if result.success: + self.revision = [line.split(': ')[1] + for line in result.stdout.splitlines() + if line.startswith('Revision:')][-1] + return self.revision + else: + msg = "Failed to read svn info: %s" % result.error self.revision = None raise Bcfg2.Server.Plugin.PluginExecutionError(msg) @@ -139,7 +120,8 @@ class Svn(Bcfg2.Server.Plugin.Version): '''Svn.Update() => True|False\nUpdate svn working copy\n''' try: old_revision = self.revision.number - self.revision = self.client.update(self.vcs_root, recurse=True)[0] + self.revision = self.client.update(Bcfg2.Options.setup.vcs_root, + recurse=True)[0] except pysvn.ClientError: # pylint: disable=E1101 err = sys.exc_info()[1] # try to be smart about the error we got back @@ -161,7 +143,7 @@ class Svn(Bcfg2.Server.Plugin.Version): self.logger.debug("repository is current") else: self.logger.info("Updated %s from revision %s to %s" % - (self.vcs_root, old_revision, + (Bcfg2.Options.setup.vcs_root, old_revision, self.revision.number)) return True @@ -174,10 +156,11 @@ class Svn(Bcfg2.Server.Plugin.Version): return False try: - self.revision = self.client.checkin([self.vcs_root], + self.revision = self.client.checkin([Bcfg2.Options.setup.vcs_root], 'Svn: autocommit', recurse=True) - self.revision = self.client.update(self.vcs_root, recurse=True)[0] + self.revision = self.client.update(Bcfg2.Options.setup.vcs_root, + recurse=True)[0] self.logger.info("Svn: Commited changes. At %s" % self.revision.number) return True diff --git a/src/lib/Bcfg2/Server/Plugins/TCheetah.py b/src/lib/Bcfg2/Server/Plugins/TCheetah.py deleted file mode 100644 index f2c59ce29..000000000 --- a/src/lib/Bcfg2/Server/Plugins/TCheetah.py +++ /dev/null @@ -1,79 +0,0 @@ -'''This module implements a templating generator based on Cheetah''' - -import logging -import sys -import traceback -import Bcfg2.Server.Plugin - -from Bcfg2.Compat import unicode, b64encode - -logger = logging.getLogger('Bcfg2.Plugins.TCheetah') - -try: - import Cheetah.Template - import Cheetah.Parser -except: - logger.error("TCheetah: Failed to import Cheetah. Is it installed?") - raise - - -class TemplateFile: - """Template file creates Cheetah template structures for the loaded file.""" - - def __init__(self, name, specific, encoding): - self.name = name - self.specific = specific - self.encoding = encoding - self.template = None - self.searchlist = dict() - - def handle_event(self, event): - """Handle all fs events for this template.""" - if event.code2str() == 'deleted': - return - try: - s = {'useStackFrames': False} - self.template = Cheetah.Template.Template(open(self.name).read(), - compilerSettings=s, - searchList=self.searchlist) - except Cheetah.Parser.ParseError: - perror = sys.exc_info()[1] - logger.error("Cheetah parse error for file %s" % (self.name)) - logger.error(perror.report()) - - def bind_entry(self, entry, metadata): - """Build literal file information.""" - self.template.metadata = metadata - self.searchlist['metadata'] = metadata - self.template.path = entry.get('realname', entry.get('name')) - self.searchlist['path'] = entry.get('realname', entry.get('name')) - self.template.source_path = self.name - self.searchlist['source_path'] = self.name - - if entry.tag == 'Path': - entry.set('type', 'file') - try: - if type(self.template) == unicode: - entry.text = self.template - else: - if entry.get('encoding') == 'base64': - # take care of case where file needs base64 encoding - entry.text = b64encode(self.template) - else: - entry.text = unicode(str(self.template), self.encoding) - except: - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - logger.error(msg) - logger.error("TCheetah template error for %s" % self.searchlist['path']) - del a, b, c - raise Bcfg2.Server.Plugin.PluginExecutionError - - -class TCheetah(Bcfg2.Server.Plugin.GroupSpool): - """The TCheetah generator implements a templating mechanism for configuration files.""" - name = 'TCheetah' - __author__ = 'bcfg-dev@mcs.anl.gov' - filename_pattern = 'template' - es_child_cls = TemplateFile - deprecated = True diff --git a/src/lib/Bcfg2/Server/Plugins/TGenshi.py b/src/lib/Bcfg2/Server/Plugins/TGenshi.py deleted file mode 100644 index 809587d91..000000000 --- a/src/lib/Bcfg2/Server/Plugins/TGenshi.py +++ /dev/null @@ -1,139 +0,0 @@ -"""This module implements a templating generator based on Genshi.""" - -import logging -import sys -import Bcfg2.Server.Plugin - -from Bcfg2.Compat import unicode, b64encode - -logger = logging.getLogger('Bcfg2.Plugins.TGenshi') - -# try to import genshi stuff -try: - import genshi.core - import genshi.input - from genshi.template import TemplateLoader, \ - TextTemplate, MarkupTemplate, TemplateError -except ImportError: - logger.error("TGenshi: Failed to import Genshi. Is it installed?") - raise -try: - from genshi.template import NewTextTemplate - have_ntt = True -except: - have_ntt = False - -def removecomment(stream): - """A genshi filter that removes comments from the stream.""" - for kind, data, pos in stream: - if kind is genshi.core.COMMENT: - continue - yield kind, data, pos - - -class TemplateFile(object): - """Template file creates Genshi template structures for the loaded file.""" - - def __init__(self, name, specific, encoding): - self.name = name - self.specific = specific - self.encoding = encoding - if self.specific.all: - matchname = self.name - elif self.specific.group: - matchname = self.name[:self.name.find('.G')] - else: - matchname = self.name[:self.name.find('.H')] - if matchname.endswith('.txt'): - self.template_cls = TextTemplate - elif matchname.endswith('.newtxt'): - if not have_ntt: - logger.error("Genshi NewTextTemplates not supported by this version of Genshi") - else: - self.template_cls = NewTextTemplate - else: - self.template_cls = MarkupTemplate - self.HandleEvent = self.handle_event - - def handle_event(self, event=None): - """Handle all fs events for this template.""" - if event and event.code2str() == 'deleted': - return - try: - loader = TemplateLoader() - try: - self.template = loader.load(self.name, cls=self.template_cls, - encoding=self.encoding) - except LookupError: - lerror = sys.exc_info()[1] - logger.error('Genshi lookup error: %s' % lerror) - except TemplateError: - terror = sys.exc_info()[1] - logger.error('Genshi template error: %s' % terror) - except genshi.input.ParseError: - perror = sys.exc_info()[1] - logger.error('Genshi parse error: %s' % perror) - - def bind_entry(self, entry, metadata): - """Build literal file information.""" - fname = entry.get('realname', entry.get('name')) - if entry.tag == 'Path': - entry.set('type', 'file') - try: - stream = self.template.generate( \ - name=fname, metadata=metadata, - path=self.name).filter(removecomment) - if have_ntt: - ttypes = [TextTemplate, NewTextTemplate] - else: - ttypes = [TextTemplate] - if True in [isinstance(self.template, t) for t in ttypes]: - try: - textdata = stream.render('text', strip_whitespace=False) - except TypeError: - textdata = stream.render('text') - if type(textdata) == unicode: - entry.text = textdata - else: - if entry.get('encoding') == 'base64': - # take care of case where file needs base64 encoding - entry.text = b64encode(textdata) - else: - entry.text = unicode(textdata, self.encoding) - else: - try: - xmldata = stream.render('xml', strip_whitespace=False) - except TypeError: - xmldata = stream.render('xml') - if type(xmldata) == unicode: - entry.text = xmldata - else: - entry.text = unicode(xmldata, self.encoding) - if entry.text == '': - entry.set('empty', 'true') - except TemplateError: - err = sys.exc_info()[1] - logger.exception('Genshi template error') - raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template error: %s' % err) - except AttributeError: - err = sys.exc_info()[1] - logger.exception('Genshi template loading error') - raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template loading error: %s' % err) - - -class TemplateEntrySet(Bcfg2.Server.Plugin.EntrySet): - basename_is_regex = True - - -class TGenshi(Bcfg2.Server.Plugin.GroupSpool): - """ - The TGenshi generator implements a templating - mechanism for configuration files. - - """ - name = 'TGenshi' - __author__ = 'jeff@ocjtech.us' - filename_pattern = 'template\.(txt|newtxt|xml)' - es_cls = TemplateEntrySet - es_child_cls = TemplateFile - deprecated = True diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index db7370f01..047fc062e 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -3,11 +3,10 @@ import re import imp import sys -import logging -import Bcfg2.Server.Lint -import Bcfg2.Server.Plugin - -LOGGER = logging.getLogger(__name__) +import lxml.etree +from Bcfg2.Server.Plugin import Plugin, Connector, DirectoryBacked, \ + TemplateDataProvider, DefaultTemplateDataProvider +from Bcfg2.Logger import Debuggable MODULE_RE = re.compile(r'(?P<filename>(?P<module>[^\/]+)\.py)$') @@ -20,12 +19,12 @@ def safe_module_name(module): return '__TemplateHelper_%s' % module -class HelperModule(object): +class HelperModule(Debuggable): """ Representation of a TemplateHelper module """ - def __init__(self, name, fam=None): + def __init__(self, name): + Debuggable.__init__(self) self.name = name - self.fam = fam #: The name of the module as used by get_additional_data(). #: the name of the file with .py stripped off. @@ -34,6 +33,14 @@ class HelperModule(object): #: The attributes exported by this module self._attrs = [] + #: The attributes added to the template namespace by this module + self.defaults = [] + + default_prov = DefaultTemplateDataProvider() + self.reserved_defaults = default_prov.get_template_data( + lxml.etree.Element("Path", name="/dummy"), + None, None).keys() + ["path"] + def HandleEvent(self, event=None): """ HandleEvent is called whenever the FAM registers an event. @@ -49,120 +56,76 @@ class HelperModule(object): self.name) except: # pylint: disable=W0702 err = sys.exc_info()[1] - LOGGER.error("TemplateHelper: Failed to import %s: %s" % - (self.name, err)) + self.logger.error("TemplateHelper: Failed to import %s: %s" % + (self.name, err)) return if not hasattr(module, "__export__"): - LOGGER.error("TemplateHelper: %s has no __export__ list" % - self.name) + self.logger.error("TemplateHelper: %s has no __export__ list" % + self.name) return newattrs = [] - for sym in module.__export__: + for sym in module.__export__ + getattr(module, "__default__", []): + if sym in newattrs: + # already added to attribute list + continue if sym not in self._attrs and hasattr(self, sym): - LOGGER.warning("TemplateHelper: %s: %s is a reserved keyword, " - "skipping export" % (self.name, sym)) + self.logger.warning( + "TemplateHelper: %s: %s is a reserved keyword, " + "skipping export" % (self.name, sym)) continue try: setattr(self, sym, getattr(module, sym)) newattrs.append(sym) except AttributeError: - LOGGER.warning("TemplateHelper: %s exports %s, but has no " - "such attribute" % (self.name, sym)) + self.logger.warning( + "TemplateHelper: %s exports %s, but has no such attribute" + % (self.name, sym)) + # remove old exports for sym in set(self._attrs) - set(newattrs): delattr(self, sym) self._attrs = newattrs + self.defaults = [] + for sym in getattr(module, "__default__", []): + if sym in self.reserved_defaults: + self.logger.warning( + "TemplateHelper: %s: %s is a reserved keyword, not adding " + "as default" % (self.name, sym)) + self.defaults.append(sym) + -class TemplateHelper(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector, - Bcfg2.Server.Plugin.DirectoryBacked): +class TemplateHelper(Plugin, Connector, DirectoryBacked, TemplateDataProvider): """ A plugin to provide helper classes and functions to templates """ __author__ = 'chris.a.st.pierre@gmail.com' ignore = re.compile(r'^(\.#.*|.*~|\..*\.(sw[px])|.*\.py[co])$') patterns = MODULE_RE __child__ = HelperModule - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, core.fam) + def __init__(self, core): + Plugin.__init__(self, core) + Connector.__init__(self) + DirectoryBacked.__init__(self, self.data) + TemplateDataProvider.__init__(self) def get_additional_data(self, _): return dict([(h._module_name, h) # pylint: disable=W0212 for h in self.entries.values()]) - -class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper - <server-plugins-connectors-templatehelper>` modules are valid. - This can check for: - - * A TemplateHelper module that cannot be imported due to syntax or - other compile-time errors; - * A TemplateHelper module that does not have an ``__export__`` - attribute, or whose ``__export__`` is not a list; - * Bogus symbols listed in ``__export__``, including symbols that - don't exist, that are reserved, or that start with underscores. - """ - - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.reserved_keywords = dir(HelperModule("foo.py")) - - def Run(self): - for helper in self.core.plugins['TemplateHelper'].entries.values(): - if self.HandlesFile(helper.name): - self.check_helper(helper.name) - - def check_helper(self, helper): - """ Check a single helper module. - - :param helper: The filename of the helper module - :type helper: string - """ - module_name = MODULE_RE.search(helper).group(1) - - try: - module = imp.load_source(safe_module_name(module_name), helper) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - self.LintError("templatehelper-import-error", - "Failed to import %s: %s" % - (helper, err)) - return - - if not hasattr(module, "__export__"): - self.LintError("templatehelper-no-export", - "%s has no __export__ list" % helper) - return - elif not isinstance(module.__export__, list): - self.LintError("templatehelper-nonlist-export", - "__export__ is not a list in %s" % helper) - return - - for sym in module.__export__: - if not hasattr(module, sym): - self.LintError("templatehelper-nonexistent-export", - "%s: exported symbol %s does not exist" % - (helper, sym)) - elif sym in self.reserved_keywords: - self.LintError("templatehelper-reserved-export", - "%s: exported symbol %s is reserved" % - (helper, sym)) - elif sym.startswith("_"): - self.LintError("templatehelper-underscore-export", - "%s: exported symbol %s starts with underscore" - % (helper, sym)) - - @classmethod - def Errors(cls): - return {"templatehelper-import-error": "error", - "templatehelper-no-export": "error", - "templatehelper-nonlist-export": "error", - "templatehelper-nonexistent-export": "error", - "templatehelper-reserved-export": "error", - "templatehelper-underscore-export": "warning"} + def get_template_data(self, *_): + rv = dict() + source = dict() + for helper in self.entries.values(): + for key in helper.defaults: + if key not in rv: + rv[key] = getattr(helper, key) + source[key] = helper + else: + self.logger.warning( + "TemplateHelper: Duplicate default variable %s " + "provided by both %s and %s" % + (key, helper.name, source[key].name)) + return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Trigger.py b/src/lib/Bcfg2/Server/Plugins/Trigger.py index f7c82fdb3..12672de7d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Trigger.py +++ b/src/lib/Bcfg2/Server/Plugins/Trigger.py @@ -3,18 +3,14 @@ import os import pipes import Bcfg2.Server.Plugin -from subprocess import Popen, PIPE +from Bcfg2.Utils import Executor class TriggerFile(Bcfg2.Server.Plugin.FileBacked): """ Representation of a trigger script file """ - def HandleEvent(self, event=None): return - def __str__(self): - return "%s: %s" % (self.__class__.__name__, self.name) - class Trigger(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.ClientRunHooks, @@ -22,11 +18,11 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, """Trigger is a plugin that calls external scripts (on the server).""" __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) - Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, - self.core.fam) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data) + self.cmd = Executor() def async_run(self, args): """ Run the trigger script asynchronously in a forked process @@ -39,14 +35,12 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, if not dpid: self.debug_log("Running %s" % " ".join(pipes.quote(a) for a in args)) - proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) - err = proc.communicate()[1] - rv = proc.wait() - if rv != 0: - self.logger.error("Trigger: Error running %s (%s): %s" % - (args[0], rv, err)) - elif err: - self.debug_log("Trigger: Error: %s" % err) + result = self.cmd.run(args) + if not result.success: + self.logger.error("Trigger: Error running %s: %s" % + (args[0], result.error)) + elif result.stderr: + self.debug_log("Trigger: Error: %s" % result.stderr) os._exit(0) # pylint: disable=W0212 def end_client_run(self, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/__init__.py b/src/lib/Bcfg2/Server/Plugins/__init__.py index ad51cf368..fdb20ed0a 100644 --- a/src/lib/Bcfg2/Server/Plugins/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/__init__.py @@ -1,5 +1 @@ """Imports for Bcfg2.Server.Plugins.""" - -from Bcfg2.Compat import walk_packages - -__all__ = [m[1] for m in walk_packages(path=__path__)] diff --git a/src/lib/Bcfg2/Server/Reports/reports/models.py b/src/lib/Bcfg2/Server/Reports/reports/models.py index c43c3cee7..ac4c8eac4 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/models.py +++ b/src/lib/Bcfg2/Server/Reports/reports/models.py @@ -53,7 +53,7 @@ class ClientManager(models.Manager): yet been expired as of optional timestmamp argument. Timestamp should be a datetime object.""" - if timestamp == None: + if timestamp is None: timestamp = datetime.now() elif not isinstance(timestamp, datetime): raise ValueError('Expected a datetime object') @@ -64,8 +64,9 @@ class ClientManager(models.Manager): except ValueError: return self.none() - return self.filter(Q(expiration__gt=timestamp) | Q(expiration__isnull=True), - creation__lt=timestamp) + return self.filter( + Q(expiration__gt=timestamp) | Q(expiration__isnull=True), + creation__lt=timestamp) class Client(models.Model): @@ -101,7 +102,8 @@ class InteractiveManager(models.Manager): if maxdate and not isinstance(maxdate, datetime): raise ValueError('Expected a datetime object') - return self.filter(id__in=self.get_interaction_per_client_ids(maxdate, active_only)) + return self.filter( + id__in=self.get_interaction_per_client_ids(maxdate, active_only)) def get_interaction_per_client_ids(self, maxdate=None, active_only=True): """ @@ -116,15 +118,17 @@ class InteractiveManager(models.Manager): cursor = connection.cursor() cfilter = "expiration is null" - sql = 'select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) ' + \ - 'as timer from reports_interaction' + sql = 'select reports_interaction.id, x.client_id ' + \ + 'from (select client_id, MAX(timestamp) ' + \ + 'as timer from reports_interaction' if maxdate: if not isinstance(maxdate, datetime): raise ValueError('Expected a datetime object') sql = sql + " where timestamp <= '%s' " % maxdate cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate) sql = sql + ' GROUP BY client_id) x, reports_interaction where ' + \ - 'reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer' + 'reports_interaction.client_id = x.client_id AND ' + \ + 'reports_interaction.timestamp = x.timer' if active_only: sql = sql + " and x.client_id in (select id from reports_client where %s)" % \ cfilter @@ -138,14 +142,16 @@ class InteractiveManager(models.Manager): class Interaction(models.Model): - """Models each reconfiguration operation interaction between client and server.""" + """Models each reconfiguration operation + interaction between client and server.""" client = models.ForeignKey(Client, related_name="interactions") - timestamp = models.DateTimeField(db_index=True) # Timestamp for this record + timestamp = models.DateTimeField(db_index=True) # record timestamp state = models.CharField(max_length=32) # good/bad/modified/etc - repo_rev_code = models.CharField(max_length=64) # repo revision at time of interaction + # repository revision at the time of the latest interaction + repo_rev_code = models.CharField(max_length=64) goodcount = models.IntegerField() # of good config-items totalcount = models.IntegerField() # of total config-items - server = models.CharField(max_length=256) # Name of the server used for the interaction + server = models.CharField(max_length=256) # server used for interaction bad_entries = models.IntegerField(default=-1) modified_entries = models.IntegerField(default=-1) extra_entries = models.IntegerField(default=-1) @@ -391,5 +397,3 @@ class InteractionMetadata(models.Model): profile = models.ForeignKey(Group, related_name="+") groups = models.ManyToManyField(Group) bundles = models.ManyToManyField(Bundle) - - diff --git a/src/lib/Bcfg2/Server/Reports/updatefix.py b/src/lib/Bcfg2/Server/Reports/updatefix.py index cb131c29d..91c370994 100644 --- a/src/lib/Bcfg2/Server/Reports/updatefix.py +++ b/src/lib/Bcfg2/Server/Reports/updatefix.py @@ -1,5 +1,4 @@ -import Bcfg2.settings - +import Bcfg2.DBSettings from django.db import connection import django.core.management import sys @@ -16,9 +15,9 @@ def _merge_database_table_entries(): find_cursor = connection.cursor() cursor.execute(""" Select name, kind from reports_bad - union + union select name, kind from reports_modified - union + union select name, kind from reports_extra """) # this fetch could be better done @@ -43,20 +42,26 @@ def _merge_database_table_entries(): if entries_map.get(key, None): entry_id = entries_map[key] else: - find_cursor.execute("Select id from reports_entries where name=%s and kind=%s", key) + find_cursor.execute("Select id from reports_entries where " + "name=%s and kind=%s", key) rowe = find_cursor.fetchone() entry_id = rowe[0] - insert_cursor.execute("insert into reports_entries_interactions \ - (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4])) + insert_cursor.execute("insert into reports_entries_interactions " + "(entry_id, interaction_id, reason_id, type) " + "values (%s, %s, %s, %s)", + (entry_id, row[3], row[2], row[4])) def _interactions_constraint_or_idx(): '''sqlite doesn't support alter tables.. or constraints''' cursor = connection.cursor() try: - cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') + cursor.execute('alter table reports_interaction ' + 'add constraint reports_interaction_20100601 ' + 'unique (client_id,timestamp)') except: - cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') + cursor.execute('create unique index reports_interaction_20100601 ' + 'on reports_interaction (client_id,timestamp)') def _populate_interaction_entry_counts(): @@ -67,13 +72,16 @@ def _populate_interaction_entry_counts(): 3: 'extra_entries'} for type in list(count_field.keys()): - cursor.execute("select count(type), interaction_id " + - "from reports_entries_interactions where type = %s group by interaction_id" % type) + cursor.execute("select count(type), interaction_id " + "from reports_entries_interactions " + "where type = %s group by interaction_id" % type) updates = [] for row in cursor.fetchall(): updates.append(row) try: - cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) + cursor.executemany("update reports_interaction set " + + count_field[type] + + "=%s where id = %s", updates) except Exception: e = sys.exc_info()[1] print(e) @@ -106,9 +114,8 @@ _fixes = [_merge_database_table_entries, _interactions_constraint_or_idx, 'alter table reports_reason add is_binary bool NOT NULL default False;', 'alter table reports_reason add is_sensitive bool NOT NULL default False;', - update_noop, #_remove_table_column('reports_interaction', 'client_version'), - "alter table reports_reason add unpruned varchar(1280) not null default 'N/A';", -] + update_noop, # _remove_table_column('reports_interaction', 'client_version'), + "alter table reports_reason add unpruned varchar(1280) not null default 'N/A';"] # this will calculate the last possible version of the database lastversion = len(_fixes) @@ -127,8 +134,10 @@ def rollupdate(current_version): else: _fixes[i]() except: - logger.error("Failed to perform db update %s" % (_fixes[i]), exc_info=1) - # since array start at 0 but version start at 1 we add 1 to the normal count + logger.error("Failed to perform db update %s" % (_fixes[i]), + exc_info=1) + # since array start at 0 but version start at 1 + # we add 1 to the normal count ret = InternalDatabaseVersion.objects.create(version=i + 1) return ret else: diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index ab7e56f33..6ad5b5635 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -14,11 +14,13 @@ from Bcfg2.Compat import xmlrpclib, SimpleXMLRPCServer, SocketServer, \ b64decode +class XMLRPCACLCheckException(Exception): + """ Raised when ACL checks fail on an RPC request """ + + class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): """ An XML-RPC dispatcher. """ - logger = logging.getLogger("Bcfg2.SSLServer.XMLRPCDispatcher") - def __init__(self, allow_none, encoding): try: SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self, @@ -28,11 +30,14 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): # Python 2.4? SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self) + self.logger = logging.getLogger(self.__class__.__name__) self.allow_none = allow_none self.encoding = encoding def _marshaled_dispatch(self, address, data): params, method = xmlrpclib.loads(data) + if not self.instance.check_acls(address, method): + raise XMLRPCACLCheckException try: if '.' not in method: params = (address, ) + params @@ -42,12 +47,12 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): response = (response.decode('utf-8'), ) else: response = (response, ) - raw_response = xmlrpclib.dumps(response, methodresponse=1, + raw_response = xmlrpclib.dumps(response, methodresponse=True, allow_none=self.allow_none, encoding=self.encoding) except xmlrpclib.Fault: fault = sys.exc_info()[1] - raw_response = xmlrpclib.dumps(fault, + raw_response = xmlrpclib.dumps(fault, methodresponse=True, allow_none=self.allow_none, encoding=self.encoding) except: @@ -56,19 +61,18 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): # report exception back to server raw_response = xmlrpclib.dumps( xmlrpclib.Fault(1, "%s:%s" % (err[0].__name__, err[1])), - allow_none=self.allow_none, encoding=self.encoding) + methodresponse=True, allow_none=self.allow_none, + encoding=self.encoding) return raw_response class SSLServer(SocketServer.TCPServer, object): """ TCP server supporting SSL encryption. """ - allow_reuse_address = True - logger = logging.getLogger("Bcfg2.SSLServer.SSLServer") def __init__(self, listen_all, server_address, RequestHandlerClass, keyfile=None, certfile=None, reqCert=False, ca=None, - timeout=None, protocol='xmlrpc/ssl'): + timeout=None, protocol='xmlrpc/tlsv1'): """ :param listen_all: Listen on all interfaces :type listen_all: bool @@ -97,6 +101,8 @@ class SSLServer(SocketServer.TCPServer, object): if ':' in server_address[0]: self.address_family = socket.AF_INET6 + self.logger = logging.getLogger(self.__class__.__name__) + try: SocketServer.TCPServer.__init__(self, listen_address, RequestHandlerClass) @@ -112,7 +118,7 @@ class SSLServer(SocketServer.TCPServer, object): self.socket.settimeout(timeout) self.keyfile = keyfile if (keyfile is not None and - (keyfile == False or + (keyfile is False or not os.path.exists(keyfile) or not os.access(keyfile, os.R_OK))): msg = "Keyfile %s does not exist or is not readable" % keyfile @@ -120,7 +126,7 @@ class SSLServer(SocketServer.TCPServer, object): raise Exception(msg) self.certfile = certfile if (certfile is not None and - (certfile == False or + (certfile is False or not os.path.exists(certfile) or not os.access(certfile, os.R_OK))): msg = "Certfile %s does not exist or is not readable" % certfile @@ -128,7 +134,7 @@ class SSLServer(SocketServer.TCPServer, object): raise Exception(msg) self.ca = ca if (ca is not None and - (ca == False or + (ca is False or not os.path.exists(ca) or not os.access(ca, os.R_OK))): msg = "CA %s does not exist or is not readable" % ca @@ -182,7 +188,11 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): Adds support for HTTP authentication. """ - logger = logging.getLogger("Bcfg2.SSLServer.XMLRPCRequestHandler") + + def __init__(self, *args, **kwargs): + self.logger = logging.getLogger(self.__class__.__name__) + SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.__init__(self, *args, + **kwargs) def authenticate(self): try: @@ -246,7 +256,12 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): data) if sys.hexversion >= 0x03000000: response = response.encode('utf-8') + except XMLRPCACLCheckException: + self.send_error(401, self.responses[401][0]) + self.end_headers() except: # pylint: disable=W0702 + self.logger.error("Unexpected dispatch error for %s: %s" % + (self.client_address, sys.exc_info()[1])) try: self.send_response(500) self.send_header("Content-length", "0") @@ -258,6 +273,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): raise else: # got a valid XML RPC response + client_address = self.request.getpeername() try: self.send_response(200) self.send_header("Content-type", "text/xml") @@ -317,7 +333,7 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, """ Component XMLRPCServer. """ def __init__(self, listen_all, server_address, RequestHandlerClass=None, - keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl', + keyfile=None, certfile=None, ca=None, protocol='xmlrpc/tlsv1', timeout=10, logRequests=False, register=True, allow_none=True, encoding=None): """ diff --git a/src/lib/Bcfg2/Server/Snapshots/__init__.py b/src/lib/Bcfg2/Server/Snapshots/__init__.py deleted file mode 100644 index d42aa0525..000000000 --- a/src/lib/Bcfg2/Server/Snapshots/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -__all__ = ['models', 'db_from_config', 'setup_session'] - -import sqlalchemy -import sqlalchemy.orm -# Compatibility import -from Bcfg2.Compat import ConfigParser - - -def db_from_config(cfile): - cp = ConfigParser.ConfigParser() - cp.read([cfile]) - driver = cp.get('snapshots', 'driver') - if driver == 'sqlite': - path = cp.get('snapshots', 'database') - return 'sqlite:///%s' % path - elif driver in ['mysql', 'postgres']: - user = cp.get('snapshots', 'user') - password = cp.get('snapshots', 'password') - host = cp.get('snapshots', 'host') - db = cp.get('snapshots', 'database') - return '%s://%s:%s@%s/%s' % (driver, user, password, host, db) - else: - raise Exception("unsupported db driver %s" % driver) - - -def setup_session(cfile, debug=False): - engine = sqlalchemy.create_engine(db_from_config(cfile), - echo=debug) - Session = sqlalchemy.orm.sessionmaker() - Session.configure(bind=engine) - return Session() diff --git a/src/lib/Bcfg2/Server/Snapshots/model.py b/src/lib/Bcfg2/Server/Snapshots/model.py deleted file mode 100644 index d578cd2c0..000000000 --- a/src/lib/Bcfg2/Server/Snapshots/model.py +++ /dev/null @@ -1,323 +0,0 @@ -import sys -from sqlalchemy import Table, Column, Integer, Unicode, ForeignKey, Boolean, \ - DateTime, UnicodeText, desc -import datetime -import sqlalchemy.exceptions -from sqlalchemy.orm import relation, backref -from sqlalchemy.ext.declarative import declarative_base - -from Bcfg2.Compat import u_str - - -class Uniquer(object): - force_rt = True - - @classmethod - def by_value(cls, session, **kwargs): - if cls.force_rt: - try: - return session.query(cls).filter_by(**kwargs).one() - except sqlalchemy.exceptions.InvalidRequestError: - return cls(**kwargs) - else: - return cls(**kwargs) - - @classmethod - def from_record(cls, session, data): - return cls.by_value(session, **data) - -Base = declarative_base() - - -class Administrator(Uniquer, Base): - __tablename__ = 'administrator' - id = Column(Integer, primary_key=True) - name = Column(Unicode(20), unique=True) - email = Column(Unicode(64)) - -admin_client = Table('admin_client', Base.metadata, - Column('admin_id', - Integer, - ForeignKey('administrator.id')), - Column('client_id', - Integer, - ForeignKey('client.id'))) - -admin_group = Table('admin_group', Base.metadata, - Column('admin_id', - Integer, - ForeignKey('administrator.id')), - Column('group_id', - Integer, - ForeignKey('group.id'))) - - -class Client(Uniquer, Base): - __tablename__ = 'client' - id = Column(Integer, primary_key=True) - name = Column(Unicode(64), unique=True) - admins = relation("Administrator", secondary=admin_client, - backref='clients') - active = Column(Boolean, default=True) - online = Column(Boolean, default=True) - online_ts = Column(DateTime) - - -class Group(Uniquer, Base): - __tablename__ = 'group' - id = Column(Integer, primary_key=True) - name = Column(Unicode(32), unique=True) - admins = relation("Administrator", secondary=admin_group, - backref='groups') - - -class ConnectorKeyVal(Uniquer, Base): - __tablename__ = 'connkeyval' - id = Column(Integer, primary_key=True) - connector = Column(Unicode(16)) - key = Column(Unicode(32)) - value = Column(UnicodeText) - -meta_group = Table('meta_group', Base.metadata, - Column('metadata_id', - Integer, - ForeignKey('metadata.id')), - Column('group_id', - Integer, - ForeignKey('group.id'))) - -meta_conn = Table('meta_conn', Base.metadata, - Column('metadata_id', - Integer, - ForeignKey('metadata.id')), - Column('connkeyval_id', - Integer, - ForeignKey('connkeyval.id'))) - - -class Metadata(Base): - __tablename__ = 'metadata' - id = Column(Integer, primary_key=True) - client_id = Column(Integer, ForeignKey('client.id')) - client = relation(Client) - groups = relation("Group", secondary=meta_group) - keyvals = relation(ConnectorKeyVal, secondary=meta_conn) - timestamp = Column(DateTime) - - @classmethod - def from_metadata(cls, mysession, mymetadata): - client = Client.by_value(mysession, name=u_str(mymetadata.hostname)) - m = cls(client=client) - for group in mymetadata.groups: - m.groups.append(Group.by_value(mysession, name=u_str(group))) - for connector in mymetadata.connectors: - data = getattr(mymetadata, connector) - if not isinstance(data, dict): - continue - for key, value in list(data.items()): - if not isinstance(value, str): - continue - m.keyvals.append(ConnectorKeyVal.by_value(mysession, - connector=u_str(connector), - key=u_str(key), - value=u_str(value))) - return m - - -class Package(Base, Uniquer): - __tablename__ = 'package' - id = Column(Integer, primary_key=True) - name = Column(Unicode(24)) - type = Column(Unicode(16)) - version = Column(Unicode(16)) - verification_status = Column(Boolean) - - -class CorrespondenceType(object): - mtype = Package - - @classmethod - def from_record(cls, mysession, record): - (mod, corr, name, s_dict, e_dict) = record - if not s_dict: - start = None - else: - start = cls.mtype.by_value(mysession, name=name, **s_dict) - if s_dict != e_dict: - end = cls.mtype.by_value(mysession, name=name, **e_dict) - else: - end = start - return cls(start=start, end=end, modified=mod, correct=corr) - - -class PackageCorrespondence(Base, CorrespondenceType): - mtype = Package - __tablename__ = 'package_pair' - id = Column(Integer, primary_key=True) - start_id = Column(Integer, ForeignKey('package.id')) - start = relation(Package, primaryjoin=start_id == Package.id) - end_id = Column(Integer, ForeignKey('package.id'), nullable=True) - end = relation(Package, primaryjoin=end_id == Package.id) - modified = Column(Boolean) - correct = Column(Boolean) - -package_snap = Table('package_snap', Base.metadata, - Column('ppair_id', - Integer, - ForeignKey('package_pair.id')), - Column('snapshot_id', - Integer, - ForeignKey('snapshot.id'))) - - -class Service(Base, Uniquer): - __tablename__ = 'service' - id = Column(Integer, primary_key=True) - name = Column(Unicode(16)) - type = Column(Unicode(12)) - status = Column(Boolean) - - -class ServiceCorrespondence(Base, CorrespondenceType): - mtype = Service - __tablename__ = 'service_pair' - id = Column(Integer, primary_key=True) - start_id = Column(Integer, ForeignKey('service.id')) - start = relation(Service, primaryjoin=start_id == Service.id) - end_id = Column(Integer, ForeignKey('service.id'), nullable=True) - end = relation(Service, primaryjoin=end_id == Service.id) - modified = Column(Boolean) - correct = Column(Boolean) - -service_snap = Table('service_snap', Base.metadata, - Column('spair_id', - Integer, - ForeignKey('service_pair.id')), - Column('snapshot_id', - Integer, - ForeignKey('snapshot.id'))) - - -class File(Base, Uniquer): - __tablename__ = 'file' - id = Column(Integer, primary_key=True) - name = Column(UnicodeText) - type = Column(Unicode(12)) - owner = Column(Unicode(12)) - group = Column(Unicode(16)) - perms = Column(Integer) - contents = Column(UnicodeText) - - -class FileCorrespondence(Base, CorrespondenceType): - mtype = File - __tablename__ = 'file_pair' - id = Column(Integer, primary_key=True) - start_id = Column(Integer, ForeignKey('file.id')) - start = relation(File, primaryjoin=start_id == File.id) - end_id = Column(Integer, ForeignKey('file.id'), nullable=True) - end = relation(File, primaryjoin=end_id == File.id) - modified = Column(Boolean) - correct = Column(Boolean) - -file_snap = Table('file_snap', Base.metadata, - Column('fpair_id', - Integer, - ForeignKey('file_pair.id')), - Column('snapshot_id', - Integer, - ForeignKey('snapshot.id'))) - -extra_pkg_snap = Table('extra_pkg_snap', Base.metadata, - Column('package_id', - Integer, - ForeignKey('package.id')), - Column('snapshot_id', - Integer, - ForeignKey('snapshot.id'))) - -extra_file_snap = Table('extra_file_snap', Base.metadata, - Column('file_id', - Integer, - ForeignKey('file.id')), - Column('snapshot_id', - Integer, - ForeignKey('snapshot.id'))) - -extra_service_snap = Table('extra_service_snap', Base.metadata, - Column('service_id', - Integer, - ForeignKey('service.id')), - Column('snapshot_id', - Integer, - ForeignKey('snapshot.id'))) - - -class Action(Base): - __tablename__ = 'action' - id = Column(Integer, primary_key=True) - command = Column(UnicodeText) - return_code = Column(Integer) - output = Column(UnicodeText) - -action_snap = Table('action_snap', Base.metadata, - Column('action_id', Integer, ForeignKey('action.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) - - -class Snapshot(Base): - __tablename__ = 'snapshot' - id = Column(Integer, primary_key=True) - correct = Column(Boolean) - revision = Column(Unicode(36)) - metadata_id = Column(Integer, ForeignKey('metadata.id')) - client_metadata = relation(Metadata, primaryjoin=metadata_id == Metadata.id) - timestamp = Column(DateTime, default=datetime.datetime.now) - client_id = Column(Integer, ForeignKey('client.id')) - client = relation(Client, backref=backref('snapshots')) - packages = relation(PackageCorrespondence, secondary=package_snap) - services = relation(ServiceCorrespondence, secondary=service_snap) - files = relation(FileCorrespondence, secondary=file_snap) - actions = relation(Action, secondary=action_snap) - extra_packages = relation(Package, secondary=extra_pkg_snap) - extra_services = relation(Service, secondary=extra_service_snap) - extra_files = relation(File, secondary=extra_file_snap) - - c_dispatch = dict([('Package', ('packages', PackageCorrespondence)), - ('Service', ('services', ServiceCorrespondence)), - ('Path', ('files', FileCorrespondence))]) - e_dispatch = dict([('Package', ('extra_packages', Package)), - ('Service', ('extra_services', Service)), - ('Path', ('extra_files', File))]) - - @classmethod - def from_data(cls, session, correct, revision, metadata, entries, extra): - dbm = Metadata.from_metadata(session, metadata) - snap = cls(correct=correct, client_metadata=dbm, revision=revision, - timestamp=datetime.datetime.now(), client=dbm.client) - for (dispatch, data) in [(cls.c_dispatch, entries), - (cls.e_dispatch, extra)]: - for key in dispatch: - dest, ecls = dispatch[key] - for edata in list(data[key].values()): - getattr(snap, dest).append(ecls.from_record(session, edata)) - return snap - - @classmethod - def by_client(cls, session, clientname): - return session.query(cls).join(cls.client_metadata, - Metadata.client).filter(Client.name == clientname) - - @classmethod - def get_current(cls, session, clientname): - return session.query(Snapshot).join(Snapshot.client_metadata, - Metadata.client).filter(Client.name == clientname).order_by(desc(Snapshot.timestamp)).first() - - @classmethod - def get_by_date(cls, session, clientname, timestamp): - return session.query(Snapshot)\ - .join(Snapshot.client_metadata, Metadata.client)\ - .filter(Snapshot.timestamp < timestamp)\ - .filter(Client.name == clientname)\ - .order_by(desc(Snapshot.timestamp))\ - .first() diff --git a/src/lib/Bcfg2/Statistics.py b/src/lib/Bcfg2/Server/Statistics.py index 3825941af..e34135d4b 100644 --- a/src/lib/Bcfg2/Statistics.py +++ b/src/lib/Bcfg2/Server/Statistics.py @@ -2,6 +2,9 @@ server core. This data is exposed by :func:`Bcfg2.Server.Core.BaseCore.get_statistics`.""" +import time +from Bcfg2.Compat import wraps + class Statistic(object): """ A single named statistic, tracking minimum, maximum, and @@ -85,3 +88,37 @@ class Statistics(object): #: A module-level :class:`Statistics` objects used to track all #: execution time metrics for the server. stats = Statistics() # pylint: disable=C0103 + + +class track_statistics(object): # pylint: disable=C0103 + """ Decorator that tracks execution time for the given method with + :mod:`Bcfg2.Server.Statistics` for reporting via ``bcfg2-admin + perf`` """ + + def __init__(self, name=None): + """ + :param name: The name under which statistics for this function + will be tracked. By default, the name will be + the name of the function concatenated with the + name of the class the function is a member of. + :type name: string + """ + # if this is None, it will be set later during __call_ + self.name = name + + def __call__(self, func): + if self.name is None: + self.name = func.__name__ + + @wraps(func) + def inner(obj, *args, **kwargs): + """ The decorated function """ + name = "%s:%s" % (obj.__class__.__name__, self.name) + + start = time.time() + try: + return func(obj, *args, **kwargs) + finally: + stats.add_value(name, time.time() - start) + + return inner diff --git a/src/lib/Bcfg2/Server/Test.py b/src/lib/Bcfg2/Server/Test.py new file mode 100644 index 000000000..ecbba2fea --- /dev/null +++ b/src/lib/Bcfg2/Server/Test.py @@ -0,0 +1,284 @@ +""" bcfg2-test libraries and CLI """ + +import os +import sys +import shlex +import signal +import fnmatch +import logging +import Bcfg2.Logger +import Bcfg2.Server.Core +from math import ceil +from nose.core import TestProgram +from nose.suite import LazySuite +from unittest import TestCase + +try: + from multiprocessing import Process, Queue, active_children + HAS_MULTIPROC = True +except ImportError: + HAS_MULTIPROC = False + active_children = lambda: [] # pylint: disable=C0103 + + +def get_sigint_handler(core): + """ Get a function that handles SIGINT/Ctrl-C by shutting down the + core and exiting properly.""" + + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + core.shutdown() + os._exit(1) # pylint: disable=W0212 + + return hdlr + + +class CapturingLogger(object): + """ Fake logger that captures logging output so that errors are + only displayed for clients that fail tests """ + def __init__(self, *args, **kwargs): # pylint: disable=W0613 + self.output = [] + + def error(self, msg): + """ discard error messages """ + self.output.append(msg) + + def warning(self, msg): + """ discard error messages """ + self.output.append(msg) + + def info(self, msg): + """ discard error messages """ + self.output.append(msg) + + def debug(self, msg): + """ discard error messages """ + if Bcfg2.Options.setup.debug: + self.output.append(msg) + + def reset_output(self): + """ Reset the captured output """ + self.output = [] + + +class ClientTestFromQueue(TestCase): + """ A test case that tests a value that has been enqueued by a + child test process. ``client`` is the name of the client that has + been tested; ``result`` is the result from the :class:`ClientTest` + test. ``None`` indicates a successful test; a string value + indicates a failed test; and an exception indicates an error while + running the test. """ + __test__ = False # Do not collect + + def __init__(self, client, result): + TestCase.__init__(self) + self.client = client + self.result = result + + def shortDescription(self): + return "Building configuration for %s" % self.client + + def runTest(self): + """ parse the result from this test """ + if isinstance(self.result, Exception): + raise self.result + assert self.result is None, self.result + + +class ClientTest(TestCase): + """ A test case representing the build of all of the configuration for + a single host. Checks that none of the build config entities has + had a failure when it is building. Optionally ignores some config + files that we know will cause errors (because they are private + files we don't have access to, for instance) """ + __test__ = False # Do not collect + divider = "-" * 70 + + def __init__(self, core, client, ignore=None): + TestCase.__init__(self) + self.core = core + self.core.logger = CapturingLogger() + self.client = client + if ignore is None: + self.ignore = dict() + else: + self.ignore = ignore + + def ignore_entry(self, tag, name): + """ return True if an error on a given entry should be ignored + """ + if tag in self.ignore: + if name in self.ignore[tag]: + return True + else: + # try wildcard matching + for pattern in self.ignore[tag]: + if fnmatch.fnmatch(name, pattern): + return True + return False + + def shortDescription(self): + return "Building configuration for %s" % self.client + + def runTest(self): + """ run this individual test """ + config = self.core.BuildConfiguration(self.client) + output = self.core.logger.output[:] + if output: + output.append(self.divider) + self.core.logger.reset_output() + + # check for empty client configuration + assert len(config.findall("Bundle")) > 0, \ + "\n".join(output + ["%s has no content" % self.client]) + + # check for missing bundles + metadata = self.core.build_metadata(self.client) + sbundles = [el.get('name') for el in config.findall("Bundle")] + missing = [b for b in metadata.bundles if b not in sbundles] + assert len(missing) == 0, \ + "\n".join(output + ["Configuration is missing bundle(s): %s" % + ':'.join(missing)]) + + # check for unknown packages + unknown_pkgs = [el.get("name") + for el in config.xpath('//Package[@type="unknown"]') + if not self.ignore_entry(el.tag, el.get("name"))] + assert len(unknown_pkgs) == 0, \ + "Configuration contains unknown packages: %s" % \ + ", ".join(unknown_pkgs) + + failures = [] + msg = output + ["Failures:"] + for failure in config.xpath('//*[@failure]'): + if not self.ignore_entry(failure.tag, failure.get('name')): + failures.append(failure) + msg.append("%s:%s: %s" % (failure.tag, failure.get("name"), + failure.get("failure"))) + + assert len(failures) == 0, "\n".join(msg) + + def __str__(self): + return "ClientTest(%s)" % self.client + + id = __str__ + + +class CLI(object): + """ The bcfg2-test CLI """ + options = [ + Bcfg2.Options.PositionalArgument( + "clients", help="Specific clients to build", nargs="*"), + Bcfg2.Options.Option( + "--nose-options", cf=("bcfg2_test", "nose_options"), + type=shlex.split, default=[], + help='Options to pass to nosetests. Only honored with ' + '--children 0'), + Bcfg2.Options.Option( + "--ignore", cf=('bcfg2_test', 'ignore_entries'), default=[], + dest="test_ignore", type=Bcfg2.Options.Types.comma_list, + help='Ignore these entries if they fail to build'), + Bcfg2.Options.Option( + "--children", cf=('bcfg2_test', 'children'), default=0, type=int, + help='Spawn this number of children for bcfg2-test (python 2.6+)')] + + def __init__(self): + parser = Bcfg2.Options.get_parser( + description="Verify that all clients build without failures", + components=[Bcfg2.Server.Core.Core, self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + if Bcfg2.Options.setup.children and not HAS_MULTIPROC: + self.logger.warning("Python multiprocessing library not found, " + "running with no children") + Bcfg2.Options.setup.children = 0 + + def get_core(self): + """ Get a server core, with events handled """ + core = Bcfg2.Server.Core.Core() + core.load_plugins() + core.block_for_fam_events(handle_events=True) + signal.signal(signal.SIGINT, get_sigint_handler(core)) + return core + + def get_ignore(self): + """ Get a dict of entry tags and names to + ignore errors from """ + ignore = dict() + for entry in Bcfg2.Options.setup.test_ignore: + tag, name = entry.split(":") + try: + ignore[tag].append(name) + except KeyError: + ignore[tag] = [name] + return ignore + + def run_child(self, clients, queue): + """ Run tests for the given clients in a child process, returning + results via the given Queue """ + core = self.get_core() + ignore = self.get_ignore() + for client in clients: + try: + ClientTest(core, client, ignore).runTest() + queue.put((client, None)) + except AssertionError: + queue.put((client, str(sys.exc_info()[1]))) + except: + queue.put((client, sys.exc_info()[1])) + + core.shutdown() + + def run(self): + """ Run bcfg2-test """ + core = self.get_core() + clients = Bcfg2.Options.setup.clients or core.metadata.clients + ignore = self.get_ignore() + + if Bcfg2.Options.setup.children: + if Bcfg2.Options.setup.children > len(clients): + self.logger.info("Refusing to spawn more children than " + "clients to test, setting children=%s" % + len(clients)) + Bcfg2.Options.setup.children = len(clients) + perchild = int(ceil(len(clients) / + float(Bcfg2.Options.setup.children + 1))) + queue = Queue() + for child in range(Bcfg2.Options.setup.children): + start = child * perchild + end = (child + 1) * perchild + child = Process(target=self.run_child, + args=(clients[start:end], queue)) + child.start() + + def generate_tests(): + """ Read test results for the clients """ + start = Bcfg2.Options.setup.children * perchild + for client in clients[start:]: + yield ClientTest(core, client, ignore) + + for i in range(start): # pylint: disable=W0612 + yield ClientTestFromQueue(*queue.get()) + else: + def generate_tests(): + """ Run tests for the clients """ + for client in clients: + yield ClientTest(core, client, ignore) + + result = TestProgram( + argv=sys.argv[:1] + Bcfg2.Options.setup.nose_options, + suite=LazySuite(generate_tests), exit=False) + + # block until all children have completed -- should be + # immediate since we've already gotten all the results we + # expect + for child in active_children(): + child.join() + + core.shutdown() + if result.success: + os._exit(0) # pylint: disable=W0212 + else: + os._exit(1) # pylint: disable=W0212 diff --git a/src/lib/Bcfg2/Server/__init__.py b/src/lib/Bcfg2/Server/__init__.py index 0678e4579..39ed2ec91 100644 --- a/src/lib/Bcfg2/Server/__init__.py +++ b/src/lib/Bcfg2/Server/__init__.py @@ -1,12 +1,11 @@ """This is the set of modules for Bcfg2.Server.""" import lxml.etree -from Bcfg2.Compat import walk_packages - -__all__ = [m[1] for m in walk_packages(path=__path__)] XI = 'http://www.w3.org/2001/XInclude' XI_NAMESPACE = '{%s}' % XI # pylint: disable=C0103 XMLParser = lxml.etree.XMLParser(remove_blank_text=True) + +core = None diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 7e2f5b09d..7150c245a 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -1,34 +1,64 @@ """ Django database models for all plugins """ import sys -import copy import logging import Bcfg2.Options import Bcfg2.Server.Plugins -from django.db import models +from Bcfg2.Compat import walk_packages LOGGER = logging.getLogger('Bcfg2.Server.models') MODELS = [] -def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): +def _get_all_plugins(): + rv = [] + for submodule in walk_packages(path=Bcfg2.Server.Plugins.__path__, + prefix="Bcfg2.Server.Plugins."): + module = submodule[1].rsplit('.', 1)[-1] + if submodule[1] == "Bcfg2.Server.Plugins.%s" % module: + # we only include direct children of + # Bcfg2.Server.Plugins -- e.g., all_plugins should + # include Bcfg2.Server.Plugins.Cfg, but not + # Bcfg2.Server.Plugins.Cfg.CfgInfoXML + rv.append(module) + return rv + + +_ALL_PLUGINS = _get_all_plugins() + + +class _OptionContainer(object): + # we want to provide a different default plugin list -- + # namely, _all_ plugins, so that the database is guaranteed to + # work, even if /etc/bcfg2.conf isn't set up properly + options = [ + Bcfg2.Options.Option( + cf=('server', 'plugins'), type=Bcfg2.Options.Types.comma_list, + default=_ALL_PLUGINS, dest="models_plugins", + action=Bcfg2.Options.PluginsAction)] + + @staticmethod + def options_parsed_hook(): + # basic invocation to ensure that a default set of models is + # loaded, and thus that this module will always work. + load_models() + +Bcfg2.Options.get_parser().add_component(_OptionContainer) + + +def load_models(plugins=None): """ load models from plugins specified in the config """ + # this has to be imported after options are parsed, because Django + # finalizes its settings as soon as it's loaded, which means that + # if we import this before Bcfg2.DBSettings has been populated, + # Django gets a null configuration, and subsequent updates to + # Bcfg2.DBSettings won't help. + from django.db import models global MODELS - if plugins is None: - # we want to provide a different default plugin list -- - # namely, _all_ plugins, so that the database is guaranteed to - # work, even if /etc/bcfg2.conf isn't set up properly - plugin_opt = copy.deepcopy(Bcfg2.Options.SERVER_PLUGINS) - plugin_opt.default = Bcfg2.Server.Plugins.__all__ - - setup = \ - Bcfg2.Options.OptionParser(dict(plugins=plugin_opt, - configfile=Bcfg2.Options.CFILE), - quiet=quiet) - setup.parse([Bcfg2.Options.CFILE.cmd, cfile]) - plugins = setup['plugins'] + if not plugins: + plugins = Bcfg2.Options.setup.models_plugins if MODELS: # load_models() has been called once, so first unload all of @@ -39,45 +69,22 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): delattr(sys.modules[__name__], model) MODELS = [] - for plugin in plugins: - try: - mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % - plugin).Server.Plugins, plugin) - except ImportError: - try: - err = sys.exc_info()[1] - mod = __import__(plugin) - except: # pylint: disable=W0702 - if plugins != Bcfg2.Server.Plugins.__all__: - # only produce errors if the default plugin list - # was not used -- i.e., if the config file was set - # up. don't produce errors when trying to load - # all plugins, IOW. the error from the first - # attempt to import is probably more accurate than - # the second attempt. - LOGGER.error("Failed to load plugin %s: %s" % (plugin, - err)) - continue + for mod in plugins: for sym in dir(mod): obj = getattr(mod, sym) - if hasattr(obj, "__bases__") and models.Model in obj.__bases__: + if isinstance(obj, type) and issubclass(obj, models.Model): setattr(sys.modules[__name__], sym, obj) MODELS.append(sym) -# basic invocation to ensure that a default set of models is loaded, -# and thus that this module will always work. -load_models(quiet=True) - - -class InternalDatabaseVersion(models.Model): - """ Object that tell us to which version the database is """ - version = models.IntegerField() - updated = models.DateTimeField(auto_now_add=True) + class InternalDatabaseVersion(models.Model): + """ Object that tell us to which version the database is """ + version = models.IntegerField() + updated = models.DateTimeField(auto_now_add=True) - def __str__(self): - return "version %d updated the %s" % (self.version, + def __str__(self): + return "version %d updated %s" % (self.version, self.updated.isoformat()) - class Meta: # pylint: disable=C0111,W0232 - app_label = "reports" - get_latest_by = "version" + class Meta: # pylint: disable=C0111,W0232 + app_label = "reports" + get_latest_by = "version" diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index ab1276178..10057b63e 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -2,12 +2,16 @@ used by both client and server. Stuff that doesn't fit anywhere else. """ -import shlex import fcntl import logging -import threading +import os +import re +import select +import shlex +import sys import subprocess -from Bcfg2.Compat import any # pylint: disable=W0622 +import threading +from Bcfg2.Compat import input, any # pylint: disable=W0622 class ClassName(object): @@ -196,9 +200,10 @@ class Executor(object): except OSError: pass - def run(self, command, inputdata=None, shell=False, timeout=None): + def run(self, command, inputdata=None, timeout=None, **kwargs): """ Run a command, given as a list, optionally giving it the - specified input data. + specified input data. All additional keyword arguments are + passed through to :class:`subprocess.Popen`. :param command: The command to run, as a list (preferred) or as a string. See :class:`subprocess.Popen` for @@ -206,31 +211,27 @@ class Executor(object): :type command: list or string :param inputdata: Data to pass to the command on stdin :type inputdata: string - :param shell: Run the given command in a shell (not recommended) - :type shell: bool :param timeout: Kill the command if it runs longer than this many seconds. Set to 0 or -1 to explicitly override a default timeout. :type timeout: float :returns: :class:`Bcfg2.Utils.ExecutorResult` """ + shell = False + if 'shell' in kwargs: + shell = kwargs['shell'] if isinstance(command, str): cmdstr = command - if not shell: command = shlex.split(cmdstr) else: cmdstr = " ".join(command) self.logger.debug("Running: %s" % cmdstr) - try: - proc = subprocess.Popen(command, shell=shell, bufsize=16384, - close_fds=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except OSError: - return ExecutorResult('', 'No such command: %s' % cmdstr, - 127) + args = dict(shell=shell, bufsize=16384, close_fds=True) + args.update(kwargs) + args.update(stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc = subprocess.Popen(command, **args) if timeout is None: timeout = self.timeout if timeout is not None: @@ -252,7 +253,80 @@ class Executor(object): self.logger.debug('< %s' % line) for line in stderr.splitlines(): # pylint: disable=E1103 self.logger.info(line) - return ExecutorResult(stdout, stderr, proc.wait()) + return ExecutorResult(stdout, stderr, + proc.wait()) # pylint: disable=E1101 finally: if timeout is not None: timer.cancel() + + +def list2range(lst): + ''' convert a list of integers to a set of human-readable ranges. e.g.: + + [1, 2, 3, 6, 9, 10, 11] -> "[1-3,6,9-11]" ''' + ilst = sorted(int(i) for i in lst) + ranges = [] + start = None + last = None + for i in ilst: + if not last or i != last + 1: + if start: + if start == last: + ranges.append(str(start)) + else: + ranges.append("%d-%d" % (start, last)) + start = i + last = i + if start: + if start == last: + ranges.append(str(start)) + else: + ranges.append("%d-%d" % (start, last)) + if not ranges: + return "" + elif len(ranges) > 1 or "-" in ranges[0]: + return "[%s]" % ",".join(ranges) + else: + # only one range consisting of only a single number + return ranges[0] + + +def hostnames2ranges(hostnames): + ''' convert a list of hostnames to a set of human-readable ranges. e.g.: + + ["foo1.example.com", "foo2.example.com", "foo3.example.com", + "foo6.example.com"] -> ["foo[1-3,6].example.com"]''' + hosts = {} + hostre = re.compile(r'(\w+?)(\d+)(\..*)$') + for host in hostnames: + match = hostre.match(host) + if match: + key = (match.group(1), match.group(3)) + try: + hosts[key].append(match.group(2)) + except KeyError: + hosts[key] = [match.group(2)] + + ranges = [] + for name, nums in hosts.items(): + ranges.append(name[0] + list2range(nums) + name[1]) + return ranges + + +def safe_input(msg): + """ input() that flushes the input buffer before accepting input """ + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) + return input(msg) + + +class classproperty(object): # pylint: disable=C0103 + """ Decorator that can be used to create read-only class + properties. """ + + def __init__(self, getter): + self.getter = getter + + def __get__(self, instance, owner): + return self.getter(owner) diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py deleted file mode 100644 index 2c5466abb..000000000 --- a/src/lib/Bcfg2/settings.py +++ /dev/null @@ -1,226 +0,0 @@ -""" Django settings for the Bcfg2 server """ - -import os -import sys -import Bcfg2.Options - -try: - import django - HAS_DJANGO = True -except ImportError: - HAS_DJANGO = False - -# required for reporting -try: - import south # pylint: disable=W0611 - HAS_SOUTH = True -except ImportError: - HAS_SOUTH = False - -DATABASES = dict() - -# Django < 1.2 compat -DATABASE_ENGINE = None -DATABASE_NAME = None -DATABASE_USER = None -DATABASE_PASSWORD = None -DATABASE_HOST = None -DATABASE_PORT = None -DATABASE_OPTIONS = None -DATABASE_SCHEMA = None - -TIME_ZONE = None - -DEBUG = False -TEMPLATE_DEBUG = DEBUG - -ALLOWED_HOSTS = ['*'] - -MEDIA_URL = '/site_media/' - - -def _default_config(): - """ get the default config file. returns /etc/bcfg2-web.conf, - UNLESS /etc/bcfg2.conf exists AND /etc/bcfg2-web.conf does not - exist. """ - optinfo = dict(cfile=Bcfg2.Options.CFILE, - web_cfile=Bcfg2.Options.WEB_CFILE) - setup = Bcfg2.Options.OptionParser(optinfo, quiet=True) - setup.parse(sys.argv[1:], do_getopt=False) - if (not os.path.exists(setup['web_cfile']) and - os.path.exists(setup['cfile'])): - return setup['cfile'] - else: - return setup['web_cfile'] - -DEFAULT_CONFIG = _default_config() - - -def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): - """ read the config file and set django settings based on it """ - # pylint: disable=W0602,W0603 - global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ - DATABASE_HOST, DATABASE_PORT, DATABASE_OPTIONS, DATABASE_SCHEMA, \ - DEBUG, TEMPLATE_DEBUG, TIME_ZONE, MEDIA_URL - # pylint: enable=W0602,W0603 - - if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): - print("%s does not exist, using %s for database configuration" % - (cfile, DEFAULT_CONFIG)) - cfile = DEFAULT_CONFIG - - optinfo = Bcfg2.Options.DATABASE_COMMON_OPTIONS - optinfo['repo'] = Bcfg2.Options.SERVER_REPOSITORY - # when setting a different config file, it has to be set in either - # sys.argv or in the OptionSet() constructor AS WELL AS the argv - # that's passed to setup.parse() - argv = [Bcfg2.Options.CFILE.cmd, cfile, - Bcfg2.Options.WEB_CFILE.cmd, cfile] - setup = Bcfg2.Options.OptionParser(optinfo, argv=argv, quiet=quiet) - setup.parse(argv) - - if repo is None: - repo = setup['repo'] - - if setup['db_engine'] == 'ibm_db_django': - db_engine = setup['db_engine'] - else: - db_engine = "django.db.backends.%s" % setup['db_engine'] - - DATABASES['default'] = \ - dict(ENGINE=db_engine, - NAME=setup['db_name'], - USER=setup['db_user'], - PASSWORD=setup['db_password'], - HOST=setup['db_host'], - PORT=setup['db_port'], - OPTIONS=setup['db_options'], - SCHEMA=setup['db_schema']) - - if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: - DATABASE_ENGINE = setup['db_engine'] - DATABASE_NAME = DATABASES['default']['NAME'] - DATABASE_USER = DATABASES['default']['USER'] - DATABASE_PASSWORD = DATABASES['default']['PASSWORD'] - DATABASE_HOST = DATABASES['default']['HOST'] - DATABASE_PORT = DATABASES['default']['PORT'] - DATABASE_OPTIONS = DATABASES['default']['OPTIONS'] - DATABASE_SCHEMA = DATABASES['default']['SCHEMA'] - - # dropping the version check. This was added in 1.1.2 - TIME_ZONE = setup['time_zone'] - - DEBUG = setup['django_debug'] - TEMPLATE_DEBUG = DEBUG - if DEBUG: - print("Warning: Setting web_debug to True causes extraordinary memory " - "leaks. Only use this setting if you know what you're doing.") - - if setup['web_prefix']: - MEDIA_URL = setup['web_prefix'].rstrip('/') + MEDIA_URL - else: - MEDIA_URL = '/site_media/' - -# initialize settings from /etc/bcfg2-web.conf or /etc/bcfg2.conf, or -# set up basic defaults. this lets manage.py work in all cases -read_config(quiet=True) - -ADMINS = (('Root', 'root'),) -MANAGERS = ADMINS - -# Language code for this installation. All choices can be found here: -# http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes -# http://blogs.law.harvard.edu/tech/stories/storyReader$15 -LANGUAGE_CODE = 'en-us' - -SITE_ID = 1 - -# TODO - sanitize this -INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.admin', - 'Bcfg2.Server', -) -if HAS_SOUTH: - INSTALLED_APPS = INSTALLED_APPS + ( - 'south', - 'Bcfg2.Reporting', - ) -if 'BCFG2_LEGACY_MODELS' in os.environ: - INSTALLED_APPS += ('Bcfg2.Server.Reports.reports',) - -# Imported from Bcfg2.Server.Reports -MEDIA_ROOT = '' - -# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a -# trailing slash. -STATIC_URL = '/media/' - -# TODO - make this unique -# Make this unique, and don't share it with anybody. -SECRET_KEY = 'eb5+y%oy-qx*2+62vv=gtnnxg1yig_odu0se5$h0hh#pc*lmo7' - -if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 3: - CACHE_BACKEND = 'locmem:///' -else: - CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - } - } - -if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: - TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.load_template_source', - 'django.template.loaders.app_directories.load_template_source', - ) -else: - TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', - ) - -# TODO - review these. auth and sessions aren't really used -MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.middleware.doc.XViewMiddleware', -) - -# TODO - move this to a higher root and dynamically import -ROOT_URLCONF = 'Bcfg2.Reporting.urls' - -# TODO - this isn't usable -# Authentication Settings -AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend') - -LOGIN_URL = '/login' - -SESSION_EXPIRE_AT_BROWSER_CLOSE = True - -TEMPLATE_DIRS = ( - # App loaders should take care of this.. not sure why this is here - '/usr/share/python-support/python-django/django/contrib/admin/templates/', -) - -# TODO - sanitize this -if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: - TEMPLATE_CONTEXT_PROCESSORS = ( - 'django.core.context_processors.auth', - 'django.core.context_processors.debug', - 'django.core.context_processors.i18n', - 'django.core.context_processors.media', - 'django.core.context_processors.request' - ) -else: - TEMPLATE_CONTEXT_PROCESSORS = ( - 'django.contrib.auth.context_processors.auth', - 'django.core.context_processors.debug', - 'django.core.context_processors.i18n', - 'django.core.context_processors.media', - 'django.core.context_processors.request' - ) diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 61ba7a405..196d77273 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.5" +__version__ = "1.4.0pre1" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 444e86a7c..eca7c3395 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -2,28 +2,9 @@ """Bcfg2 Client""" import sys -import signal -import Bcfg2.Options -from Bcfg2.Client.Client import Client - - -def cb_sigint_handler(signum, frame): - """ Exit upon CTRL-C. """ - raise SystemExit(1) - - -def main(): - optinfo = Bcfg2.Options.CLIENT_COMMON_OPTIONS - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - if setup['args']: - print("Bcfg2 takes no arguments, only options") - print(setup.buildHelpMessage()) - raise SystemExit(1) - - signal.signal(signal.SIGINT, cb_sigint_handler) - return Client(setup).run() +from Bcfg2.Options import get_parser +from Bcfg2.Client import Client if __name__ == '__main__': - sys.exit(main()) + get_parser("Bcfg2 client", components=[Client]).parse() + sys.exit(Client().run()) diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 14d188342..d57cd8b35 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -2,97 +2,11 @@ """ bcfg2-admin is a script that helps to administer a Bcfg2 deployment. """ -import re import sys -import logging -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Admin -from Bcfg2.Compat import StringIO - - -def mode_import(modename): - """Load Bcfg2.Server.Admin.<mode>.""" - modname = modename.capitalize() - mod = getattr(__import__("Bcfg2.Server.Admin.%s" % - (modname)).Server.Admin, modname) - return getattr(mod, modname) - - -def get_modes(): - """Get all available modes, except for the base mode.""" - return [x.lower() for x in Bcfg2.Server.Admin.__all__ if x != 'mode'] - - -def create_description(): - """Create the description string from the list of modes.""" - modes = get_modes() - description = StringIO() - description.write("Available modes are:\n\n") - for mode in modes: - try: - doc = re.sub(r'\s{2,}', ' ', mode_import(mode).__doc__.strip()) - except (ImportError, SystemExit): - continue - description.write((" %-15s %s\n" % (mode, doc))) - return description.getvalue() - - -def main(): - optinfo = dict() - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - # override default help message to include description of all modes - setup.hm = "Usage:\n\n%s\n%s" % (setup.buildHelpMessage(), - create_description()) - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-admin', to_syslog=setup['syslog'], - level=level) - - log = logging.getLogger('bcfg2-admin') - - # Provide help if requested or no args were specified - if (not setup['args'] or len(setup['args']) < 1 or - setup['args'][0] == 'help' or setup['help']): - if len(setup['args']) > 1: - # Get help for a specific mode by passing it the help argument - setup['args'] = [setup['args'][1], setup['args'][0]] - else: - # Print short help for all modes - print(setup.hm) - raise SystemExit(0) - - if setup['args'][0] in get_modes(): - modname = setup['args'][0].capitalize() - if len(setup['args']) > 1 and setup['args'][1] == 'help': - mode_cls = mode_import(modname) - mode_cls.usage(rv=0) - try: - mode_cls = mode_import(modname) - except ImportError: - err = sys.exc_info()[1] - log.error("Failed to load admin mode %s: %s" % (modname, err)) - raise SystemExit(1) - mode = mode_cls(setup) - try: - return mode(setup['args'][1:]) - finally: - mode.shutdown() - else: - log.error("Error: Unknown mode '%s'\n" % setup['args'][0]) - print(create_description()) - raise SystemExit(1) +from Bcfg2.Server.Admin import CLI if __name__ == '__main__': try: - sys.exit(main()) + sys.exit(CLI().run()) except KeyboardInterrupt: raise SystemExit(1) diff --git a/src/sbin/bcfg2-build-reports b/src/sbin/bcfg2-build-reports deleted file mode 100755 index 1c9e9ad97..000000000 --- a/src/sbin/bcfg2-build-reports +++ /dev/null @@ -1,306 +0,0 @@ -#!/usr/bin/env python - -""" -bcfg2-build-reports generates & distributes reports of statistic -information for Bcfg2.""" - -import copy -import getopt -import re -import os -import socket -import sys -from time import asctime, strptime -from lxml.etree import XML, XSLT, parse, Element, ElementTree, SubElement, tostring, XMLSyntaxError -# Compatibility imports -from Bcfg2.Compat import ConfigParser, cmp - -def generatereport(rspec, nrpt): - """ - generatereport creates and returns an ElementTree representation - of a report adhering to the XML spec for intermediate reports. - """ - reportspec = copy.deepcopy(rspec) - nodereprt = copy.deepcopy(nrpt) - - reportgood = reportspec.get("good", default = 'Y') - reportmodified = reportspec.get("modified", default = 'Y') - current_date = asctime()[:10] - - """Build regex of all the nodes we are reporting about.""" - pattern = re.compile( '|'.join([item.get("name") for item in reportspec.findall('Machine')])) - - for node in nodereprt.findall('Node'): - if not (node.findall("Statistics") and pattern.match(node.get('name'))): - # Don't know enough about node. - nodereprt.remove(node) - continue - - # Reduce to most recent Statistics entry. - statisticslist = node.findall('Statistics') - # This line actually sorts from most recent to oldest. - statisticslist.sort(lambda y, x: cmp(strptime(x.get("time")), strptime(y.get("time")))) - stats = statisticslist[0] - - [node.remove(item) for item in node.findall('Statistics')] - - # Add a good tag if node is good and we wnat to report such. - if reportgood == 'Y' and stats.get('state') == 'clean': - SubElement(stats,"Good") - - [stats.remove(item) for item in stats.findall("Bad") + stats.findall("Modified") if \ - item.getchildren() == []] - [stats.remove(item) for item in stats.findall("Modified") if reportmodified == 'N'] - - # Test for staleness -if stale add Stale tag. - if stats.get("time").find(current_date) == -1: - SubElement(stats,"Stale") - node.append(stats) - return nodereprt - -def mail(mailbody, confi): - """mail mails a previously generated report.""" - - try: - mailer = confi.get('statistics', 'sendmailpath') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - mailer = "/usr/sbin/sendmail" - # Open a pipe to the mail program and - # write the data to the pipe. - pipe = os.popen("%s -t" % mailer, 'w') - pipe.write(mailbody) - exitcode = pipe.close() - if exitcode: - print("Exit code: %s" % exitcode) - -def rss(reportxml, delivery, report): - """rss appends a new report to the specified rss file - keeping the last 9 articles. - """ - # Check and see if rss file exists. - for destination in delivery.findall('Destination'): - try: - fil = open(destination.attrib['address'], 'r') - olddoc = XML(fil.read()) - - # Defines the number of recent articles to keep. - items = olddoc.find("channel").findall("item")[0:9] - fil.close() - fil = open(destination.attrib['address'], 'w') - except (IOError, XMLSyntaxError): - fil = open(destination.attrib['address'], 'w') - items = [] - - rssdata = Element("rss") - channel = SubElement(rssdata, "channel") - rssdata.set("version", "2.0") - chantitle = SubElement(channel, "title") - chantitle.text = report.attrib['name'] - chanlink = SubElement(channel, "link") - - # This can later link to WWW report if one gets published - # simultaneously? - chanlink.text = "http://www.mcs.anl.gov/cobalt/bcfg2" - chandesc = SubElement(channel, "description") - chandesc.text = "Information regarding the 10 most recent bcfg2 runs." - - channel.append(XML(reportxml)) - - if items != []: - for item in items: - channel.append(item) - - tree = tostring(rssdata, xml_declaration=False).decode('UTF-8') - fil.write(tree) - fil.close() - -def www(reportxml, delivery): - """www outputs report to.""" - - # This can later link to WWW report if one gets published - # simultaneously? - for destination in delivery.findall('Destination'): - fil = open(destination.attrib['address'], 'w') - - fil.write(reportxml) - fil.close() - -def fileout(reportxml, delivery): - """Outputs to plain text file.""" - for destination in delivery.findall('Destination'): - fil = open(destination.attrib['address'], 'w') - - fil.write(reportxml) - fil.close() - -def pretty_print(element, level=0): - """Produce a pretty-printed text representation of element.""" - if element.text: - fmt = "%s<%%s %%s>%%s</%%s>" % (level*" ") - data = (element.tag, (" ".join(["%s='%s'" % keyval for keyval in list(element.attrib.items())])), - element.text, element.tag) - if element._children: - fmt = "%s<%%s %%s>\n" % (level*" ",) + (len(element._children) * "%s") + "%s</%%s>\n" % (level*" ") - data = (element.tag, ) + (" ".join(["%s='%s'" % keyval for keyval in list(element.attrib.items())]),) - data += tuple([pretty_print(entry, level+2) for entry in element._children]) + (element.tag, ) - else: - fmt = "%s<%%s %%s/>\n" % (level * " ") - data = (element.tag, " ".join(["%s='%s'" % keyval for keyval in list(element.attrib.items())])) - return fmt % data - - -if __name__ == '__main__': - all=False - if '-C' in sys.argv: - cfpath = sys.argv[sys.argv.index('-C') + 1] - else: - cfpath = '/etc/bcfg2.conf' - c = ConfigParser.ConfigParser() - c.read([cfpath]) - configpath = "%s/etc/report-configuration.xml" % c.get('server', 'repository') - statpath = "%s/etc/statistics.xml" % c.get('server', 'repository') - clientsdatapath = "%s/Metadata/clients.xml" % c.get('server', 'repository') - try: - prefix = c.get('server', 'prefix') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - prefix = '/usr' - - transformpath = "/%s/share/bcfg2/xsl-transforms/" % (prefix) - #websrcspath = "/usr/share/bcfg2/web-rprt-srcs/" - - try: - opts, args = getopt.getopt(sys.argv[1:], "C:hAc:Ns:", ["help", "all", "config=", "stats="]) - except getopt.GetoptError: - mesg = sys.exc_info()[1] - # Print help information and exit: - print("%s\nUsage:\nbcfg2-build-reports [-h][-A (include ALL clients)] [-c <configuration-file>] [-s <statistics-file>]" % (mesg)) - raise SystemExit(2) - for o, a in opts: - if o in ("-h", "--help"): - print("Usage:\nbcfg2-build-reports [-h] [-c <configuration-file>] [-s <statistics-file>]") - raise SystemExit - if o in ("-A", "--all"): - all=True - if o in ("-c", "--config"): - configpath = a - if o in ("-s", "--stats"): - statpath = a - - - """Reads data & config files.""" - try: - statsdata = XML(open(statpath).read()) - except (IOError, XMLSyntaxError): - print("bcfg2-build-reports: Failed to parse %s"%(statpath)) - raise SystemExit(1) - try: - configdata = XML(open(configpath).read()) - except (IOError, XMLSyntaxError): - print("bcfg2-build-reports: Failed to parse %s"%(configpath)) - raise SystemExit(1) - try: - clientsdata = XML(open(clientsdatapath).read()) - except (IOError, XMLSyntaxError): - print("bcfg2-build-reports: Failed to parse %s"%(clientsdatapath)) - raise SystemExit(1) - - # Merge data from three sources. - nodereport = Element("Report", attrib={"time" : asctime()}) - # Should all of the other info in Metadata be appended? - # What about all of the package stuff for other types of reports? - for client in clientsdata.findall("Client"): - nodel = Element("Node", attrib={"name" : client.get("name")}) - nodel.append(client) - for nod in statsdata.findall("Node"): - if client.get('name').find(nod.get('name')) == 0: - for statel in nod.findall("Statistics"): - nodel.append(statel) - nodereport.append(nodel) - - if all: - for nod in statsdata.findall("Node"): - for client in clientsdata.findall("Client"): - if client.get('name').find(nod.get('name')) == 0: - break - else: - nodel = Element("Node", attrib={"name" : nod.get("name")}) - client = Element("Client", attrib={"name" : nod.get("name"), "profile" : "default"}) - nodel.append(client) - for statel in nod.findall("Statistics"): - nodel.append(statel) - nodereport.append(nodel) - - - for reprt in configdata.findall('Report'): - nodereport.set("name", reprt.get("name", default="BCFG Report")) - - if reprt.get('refresh-time') != None: - nodereport.set("refresh-time", reprt.get("refresh-time", default="600")) - - procnodereport = generatereport(reprt, nodereport) - - for deliv in reprt.findall('Delivery'): - # Is a deepcopy of procnodereport necessary? - - delivtype = deliv.get('type', default='nodes-digest') - deliverymechanism = deliv.get('mechanism', default='www') - - # Apply XSLT, different ones based on report type, and options - if deliverymechanism == 'null-operator': # Special Cases - fileout(tostring(ElementTree(procnodereport).getroot(), xml_declaration=False).decode('UTF-8'), deliv) - break - transform = delivtype + '-' + deliverymechanism + '.xsl' - - try: # Make sure valid stylesheet is selected. - os.stat(transformpath + transform) - except: - print("bcfg2-build-reports: Invalid report type or delivery mechanism.\n Can't find: "\ - + transformpath + transform) - raise SystemExit(1) - - try: # Try to parse stylesheet. - stylesheet = XSLT(parse(transformpath + transform)) - except: - print("bcfg2-build-reports: invalid XSLT transform file.") - raise SystemExit(1) - - if deliverymechanism == 'mail': - if delivtype == 'nodes-individual': - reportdata = copy.deepcopy(procnodereport) - for noden in reportdata.findall("Node"): - [reportdata.remove(y) for y in reportdata.findall("Node")] - reportdata.append(noden) - result = stylesheet.apply(ElementTree(reportdata)) - outputstring = stylesheet.tostring(result) - - if not outputstring == None: - toastring = '' - for desti in deliv.findall("Destination"): - toastring = "%s%s " % \ - (toastring, desti.get('address')) - # Prepend To: and From: - outputstring = "To: %s\nFrom: root@%s\n%s"% \ - (toastring, socket.getfqdn(), outputstring) - mail(outputstring, c) #call function to send - - else: - reportdata = copy.deepcopy(procnodereport) - - result = stylesheet.apply(ElementTree(reportdata)) - outputstring = stylesheet.tostring(result) - - if not outputstring == None: - toastring = '' - for desti in deliv.findall("Destination"): - toastring = "%s%s " % \ - (toastring, desti.get('address')) - # Prepend To: and From: - outputstring = "To: %s\nFrom: root@%s\n%s"% \ - (toastring, socket.getfqdn(), outputstring) - mail(outputstring, c) #call function to send - else: - outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot(), xml_declaration=False).decode('UTF-8') - if deliverymechanism == 'rss': - rss(outputstring, deliv, reprt) - else: # Must be deliverymechanism == 'www': - www(outputstring, deliv) diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index 2212c2360..26d5eedf1 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -1,483 +1,8 @@ #!/usr/bin/env python """ helper for encrypting/decrypting Cfg and Properties files """ -import os import sys -import copy -import select -import logging -import lxml.etree -import Bcfg2.Logger -import Bcfg2.Options -from Bcfg2.Server import XMLParser -from Bcfg2.Compat import input # pylint: disable=W0622 -try: - import Bcfg2.Encryption -except ImportError: - print("Could not import %s. Is M2Crypto installed?" % sys.exc_info()[1]) - raise SystemExit(1) - - -def print_xml(element, keep_text=False): - """ Render an XML element for error output. This prefixes the - line number and removes children for nicer display. - - :param element: The element to render - :type element: lxml.etree._Element - :param keep_text: Do not discard text content from the element for - display - :type keep_text: boolean - """ - xml = None - if len(element) or element.text: - el = copy.copy(element) - if el.text and not keep_text: - el.text = '...' - for child in el.iterchildren(): - el.remove(child) - xml = lxml.etree.tostring( - el, - xml_declaration=False).decode("UTF-8").strip() - else: - xml = lxml.etree.tostring( - element, - xml_declaration=False).decode("UTF-8").strip() - return "%s (line %s)" % (xml, element.sourceline) - - -class PassphraseError(Exception): - """ Exception raised when there's a problem determining the - passphrase to encrypt or decrypt with """ - - -class DecryptError(Exception): - """ Exception raised when decryption fails. """ - - -class EncryptError(Exception): - """ Exception raised when encryption fails. """ - - -class CryptoTool(object): - """ Generic decryption/encryption interface base object """ - def __init__(self, filename, setup): - self.setup = setup - self.logger = logging.getLogger(self.__class__.__name__) - self.passphrases = Bcfg2.Encryption.get_passphrases(self.setup) - - self.filename = filename - try: - self.data = open(self.filename).read() - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error reading %s, skipping: %s" % (filename, - err)) - return False - - self.pname, self.passphrase = self._get_passphrase() - - def _get_passphrase(self): - """ get the passphrase for the current file """ - if (not self.setup.cfp.has_section(Bcfg2.Encryption.CFG_SECTION) or - len(Bcfg2.Encryption.get_passphrases(self.setup)) == 0): - raise PassphraseError("No passphrases available in %s" % - self.setup['configfile']) - - pname = None - if self.setup['passphrase']: - pname = self.setup['passphrase'] - - if pname: - if self.setup.cfp.has_option(Bcfg2.Encryption.CFG_SECTION, - pname): - passphrase = self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, - pname) - self.logger.debug("Using passphrase %s specified on command " - "line" % pname) - return (pname, passphrase) - else: - raise PassphraseError("Could not find passphrase %s in %s" % - (pname, self.setup['configfile'])) - else: - pnames = Bcfg2.Encryption.get_passphrases(self.setup) - if len(pnames) == 1: - pname = pnames.keys()[0] - passphrase = pnames[pname] - self.logger.info("Using passphrase %s" % pname) - return (pname, passphrase) - elif len(pnames) > 1: - return (None, None) - raise PassphraseError("No passphrase could be determined") - - def get_destination_filename(self, original_filename): - """ Get the filename where data should be written """ - return original_filename - - def write(self, data): - """ write data to disk """ - new_fname = self.get_destination_filename(self.filename) - try: - self._write(new_fname, data) - self.logger.info("Wrote data to %s" % new_fname) - return True - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error writing data from %s to %s: %s" % - (self.filename, new_fname, err)) - return False - - def _write(self, filename, data): - """ Perform the actual write of data. This is separate from - :func:`CryptoTool.write` so it can be easily - overridden. """ - open(filename, "wb").write(data) - - -class Decryptor(CryptoTool): - """ Decryptor interface """ - def decrypt(self): - """ decrypt the file, returning the encrypted data """ - raise NotImplementedError - - -class Encryptor(CryptoTool): - """ encryptor interface """ - def encrypt(self): - """ encrypt the file, returning the encrypted data """ - raise NotImplementedError - - -class CfgEncryptor(Encryptor): - """ encryptor class for Cfg files """ - - def __init__(self, filename, setup): - Encryptor.__init__(self, filename, setup) - if self.passphrase is None: - raise PassphraseError("Multiple passphrases found in %s, " - "specify one on the command line with -p" % - self.setup['configfile']) - - def encrypt(self): - return Bcfg2.Encryption.ssl_encrypt( - self.data, self.passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)) - - def get_destination_filename(self, original_filename): - return original_filename + ".crypt" - - -class CfgDecryptor(Decryptor): - """ Decrypt Cfg files """ - - def decrypt(self): - """ decrypt the given file, returning the plaintext data """ - if self.passphrase: - try: - return Bcfg2.Encryption.ssl_decrypt( - self.data, self.passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)) - except Bcfg2.Encryption.EVPError: - raise DecryptError("Could not decrypt %s with the " - "specified passphrase" % self.filename) - except: - raise DecryptError("Error decrypting %s: %s" % - (self.filename, sys.exc_info()[1])) - else: # no passphrase given, brute force - try: - return Bcfg2.Encryption.bruteforce_decrypt( - self.data, passphrases=self.passphrases.values(), - algorithm=Bcfg2.Encryption.get_algorithm(self.setup)) - except Bcfg2.Encryption.EVPError: - raise DecryptError("Could not decrypt %s with any passphrase" % - self.filename) - - def get_destination_filename(self, original_filename): - if original_filename.endswith(".crypt"): - return original_filename[:-6] - else: - return Decryptor.get_plaintext_filename(self, original_filename) - - -class PropertiesCryptoMixin(object): - """ Mixin to provide some common methods for Properties crypto """ - default_xpath = '//*[@encrypted]' - - def _get_elements(self, xdata): - """ Get the list of elements to encrypt or decrypt """ - if self.setup['xpath']: - elements = xdata.xpath(self.setup['xpath']) - if not elements: - self.logger.warning("XPath expression %s matched no " - "elements" % self.setup['xpath']) - else: - elements = xdata.xpath(self.default_xpath) - if not elements: - elements = list(xdata.getiterator(tag=lxml.etree.Element)) - - # filter out elements without text data - for el in elements[:]: - if not el.text: - elements.remove(el) - - if self.setup['interactive']: - for element in elements[:]: - if len(element): - elt = copy.copy(element) - for child in elt.iterchildren(): - elt.remove(child) - else: - elt = element - print(lxml.etree.tostring( - elt, - xml_declaration=False).decode("UTF-8").strip()) - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], - 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - ans = input("Encrypt this element? [y/N] ") - if not ans.lower().startswith("y"): - elements.remove(element) - return elements - - def _get_element_passphrase(self, element): - """ Get the passphrase to use to encrypt or decrypt a given - element """ - pname = element.get("encrypted") - if pname in self.passphrases: - passphrase = self.passphrases[pname] - elif self.passphrase: - if pname: - self.logger.warning("Passphrase %s not found in %s, " - "using passphrase given on command line" - % (pname, self.setup['configfile'])) - passphrase = self.passphrase - pname = self.pname - else: - raise PassphraseError("Multiple passphrases found in %s, " - "specify one on the command line with -p" % - self.setup['configfile']) - return (pname, passphrase) - - def _write(self, filename, data): - """ Write the data """ - data.getroottree().write(filename, - xml_declaration=False, - pretty_print=True) - - -class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): - """ encryptor class for Properties files """ - - def encrypt(self): - xdata = lxml.etree.XML(self.data, parser=XMLParser) - for elt in self._get_elements(xdata): - try: - pname, passphrase = self._get_element_passphrase(elt) - except PassphraseError: - raise EncryptError(str(sys.exc_info()[1])) - self.logger.debug("Encrypting %s" % print_xml(elt)) - elt.text = Bcfg2.Encryption.ssl_encrypt( - elt.text, passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() - elt.set("encrypted", pname) - return xdata - - def _write(self, filename, data): - PropertiesCryptoMixin._write(self, filename, data) - - -class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): - """ decryptor class for Properties files """ - - def decrypt(self): - decrypted_any = False - xdata = lxml.etree.XML(self.data, parser=XMLParser) - for elt in self._get_elements(xdata): - try: - pname, passphrase = self._get_element_passphrase(elt) - except PassphraseError: - raise DecryptError(str(sys.exc_info()[1])) - self.logger.debug("Decrypting %s" % print_xml(elt)) - try: - decrypted = Bcfg2.Encryption.ssl_decrypt( - elt.text, passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() - decrypted_any = True - except (Bcfg2.Encryption.EVPError, TypeError): - self.logger.error("Could not decrypt %s, skipping" % - print_xml(elt)) - continue - try: - elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') - elt.set("encrypted", pname) - except UnicodeDecodeError: - # we managed to decrypt the value, but it contains - # content that can't even be encoded into xml - # entities. what probably happened here is that we - # coincidentally could decrypt a value encrypted with - # a different key, and wound up with gibberish. - self.logger.warning("Decrypted %s to gibberish, skipping" % - elt.tag) - if decrypted_any: - return xdata - else: - raise DecryptError("Failed to decrypt any data in %s" % - self.filename) - - def _write(self, filename, data): - PropertiesCryptoMixin._write(self, filename, data) - - -def main(): # pylint: disable=R0912,R0915 - optinfo = dict(interactive=Bcfg2.Options.INTERACTIVE) - optinfo.update(Bcfg2.Options.CRYPT_OPTIONS) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.hm = " bcfg2-crypt [options] <filename>\nOptions:\n%s" % \ - setup.buildHelpMessage() - setup.parse(sys.argv[1:]) - - if not setup['args']: - print(setup.hm) - raise SystemExit(1) - - log_args = dict(to_syslog=setup['syslog'], to_console=logging.WARNING) - if setup['verbose']: - log_args['to_console'] = logging.DEBUG - Bcfg2.Logger.setup_logging('bcfg2-crypt', **log_args) - logger = logging.getLogger('bcfg2-crypt') - - if setup['decrypt']: - if setup['encrypt']: - logger.error("You cannot specify both --encrypt and --decrypt") - raise SystemExit(1) - elif setup['remove']: - logger.error("--remove cannot be used with --decrypt, ignoring") - setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default - elif setup['interactive']: - logger.error("Cannot decrypt interactively") - setup['interactive'] = False - - if setup['cfg']: - if setup['properties']: - logger.error("You cannot specify both --cfg and --properties") - raise SystemExit(1) - if setup['xpath']: - logger.error("Specifying --xpath with --cfg is nonsensical, " - "ignoring --xpath") - setup['xpath'] = Bcfg2.Options.CRYPT_XPATH.default - if setup['interactive']: - logger.error("You cannot use interactive mode with --cfg, " - "ignoring -I") - setup['interactive'] = False - elif setup['properties']: - if setup['remove']: - logger.error("--remove cannot be used with --properties, ignoring") - setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default - - for fname in setup['args']: - if not os.path.exists(fname): - logger.error("%s does not exist, skipping" % fname) - continue - - # figure out if we need to encrypt this as a Properties file - # or as a Cfg file - props = False - if setup['properties']: - props = True - elif setup['cfg']: - props = False - elif fname.endswith(".xml"): - try: - xroot = lxml.etree.parse(fname).getroot() - if xroot.tag == "Properties": - props = True - else: - props = False - except IOError: - err = sys.exc_info()[1] - logger.error("Error reading %s, skipping: %s" % (fname, err)) - continue - except lxml.etree.XMLSyntaxError: - props = False - else: - props = False - - if props: - if setup['remove']: - logger.info("Cannot use --remove with Properties file %s, " - "ignoring for this file" % fname) - tools = (PropertiesEncryptor, PropertiesDecryptor) - else: - if setup['xpath']: - logger.info("Cannot use xpath with Cfg file %s, ignoring " - "xpath for this file" % fname) - if setup['interactive']: - logger.info("Cannot use interactive mode with Cfg file %s, " - "ignoring -I for this file" % fname) - tools = (CfgEncryptor, CfgDecryptor) - - data = None - mode = None - if setup['encrypt']: - try: - tool = tools[0](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - mode = "encrypt" - elif setup['decrypt']: - try: - tool = tools[1](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - mode = "decrypt" - else: - logger.info("Neither --encrypt nor --decrypt specified, " - "determining mode") - try: - tool = tools[1](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - - try: - data = tool.decrypt() - mode = "decrypt" - except DecryptError: - logger.info("Failed to decrypt %s, trying encryption" % fname) - try: - tool = tools[0](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - mode = "encrypt" - - if data is None: - try: - data = getattr(tool, mode)() - except (EncryptError, DecryptError): - logger.error("Failed to %s %s, skipping: %s" % - (mode, fname, sys.exc_info()[1])) - continue - if setup['crypt_stdout']: - if len(setup['args']) > 1: - print("----- %s -----" % fname) - print(data) - if len(setup['args']) > 1: - print("") - else: - tool.write(data) - - if (setup['remove'] and - tool.get_destination_filename(fname) != fname): - try: - os.unlink(fname) - except IOError: - err = sys.exc_info()[1] - logger.error("Error removing %s: %s" % (fname, err)) - continue +from Bcfg2.Server.Encryption import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index a6c3149bc..adfa96852 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -1,806 +1,8 @@ #!/usr/bin/env python """This tool loads the Bcfg2 core into an interactive debugger.""" -import os -import re import sys -import cmd -import getopt -import fnmatch -import logging -import lxml.etree -import traceback -from code import InteractiveConsole -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Core -import Bcfg2.Server.Plugin -import Bcfg2.Client.Tools.POSIX - -try: - try: - import cProfile as profile - except ImportError: - import profile - import pstats - HAS_PROFILE = True -except ImportError: - HAS_PROFILE = False - - -class MockLog(object): - """ Fake logger that just discards all messages in order to mask - errors from builddir being unable to chown files it creates """ - def error(self, *args, **kwargs): - """ discard error messages """ - pass - - def warning(self, *args, **kwargs): - """ discard warning messages """ - pass - - def info(self, *args, **kwargs): - """ discard info messages """ - pass - - def debug(self, *args, **kwargs): - """ discard debug messages """ - pass - - -class FileNotBuilt(Exception): - """Thrown when File entry contains no content.""" - def __init__(self, value): - Exception.__init__(self) - self.value = value - - def __str__(self): - return repr(self.value) - - -def print_tabular(rows): - """Print data in tabular format.""" - cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 - for index in range(len(rows[0]))]) - fstring = (" %%-%ss |" * len(cmax)) % cmax - fstring = ('|'.join([" %%-%ss "] * len(cmax))) % cmax - print(fstring % rows[0]) - print((sum(cmax) + (len(cmax) * 2) + (len(cmax) - 1)) * '=') - for row in rows[1:]: - print(fstring % row) - - -def display_trace(trace): - """ display statistics from a profile trace """ - stats = pstats.Stats(trace) - stats.sort_stats('cumulative', 'calls', 'time') - stats.print_stats(200) - - -def load_interpreters(): - """ Load a dict of available Python interpreters """ - interpreters = dict(python=lambda v: InteractiveConsole(v).interact()) - best = "python" - try: - import bpython.cli - interpreters["bpython"] = lambda v: bpython.cli.main(args=[], - locals_=v) - best = "bpython" - except ImportError: - pass - - try: - # whether ipython is actually better than bpython is - # up for debate, but this is the behavior that existed - # before --interpreter was added, so we call IPython - # better - import IPython - # pylint: disable=E1101 - if hasattr(IPython, "Shell"): - interpreters["ipython"] = lambda v: \ - IPython.Shell.IPShell(argv=[], user_ns=v).mainloop() - best = "ipython" - elif hasattr(IPython, "embed"): - interpreters["ipython"] = lambda v: IPython.embed(user_ns=v) - best = "ipython" - else: - print("Unknown IPython API version") - # pylint: enable=E1101 - except ImportError: - pass - - interpreters['best'] = interpreters[best] - return interpreters - - -class InfoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): - """Main class for bcfg2-info.""" - def __init__(self, setup): - cmd.Cmd.__init__(self) - Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) - self.prompt = '> ' - self.cont = True - - def _get_client_list(self, hostglobs): - """ given a host glob, get a list of clients that match it """ - # special cases to speed things up: - if '*' in hostglobs: - return self.metadata.clients - has_wildcards = False - for glob in hostglobs: - # check if any wildcard characters are in the string - if set('*?[]') & set(glob): - has_wildcards = True - break - if not has_wildcards: - return hostglobs - - rv = set() - clist = set(self.metadata.clients) - for glob in hostglobs: - for client in clist: - if fnmatch.fnmatch(client, glob): - rv.update(client) - clist.difference_update(rv) - return list(rv) - - def _get_usage(self, func): - """ get the short usage message for a given function """ - return "Usage: " + re.sub(r'\s+', ' ', func.__doc__).split(" - ", 1)[0] - - def do_loop(self): - """Looping.""" - self.cont = True - while self.cont: - try: - self.cmdloop('Welcome to bcfg2-info\n' - 'Type "help" for more information') - except SystemExit: - raise - except Bcfg2.Server.Plugin.PluginExecutionError: - continue - except KeyboardInterrupt: - print("Ctrl-C pressed exiting...") - self.do_exit([]) - except: - self.logger.error("Command failure", exc_info=1) - - def do_debug(self, args): - """ debug [-n] [-f <command list>] - Shell out to native - python interpreter """ - try: - opts, _ = getopt.getopt(args.split(), 'nf:') - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self._get_usage(self.do_debug)) - return - self.cont = False - scriptmode = False - interactive = True - for opt in opts: - if opt[0] == '-f': - scriptmode = True - spath = opt[1] - elif opt[0] == '-n': - interactive = False - if scriptmode: - console = InteractiveConsole(locals()) - for command in [c.strip() for c in open(spath).readlines()]: - if command: - console.push(command) - if interactive: - interpreters = load_interpreters() - if self.setup['interpreter'] in interpreters: - print("Dropping to %s interpreter; press ^D to resume" % - self.setup['interpreter']) - interpreters[self.setup['interpreter']](locals()) - else: - self.logger.error("Invalid interpreter %s" % - self.setup['interpreter']) - self.logger.error("Valid interpreters are: %s" % - ", ".join(interpreters.keys())) - - def do_quit(self, _): - """ quit|exit - Exit program """ - self.shutdown() - os._exit(0) # pylint: disable=W0212 - - do_EOF = do_quit - do_exit = do_quit - - def do_help(self, _): - """ help - Print this list of available commands """ - print(USAGE) - - def do_update(self, _): - """ update - Process pending filesystem events""" - self.fam.handle_events_in_interval(0.1) - - def do_build(self, args): - """ build [-f] <hostname> <filename> - Build config for - hostname, writing to filename""" - alist = args.split() - path_force = False - for arg in alist: - if arg == '-f': - alist.remove('-f') - path_force = True - if len(alist) == 2: - client, ofile = alist - if not ofile.startswith('/tmp') and not path_force: - print("Refusing to write files outside of /tmp without -f " - "option") - return - try: - lxml.etree.ElementTree(self.BuildConfiguration(client)).write( - ofile, - encoding='UTF-8', xml_declaration=True, - pretty_print=True) - except IOError: - err = sys.exc_info()[1] - print("Failed to write File %s: %s" % (ofile, err)) - else: - print(self._get_usage(self.do_build)) - - def help_builddir(self): - """Display help for builddir command.""" - print("""Usage: builddir [-f] <hostname> <output dir> - -Generates a config for client <hostname> and writes the -individual configuration files out separately in a tree -under <output dir>. The <output dir> directory must be -rooted under /tmp unless the -f argument is provided, in -which case it can be located anywhere. - -NOTE: Currently only handles file entries and writes -all content with the default owner and permissions. These -could be much more permissive than would be created by the -Bcfg2 client itself.""") - - def do_builddir(self, args): - """ builddir [-f] <hostname> <dirname> - Build config for - hostname, writing separate files to dirname""" - alist = args.split() - path_force = False - if '-f' in args: - alist.remove('-f') - path_force = True - if len(alist) == 2: - client, odir = alist - if not odir.startswith('/tmp') and not path_force: - print("Refusing to write files outside of /tmp without -f " - "option") - return - client_config = self.BuildConfiguration(client) - if client_config.tag == 'error': - print("Building client configuration failed.") - return - - for struct in client_config: - for entry in struct: - if entry.tag == 'Path': - entry.set('name', odir + '/' + entry.get('name')) - - posix = Bcfg2.Client.Tools.POSIX.POSIX(MockLog(), - self.setup, - client_config) - states = dict() - posix.Inventory(states) - posix.Install(list(states.keys()), states) - else: - print('Error: Incorrect number of parameters.') - self.help_builddir() - - def do_buildall(self, args): - """ buildall <directory> [<hostnames*>] - Build configs for - all clients in directory """ - alist = args.split() - if len(alist) < 1: - print(self._get_usage(self.do_buildall)) - return - - destdir = alist[0] - try: - os.mkdir(destdir) - except OSError: - err = sys.exc_info()[1] - if err.errno != 17: - print("Could not create %s: %s" % (destdir, err)) - if len(alist) > 1: - clients = self._get_client_list(alist[1:]) - else: - clients = self.metadata.clients - for client in clients: - self.do_build("%s %s" % (client, os.path.join(destdir, - client + ".xml"))) - - def do_buildallfile(self, args): - """ buildallfile <directory> <filename> [<hostnames*>] - Build - config file for all clients in directory """ - try: - opts, args = getopt.gnu_getopt(args.split(), '', ['altsrc=']) - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self._get_usage(self.do_buildallfile)) - return - altsrc = None - for opt in opts: - if opt[0] == '--altsrc': - altsrc = opt[1] - if len(args) < 2: - print(self._get_usage(self.do_buildallfile)) - return - - destdir = args[0] - filename = args[1] - try: - os.mkdir(destdir) - except OSError: - err = sys.exc_info()[1] - if err.errno != 17: - print("Could not create %s: %s" % (destdir, err)) - if len(args) > 2: - clients = self._get_client_list(args[1:]) - else: - clients = self.metadata.clients - if altsrc: - args = "--altsrc %s -f %%s %%s %%s" % altsrc - else: - args = "-f %s %s %s" - for client in clients: - self.do_buildfile(args % (os.path.join(destdir, client), - filename, client)) - - def do_buildfile(self, args): - """ buildfile [-f <outfile>] [--altsrc=<altsrc>] <filename> - <hostname> - Build config file for hostname (not written to - disk)""" - try: - opts, alist = getopt.gnu_getopt(args.split(), 'f:', ['altsrc=']) - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self.do_buildfile.__doc__) - return - altsrc = None - outfile = None - for opt in opts: - if opt[0] == '--altsrc': - altsrc = opt[1] - elif opt[0] == '-f': - outfile = opt[1] - if len(alist) != 2: - print(self.do_buildfile.__doc__) - return - - fname, client = alist - entry = lxml.etree.Element('Path', type='file', name=fname) - if altsrc: - entry.set("altsrc", altsrc) - try: - metadata = self.build_metadata(client) - self.Bind(entry, metadata) - data = lxml.etree.tostring(entry, - xml_declaration=False).decode('UTF-8') - except Exception: - print("Failed to build entry %s for host %s: %s" % - (fname, client, traceback.format_exc().splitlines()[-1])) - raise - try: - if outfile: - open(outfile, 'w').write(data) - else: - print(data) - except IOError: - err = sys.exc_info()[1] - print("Could not write to %s: %s" % (outfile, err)) - print(data) - - def do_buildbundle(self, args): - """ buildbundle <bundle> <hostname> - Render a templated - bundle for hostname (not written to disk) """ - if len(args.split()) != 2: - print(self._get_usage(self.do_buildbundle)) - return 1 - - bname, client = args.split() - try: - metadata = self.build_metadata(client) - bundle = self.plugins['Bundler'].entries[bname] - print(lxml.etree.tostring(bundle.get_xml_value(metadata), - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - except KeyError: - print("No such bundle %s" % bname) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - print("Failed to render bundle %s for host %s: %s" % (bname, - client, - err)) - - def do_automatch(self, args): - """ automatch [-f] <propertyfile> <hostname> - Perform automatch on - a Properties file """ - alist = args.split() - force = False - for arg in alist: - if arg == '-f': - alist.remove('-f') - force = True - if len(alist) != 2: - print(self._get_usage(self.do_automatch)) - return - - if 'Properties' not in self.plugins: - print("Properties plugin not enabled") - return - - pname, client = alist - automatch = self.setup.cfp.getboolean("properties", "automatch", - default=False) - pfile = self.plugins['Properties'].entries[pname] - if (not force and not automatch and - pfile.xdata.get("automatch", "false").lower() != "true"): - print("Automatch not enabled on %s" % pname) - else: - metadata = self.build_metadata(client) - print(lxml.etree.tostring(pfile.XMLMatch(metadata), - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - - def do_bundles(self, _): - """ bundles - Print out group/bundle info """ - data = [('Group', 'Bundles')] - groups = list(self.metadata.groups.keys()) - groups.sort() - for group in groups: - data.append((group, - ','.join(self.metadata.groups[group][0]))) - print_tabular(data) - - def do_clients(self, _): - """ clients - Print out client/profile info """ - data = [('Client', 'Profile')] - for client in sorted(self.metadata.list_clients()): - imd = self.metadata.get_initial_metadata(client) - data.append((client, imd.profile)) - print_tabular(data) - - def do_config(self, _): - """ config - Print out the current configuration of Bcfg2""" - output = [ - ('Description', 'Value'), - ('Path Bcfg2 repository', self.setup['repo']), - ('Plugins', self.setup['plugins']), - ('Password', self.setup['password']), - ('Filemonitor', self.setup['filemonitor']), - ('Server address', self.setup['location']), - ('Path to key', self.setup['key']), - ('Path to SSL certificate', self.setup['cert']), - ('Path to SSL CA certificate', self.setup['ca']), - ('Protocol', self.setup['protocol']), - ('Logging', self.setup['logging'])] - print_tabular(output) - - def do_expirecache(self, args): - """ expirecache [<hostname> [<hostname> ...]]- Expire the - metadata cache """ - alist = args.split() - if len(alist): - for client in self._get_client_list(alist): - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, - key=client) - else: - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) - - def do_probes(self, args): - """ probes [-p] <hostname> - Get probe list for the given - host, in XML (the default) or human-readable pretty (with -p) - format""" - alist = args.split() - pretty = False - if '-p' in alist: - pretty = True - alist.remove('-p') - if len(alist) != 1: - print(self._get_usage(self.do_probes)) - return - hostname = alist[0] - if pretty: - probes = [] - else: - probes = lxml.etree.Element('probes') - metadata = self.build_metadata(hostname) - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): - for probe in plugin.GetProbes(metadata): - probes.append(probe) - if pretty: - for probe in probes: - pname = probe.get("name") - print("=" * (len(pname) + 2)) - print(" %s" % pname) - print("=" * (len(pname) + 2)) - print("") - print(probe.text) - print("") - else: - print(lxml.etree.tostring(probes, - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - - def do_showentries(self, args): - """ showentries <hostname> <type> - Show abstract - configuration entries for a given host """ - arglen = len(args.split()) - if arglen not in [1, 2]: - print(self._get_usage(self.do_showentries)) - return - client = args.split()[0] - try: - meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Unable to find metadata for host %s" % client) - return - structures = self.GetStructures(meta) - output = [('entrytype', 'name')] - if arglen == 1: - for item in structures: - for child in item.getchildren(): - output.append((child.tag, child.get('name'))) - if arglen == 2: - etype = args.split()[1] - for item in structures: - for child in item.getchildren(): - if child.tag in [etype, "Bound%s" % etype]: - output.append((child.tag, child.get('name'))) - print_tabular(output) - - def do_groups(self, _): - """ groups - Print out group info """ - data = [("Groups", "Profile", "Category")] - grouplist = list(self.metadata.groups.keys()) - grouplist.sort() - for group in grouplist: - if self.metadata.groups[group].is_profile: - prof = 'yes' - else: - prof = 'no' - cat = self.metadata.groups[group].category - data.append((group, prof, cat)) - print_tabular(data) - - def do_showclient(self, args): - """ showclient <client> [<client> ...] - Show metadata for the - given hosts """ - if not len(args): - print(self._get_usage(self.do_showclient)) - return - for client in args.split(): - try: - client_meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Client %s not defined" % client) - continue - fmt = "%-10s %s" - print(fmt % ("Hostname:", client_meta.hostname)) - print(fmt % ("Profile:", client_meta.profile)) - - group_fmt = "%-10s %-30s %s" - header = False - for group in list(client_meta.groups): - category = "" - for cat, grp in client_meta.categories.items(): - if grp == group: - category = "Category: %s" % cat - break - if not header: - print(group_fmt % ("Groups:", group, category)) - header = True - else: - print(group_fmt % ("", group, category)) - - if client_meta.bundles: - print(fmt % ("Bundles:", list(client_meta.bundles)[0])) - for bnd in list(client_meta.bundles)[1:]: - print(fmt % ("", bnd)) - if client_meta.connectors: - print("Connector data") - print("=" * 80) - for conn in client_meta.connectors: - if getattr(client_meta, conn): - print(fmt % (conn + ":", getattr(client_meta, conn))) - print("=" * 80) - - def do_mappings(self, args): - """ mappings <type*> <name*> - Print generator mappings for - optional type and name """ - # Dump all mappings unless type specified - data = [('Plugin', 'Type', 'Name')] - arglen = len(args.split()) - for generator in self.plugins_by_type(Bcfg2.Server.Plugin.Generator): - if arglen == 0: - etypes = list(generator.Entries.keys()) - else: - etypes = [args.split()[0]] - if arglen == 2: - interested = [(etype, [args.split()[1]]) - for etype in etypes] - else: - interested = [(etype, generator.Entries[etype]) - for etype in etypes - if etype in generator.Entries] - for etype, names in interested: - for name in [name for name in names if name in - generator.Entries.get(etype, {})]: - data.append((generator.name, etype, name)) - print_tabular(data) - - def do_event_debug(self, _): - """ event_debug - Display filesystem events as they are - processed """ - self.fam.debug = True - - def do_packageresolve(self, args): - """ packageresolve <hostname> [<package> [<package>...]] - - Resolve packages for the given host, optionally specifying a - set of packages """ - arglist = args.split(" ") - if len(arglist) < 1: - print(self._get_usage(self.do_packageresolve)) - return - - try: - pkgs = self.plugins['Packages'] - except KeyError: - print("Packages plugin not enabled") - return - pkgs.toggle_debug() - - hostname = arglist[0] - metadata = self.build_metadata(hostname) - - indep = lxml.etree.Element("Independent") - if len(arglist) > 1: - structures = [lxml.etree.Element("Bundle", name="packages")] - for arg in arglist[1:]: - lxml.etree.SubElement(structures[0], "Package", name=arg) - else: - structures = self.GetStructures(metadata) - - pkgs._build_packages(metadata, indep, # pylint: disable=W0212 - structures) - print("%d new packages added" % len(indep.getchildren())) - if len(indep.getchildren()): - print(" %s" % "\n ".join(lxml.etree.tostring(p) - for p in indep.getchildren())) - - def do_packagesources(self, args): - """ packagesources <hostname> - Show package sources """ - if not args: - print(self._get_usage(self.do_packagesources)) - return - if 'Packages' not in self.plugins: - print("Packages plugin not enabled") - return - try: - metadata = self.build_metadata(args) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Unable to build metadata for host %s" % args) - return - collection = self.plugins['Packages'].get_collection(metadata) - print(collection.sourcelist()) - - def do_query(self, args): - """ query <-g group|-p profile|-b bundle> - Query clients """ - if len(args) == 0: - print("\n".join(self.metadata.clients)) - return - arglist = args.split(" ") - if len(arglist) != 2: - print(self._get_usage(self.do_query)) - return - - qtype, qparam = arglist - if qtype == '-p': - res = self.metadata.get_client_names_by_profiles(qparam.split(',')) - elif qtype == '-g': - res = self.metadata.get_client_names_by_groups(qparam.split(',')) - elif qtype == '-b': - res = self.metadata.get_client_names_by_bundles(qparam.split(',')) - else: - print(self._get_usage(self.do_query)) - return - print("\n".join(res)) - - def do_profile(self, arg): - """ profile <command> <args> - Profile a single bcfg2-info - command """ - if not HAS_PROFILE: - print("Profiling functionality not available.") - return - if len(arg) == 0: - print(self._get_usage(self.do_profile)) - return - prof = profile.Profile() - prof.runcall(self.onecmd, arg) - display_trace(prof) - - def run(self, args): # pylint: disable=W0221 - try: - self.load_plugins() - self.block_for_fam_events(handle_events=True) - if args: - self.onecmd(" ".join(args)) - else: - self.do_loop() - finally: - self.shutdown() - - def _daemonize(self): - pass - - def _run(self): - pass - - def _block(self): - pass - - -def build_usage(): - """ build usage message """ - cmd_blacklist = ["do_loop", "do_EOF"] - usage = dict() - for attrname in dir(InfoCore): - attr = getattr(InfoCore, attrname) - - # shim for python 2.4, __func__ is im_func - funcattr = getattr(attr, "__func__", getattr(attr, "im_func", None)) - if (funcattr is not None and - funcattr.func_name not in cmd_blacklist and - funcattr.func_name.startswith("do_") and - funcattr.func_doc): - usage[attr.__name__] = re.sub(r'\s+', ' ', attr.__doc__) - return "Commands:\n" + "\n".join(usage[k] for k in sorted(usage.keys())) - - -USAGE = build_usage() - - -def main(): - optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, - interactive=Bcfg2.Options.INTERACTIVE, - interpreter=Bcfg2.Options.INTERPRETER, - command_timeout=Bcfg2.Options.CLIENT_COMMAND_TIMEOUT) - optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.hm = "\n".join([" bcfg2-info [options] [command <command args>]", - "Options:", - setup.buildHelpMessage(), - USAGE]) - - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-info', to_syslog=False, level=level) - - if setup['args'] and setup['args'][0] == 'help': - print(setup.hm) - sys.exit(0) - elif setup['profile'] and HAS_PROFILE: - prof = profile.Profile() - loop = prof.runcall(InfoCore, setup) - display_trace(prof) - else: - if setup['profile']: - print("Profiling functionality not available.") - loop = InfoCore(setup) - - loop.run(setup['args']) - +from Bcfg2.Server.Info import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index bb96c2023..e818dc3be 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -1,211 +1,8 @@ #!/usr/bin/env python - """This tool examines your Bcfg2 specifications for errors.""" import sys -import time -import logging -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Core -import Bcfg2.Server.Lint - -LOGGER = logging.getLogger('bcfg2-lint') - - -def run_serverless_plugins(plugins, setup=None, errorhandler=None, files=None): - """ Run serverless plugins """ - LOGGER.debug("Running serverless plugins") - for plugin_name, plugin in list(plugins.items()): - run_plugin(plugin, plugin_name, errorhandler=errorhandler, - setup=setup, files=files) - - -def run_server_plugins(plugins, setup=None, errorhandler=None, files=None): - """ run plugins that require a running server to run """ - core = load_server(setup) - try: - LOGGER.debug("Running server plugins") - for plugin_name, plugin in list(plugins.items()): - run_plugin(plugin, plugin_name, args=[core], - errorhandler=errorhandler, setup=setup, files=files) - finally: - core.shutdown() - - -def run_plugin(plugin, plugin_name, setup=None, errorhandler=None, - args=None, files=None): - """ run a single plugin, server-ful or serverless. """ - LOGGER.debug(" Running %s" % plugin_name) - if args is None: - args = [] - - if errorhandler is None: - errorhandler = get_errorhandler(setup) - - if setup is not None and setup.cfp.has_section(plugin_name): - arg = setup - for key, val in setup.cfp.items(plugin_name): - arg[key] = val - args.append(arg) - else: - args.append(setup) - - # python 2.5 doesn't support mixing *magic and keyword arguments - start = time.time() - rv = plugin(*args, **dict(files=files, errorhandler=errorhandler)).Run() - LOGGER.debug(" Ran %s in %0.2f seconds" % (plugin_name, - time.time() - start)) - return rv - - -def get_errorhandler(setup): - """ get a Bcfg2.Server.Lint.ErrorHandler object """ - if setup.cfp.has_section("errors"): - errors = dict(setup.cfp.items("errors")) - else: - errors = None - return Bcfg2.Server.Lint.ErrorHandler(errors=errors) - - -def load_server(setup): - """ load server """ - core = Bcfg2.Server.Core.BaseCore(setup) - core.load_plugins() - core.block_for_fam_events(handle_events=True) - return core - - -def load_plugin(module, obj_name=None): - """ load a single plugin """ - parts = module.split(".") - if obj_name is None: - obj_name = parts[-1] - - mod = __import__(module) - for part in parts[1:]: - mod = getattr(mod, part) - return getattr(mod, obj_name) - - -def load_plugins(setup): - """ get list of plugins to run """ - if setup['args']: - plugin_list = setup['args'] - elif "bcfg2-repo-validate" in sys.argv[0]: - plugin_list = 'RequiredAttrs,Validate'.split(',') - elif setup['lint_plugins']: - plugin_list = setup['lint_plugins'] - else: - plugin_list = Bcfg2.Server.Lint.plugins - - allplugins = dict() - for plugin in plugin_list: - try: - allplugins[plugin] = load_plugin("Bcfg2.Server.Lint." + plugin) - except ImportError: - try: - allplugins[plugin] = \ - load_plugin("Bcfg2.Server.Plugins." + plugin, - obj_name=plugin + "Lint") - except (ImportError, AttributeError): - err = sys.exc_info()[1] - LOGGER.error("Failed to load plugin %s: %s" % - (plugin + "Lint", err)) - except AttributeError: - err = sys.exc_info()[1] - LOGGER.error("Failed to load plugin %s: %s" % (plugin, err)) - - for plugin in setup['plugins']: - if plugin in allplugins: - # already loaded - continue - - try: - allplugins[plugin] = \ - load_plugin("Bcfg2.Server.Plugins." + plugin, - obj_name=plugin + "Lint") - except AttributeError: - pass - except ImportError: - err = sys.exc_info()[1] - LOGGER.error("Failed to load plugin %s: %s" % (plugin + "Lint", - err)) - - serverplugins = dict() - serverlessplugins = dict() - for plugin_name, plugin in allplugins.items(): - if issubclass(plugin, Bcfg2.Server.Lint.ServerPlugin): - serverplugins[plugin_name] = plugin - else: - serverlessplugins[plugin_name] = plugin - return (serverlessplugins, serverplugins) - - -def main(): - optinfo = dict(lint_config=Bcfg2.Options.LINT_CONFIG, - showerrors=Bcfg2.Options.LINT_SHOW_ERRORS, - stdin=Bcfg2.Options.LINT_FILES_ON_STDIN, - schema=Bcfg2.Options.SCHEMA_PATH, - lint_plugins=Bcfg2.Options.LINT_PLUGINS) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - log_args = dict(to_syslog=setup['syslog'], to_console=logging.WARNING) - if setup['verbose']: - log_args['to_console'] = logging.DEBUG - Bcfg2.Logger.setup_logging('bcfg2-info', **log_args) - - setup.cfp.read(setup['lint_config']) - setup.reparse() - - if setup['stdin']: - files = [s.strip() for s in sys.stdin.readlines()] - else: - files = None - - (serverlessplugins, serverplugins) = load_plugins(setup) - - errorhandler = get_errorhandler(setup) - - if setup['showerrors']: - for plugin in serverplugins.values() + serverlessplugins.values(): - errorhandler.RegisterErrors(getattr(plugin, 'Errors')()) - - print("%-35s %-35s" % ("Error name", "Handler")) - for err, handler in errorhandler.errortypes.items(): - print("%-35s %-35s" % (err, handler.__name__)) - raise SystemExit(0) - - run_serverless_plugins(serverlessplugins, errorhandler=errorhandler, - setup=setup, files=files) - - if serverplugins: - if errorhandler.errors: - # it would be swell if we could try to start the server - # even if there were errors with the serverless plugins, - # but since XML parsing errors occur in the FAM thread - # (not in the core server thread), there's no way we can - # start the server and try to catch exceptions -- - # bcfg2-lint isn't in the same stack as the exceptions. - # so we're forced to assume that a serverless plugin error - # will prevent the server from starting - print("Serverless plugins encountered errors, skipping server " - "plugins") - else: - run_server_plugins(serverplugins, errorhandler=errorhandler, - setup=setup, files=files) - - if errorhandler.errors or errorhandler.warnings or setup['verbose']: - print("%d errors" % errorhandler.errors) - print("%d warnings" % errorhandler.warnings) - - if errorhandler.errors: - raise SystemExit(2) - elif errorhandler.warnings: - raise SystemExit(3) +from Bcfg2.Server.Lint import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-repo-validate b/src/sbin/bcfg2-repo-validate deleted file mode 120000 index cea09cda3..000000000 --- a/src/sbin/bcfg2-repo-validate +++ /dev/null @@ -1 +0,0 @@ -bcfg2-lint
\ No newline at end of file diff --git a/src/sbin/bcfg2-report-collector b/src/sbin/bcfg2-report-collector index 594be13bf..f7d922d13 100755 --- a/src/sbin/bcfg2-report-collector +++ b/src/sbin/bcfg2-report-collector @@ -7,24 +7,19 @@ import sys import logging import Bcfg2.Logger import Bcfg2.Options +import Bcfg2.DBSettings from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError def main(): + parser = Bcfg2.Options.get_parser(description="Collect Bcfg2 report data", + components=[ReportingCollector]) + parser.parse() logger = logging.getLogger('bcfg2-report-collector') - optinfo = dict(daemon=Bcfg2.Options.DAEMON, - repo=Bcfg2.Options.SERVER_REPOSITORY, - filemonitor=Bcfg2.Options.SERVER_FILEMONITOR, - web_configfile=Bcfg2.Options.WEB_CFILE) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.REPORTING_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) # run collector try: - collector = ReportingCollector(setup) - collector.run() + ReportingCollector().run() except ReportingError: msg = sys.exc_info()[1] logger.error(msg) diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 2a8447ae4..5a3f3784d 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -1,301 +1,8 @@ #!/usr/bin/env python """Query reporting system for client status.""" -import os import sys -import datetime -from optparse import OptionParser, OptionGroup, make_option -from Bcfg2.Compat import ConfigParser +from Bcfg2.Reporting.Reports import CLI -try: - import Bcfg2.settings -except ConfigParser.NoSectionError: - print("Your bcfg2.conf is currently missing the [database] section which " - "is necessary for the reporting interface. Please see bcfg2.conf(5) " - "for more details.") - sys.exit(1) - -project_directory = os.path.dirname(Bcfg2.settings.__file__) -project_name = os.path.basename(project_directory) -sys.path.append(os.path.join(project_directory, '..')) -project_module = __import__(project_name, '', '', ['']) -sys.path.pop() -# Set DJANGO_SETTINGS_MODULE appropriately. -os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name - -from Bcfg2.Reporting.models import (Client, BaseEntry) -from django import db - -def hosts_by_entry_type(clients, etype, entryspec): - result = [] - for entry in entryspec: - for client in clients: - items = getattr(client.current_interaction, etype)() - for item in items: - if (item.entry_type == entry[0] and - item.name == entry[1]): - result.append(client) - return result - -def print_fields(fields, client, fmt, extra=None): - """ - Prints the fields specified in fields of client, max_name - specifies the column width of the name column. - """ - fdata = [] - if extra is None: - extra = dict() - for field in fields: - if field == 'time': - fdata.append(str(client.current_interaction.timestamp)) - elif field == 'state': - if client.current_interaction.isclean(): - fdata.append("clean") - else: - fdata.append("dirty") - elif field == 'total': - fdata.append(client.current_interaction.total_count) - elif field == 'good': - fdata.append(client.current_interaction.good_count) - elif field == 'modified': - fdata.append(client.current_interaction.modified_count) - elif field == 'extra': - fdata.append(client.current_interaction.extra_count) - elif field == 'bad': - fdata.append(client.current_interaction.bad_count) - else: - try: - fdata.append(getattr(client, field)) - except: - fdata.append(extra.get(field, "N/A")) - - print(fmt % tuple(fdata)) - -def print_entries(interaction, etype): - items = getattr(interaction, etype)() - for item in items: - print("%-70s %s" % (item.entry_type + ":" + item.name, etype)) - -def main(): - parser = OptionParser(usage="%prog [options] <mode> [arg]") - - # single host modes - multimodes = [] - singlemodes = [] - multimodes.append(make_option("-b", "--bad", action="store_true", - default=False, - help="Show bad entries from HOST")) - multimodes.append(make_option("-e", "--extra", action="store_true", - default=False, - help="Show extra entries from HOST")) - multimodes.append(make_option("-m", "--modified", action="store_true", - default=False, - help="Show modified entries from HOST")) - multimodes.append(make_option("-s", "--show", action="store_true", - default=False, - help="Equivalent to --bad --extra --modified")) - singlemodes.append(make_option("-t", "--total", action="store_true", - default=False, - help="Show total number of managed and good " - "entries from HOST")) - singlemodes.append(make_option("-x", "--expire", action="store_true", - default=False, - help="Toggle expired/unexpired state of " - "HOST")) - hostmodes = \ - OptionGroup(parser, "Single-Host Modes", - "The following mode flags require a single HOST argument") - hostmodes.add_options(multimodes) - hostmodes.add_options(singlemodes) - parser.add_option_group(hostmodes) - - # all host modes - allhostmodes = OptionGroup(parser, "Host Selection Modes", - "The following mode flags require no arguments") - allhostmodes.add_option("-a", "--all", action="store_true", default=False, - help="Show all hosts, including expired hosts") - allhostmodes.add_option("-c", "--clean", action="store_true", default=False, - help="Show only clean hosts") - allhostmodes.add_option("-d", "--dirty", action="store_true", default=False, - help="Show only dirty hosts") - allhostmodes.add_option("--stale", action="store_true", default=False, - help="Show hosts that haven't run in the last 24 " - "hours") - parser.add_option_group(allhostmodes) - - # entry modes - entrymodes = \ - OptionGroup(parser, "Entry Modes", - "The following mode flags require either any number of " - "TYPE:NAME arguments describing entries, or the --file " - "option") - entrymodes.add_option("--badentry", action="store_true", default=False, - help="Show hosts that have bad entries that match " - "the argument") - entrymodes.add_option("--modifiedentry", action="store_true", default=False, - help="Show hosts that have modified entries that " - "match the argument") - entrymodes.add_option("--extraentry", action="store_true", default=False, - help="Show hosts that have extra entries that match " - "the argument") - entrymodes.add_option("--entrystatus", action="store_true", default=False, - help="Show the status of the named entry on all " - "hosts. Only supports a single entry.") - parser.add_option_group(entrymodes) - - # entry options - entryopts = OptionGroup(parser, "Entry Options", - "Options that can be used with entry modes") - entryopts.add_option("--fields", metavar="FIELD,FIELD,...", - help="Only display the listed fields", - default='name,time,state') - entryopts.add_option("--file", metavar="FILE", - help="Read TYPE:NAME pairs from the specified file " - "instead of the command line") - parser.add_option_group(entryopts) - - options, args = parser.parse_args() - - # make sure we've specified exactly one mode - mode_family = None - mode = None - for opt in allhostmodes.option_list + entrymodes.option_list + \ - singlemodes: - if getattr(options, opt.dest): - if mode is not None: - parser.error("Only one mode can be specified; found %s and %s" % - (mode.get_opt_string(), opt.get_opt_string())) - mode = opt - mode_family = parser.get_option_group(opt.get_opt_string()) - - # you can specify more than one of --bad, --extra, --modified, --show, so - # consider single-host options separately - if not mode_family: - for opt in multimodes: - if getattr(options, opt.dest): - mode_family = parser.get_option_group(opt.get_opt_string()) - break - - if not mode_family: - parser.error("You must specify a mode") - - if mode_family == hostmodes: - try: - cname = args.pop() - client = Client.objects.select_related().get(name=cname) - except IndexError: - parser.error("%s require a single HOST argument" % hostmodes.title) - except Client.DoesNotExist: - print("No such host: %s" % cname) - return 2 - - if options.expire: - if client.expiration == None: - client.expiration = datetime.datetime.now() - print("Host expired.") - else: - client.expiration = None - print("Host un-expired.") - client.save() - elif options.total: - managed = client.current_interaction.total_count - good = client.current_interaction.good_count - print("Total managed entries: %d (good: %d)" % (managed, good)) - elif mode_family == hostmodes: - if options.bad or options.show: - print_entries(client.current_interaction, "bad") - - if options.modified or options.show: - print_entries(client.current_interaction, "modified") - - if options.extra or options.show: - print_entries(client.current_interaction, "extra") - else: - clients = Client.objects.exclude(current_interaction__isnull=True) - result = list() - edata = dict() - fields = options.fields.split(',') - - if mode_family == allhostmodes: - if args: - print("%s do not take any arguments, ignoring" % - allhostmodes.title) - - for client in clients: - interaction = client.current_interaction - if (options.all or - (options.stale and interaction.isstale()) or - (options.clean and interaction.isclean()) or - (options.dirty and not interaction.isclean())): - result.append(client) - else: - # entry query modes - if options.file: - try: - entries = [l.strip().split(":") - for l in open(options.file)] - except IOError: - err = sys.exc_info()[1] - print("Cannot read entries from %s: %s" % (options.file, - err)) - return 2 - elif args: - entries = [a.split(":") for a in args] - else: - parser.error("%s require either a list of entries on the " - "command line or the --file options" % - mode_family.title) - - if options.badentry: - result = hosts_by_entry_type(clients, "bad", entries) - elif options.modifiedentry: - result = hosts_by_entry_type(clients, "modified", entries) - elif options.extraentry: - result = hosts_by_entry_type(clients, "extra", entries) - elif options.entrystatus: - if 'state' in fields: - fields.remove('state') - fields.append("entry state") - - try: - entry_cls = BaseEntry.entry_from_type(entries[0][0]) - except ValueError: - print("Unhandled/unkown type %s" % entries[0][0]) - return 2 - - # todo batch fetch this. sqlite could break - for client in clients: - ents = entry_cls.objects.filter(name=entries[0][1], - interaction=client.current_interaction) - if len(ents) == 0: - continue - edata[client] = {"entry state": ents[0].get_state_display(), - "reason": ents[0]} - result.append(client) - - - if 'name' not in fields: - fields.insert(0, "name") - if not result: - print("No match found") - return - max_name = max(len(c.name) for c in result) - ffmt = [] - for field in fields: - if field == "name": - ffmt.append("%%-%ds" % max_name) - elif field == "time": - ffmt.append("%-19s") - else: - ffmt.append("%%-%ds" % len(field)) - fmt = " ".join(ffmt) - print(fmt % tuple(f.title() for f in fields)) - for client in result: - if not client.expiration: - print_fields(fields, client, fmt, - extra=edata.get(client, None)) - db.close_connection() - - -if __name__ == "__main__": - sys.exit(main()) +if __name__ == '__main__': + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index 4c4a71fa7..00b4e92e3 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -2,63 +2,48 @@ """The XML-RPC Bcfg2 server.""" -import os import sys import logging -import Bcfg2.Logger import Bcfg2.Options from Bcfg2.Server.Core import CoreInitError -LOGGER = logging.getLogger('bcfg2-server') - -def main(): - optinfo = dict() - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.DAEMON_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - # check whether the specified bcfg2.conf exists - if not os.path.exists(setup['configfile']): - print("Could not read %s" % setup['configfile']) - sys.exit(1) - - # TODO: normalize case of various core modules so we can add a new - # core without modifying this script - backends = dict(cherrypy='CherryPyCore', - builtin='BuiltinCore', - best='BuiltinCore', - multiprocessing='MultiprocessingCore') - - if setup['backend'] not in backends: - print("Unknown server backend %s, using 'best'" % setup['backend']) - setup['backend'] = 'best' - - coremodule = backends[setup['backend']] - try: - corecls = getattr(__import__("Bcfg2.Server.%s" % coremodule).Server, - coremodule).Core - except ImportError: - err = sys.exc_info()[1] - print("Unable to import %s server core: %s" % (setup['backend'], err)) - raise - except AttributeError: - err = sys.exc_info()[1] - print("Unable to load %s server core: %s" % (setup['backend'], err)) - raise - - try: - core = corecls(setup) - core.run() - except CoreInitError: - msg = sys.exc_info()[1] - LOGGER.error(msg) - sys.exit(1) - except KeyboardInterrupt: - sys.exit(1) - sys.exit(0) +class BackendAction(Bcfg2.Options.ComponentAction): + """ Action to load Bcfg2 backends """ + islist = False + bases = ['Bcfg2.Server'] + + +class CLI(object): + """ bcfg2-server CLI class """ + parse_first = True + options = [ + Bcfg2.Options.Option( + cf=('server', 'backend'), help='Server Backend', + default='BuiltinCore', type=lambda b: b.title() + "Core", + action=BackendAction)] + + def __init__(self): + parser = Bcfg2.Options.get_parser("Bcfg2 server", components=[self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + def run(self): + """ Run the bcfg2 server """ + try: + core = Bcfg2.Options.setup.backend() + core.run() + except CoreInitError: + self.logger.error(sys.exc_info()[1]) + return 1 + except TypeError: + self.logger.error("Failed to load %s server backend: %s" % + (Bcfg2.Options.setup.backend.__name__, + sys.exc_info()[1])) + raise + except KeyboardInterrupt: + return 1 if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 7c38a65d8..73d9f13a7 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -1,318 +1,9 @@ #!/usr/bin/env python +""" This tool verifies that all clients known to the server build +without failures """ -"""This tool verifies that all clients known to the server build -without failures""" - -import os import sys -import signal -import fnmatch -import logging -import Bcfg2.Logger -import Bcfg2.Server.Core -from math import ceil -from nose.core import TestProgram -from nose.suite import LazySuite -from unittest import TestCase - -try: - from multiprocessing import Process, Queue, active_children - HAS_MULTIPROC = True -except ImportError: - HAS_MULTIPROC = False - active_children = lambda: [] # pylint: disable=C0103 - - -class CapturingLogger(object): - """ Fake logger that captures logging output so that errors are - only displayed for clients that fail tests """ - def __init__(self, *args, **kwargs): # pylint: disable=W0613 - self.output = [] - - def error(self, msg): - """ discard error messages """ - self.output.append(msg) - - def warning(self, msg): - """ discard error messages """ - self.output.append(msg) - - def info(self, msg): - """ discard error messages """ - self.output.append(msg) - - def debug(self, msg): - """ discard error messages """ - self.output.append(msg) - - def reset_output(self): - """ Reset the captured output """ - self.output = [] - - -class ClientTestFromQueue(TestCase): - """ A test case that tests a value that has been enqueued by a - child test process. ``client`` is the name of the client that has - been tested; ``result`` is the result from the :class:`ClientTest` - test. ``None`` indicates a successful test; a string value - indicates a failed test; and an exception indicates an error while - running the test. """ - __test__ = False # Do not collect - - def __init__(self, client, result): - TestCase.__init__(self) - self.client = client - self.result = result - - def shortDescription(self): - return "Building configuration for %s" % self.client - - def runTest(self): - """ parse the result from this test """ - if isinstance(self.result, Exception): - raise self.result - assert self.result is None, self.result - - -class ClientTest(TestCase): - """ A test case representing the build of all of the configuration for - a single host. Checks that none of the build config entities has - had a failure when it is building. Optionally ignores some config - files that we know will cause errors (because they are private - files we don't have access to, for instance) """ - __test__ = False # Do not collect - divider = "-" * 70 - - def __init__(self, core, client, ignore=None): - TestCase.__init__(self) - self.core = core - self.core.logger = CapturingLogger() - self.client = client - if ignore is None: - self.ignore = dict() - else: - self.ignore = ignore - - def ignore_entry(self, tag, name): - """ return True if an error on a given entry should be ignored - """ - if tag in self.ignore: - if name in self.ignore[tag]: - return True - else: - # try wildcard matching - for pattern in self.ignore[tag]: - if fnmatch.fnmatch(name, pattern): - return True - return False - - def shortDescription(self): - return "Building configuration for %s" % self.client - - def runTest(self): - """ run this individual test """ - config = self.core.BuildConfiguration(self.client) - output = self.core.logger.output[:] - if output: - output.append(self.divider) - self.core.logger.reset_output() - - # check for empty client configuration - assert len(config.findall("Bundle")) > 0, \ - "\n".join(output + ["%s has no content" % self.client]) - - # check for missing bundles - metadata = self.core.build_metadata(self.client) - sbundles = [el.get('name') for el in config.findall("Bundle")] - missing = [b for b in metadata.bundles if b not in sbundles] - assert len(missing) == 0, \ - "\n".join(output + ["Configuration is missing bundle(s): %s" % - ':'.join(missing)]) - - # check for unknown packages - unknown_pkgs = [el.get("name") - for el in config.xpath('//Package[@type="unknown"]') - if not self.ignore_entry(el.tag, el.get("name"))] - assert len(unknown_pkgs) == 0, \ - "Configuration contains unknown packages: %s" % \ - ", ".join(unknown_pkgs) - - failures = [] - msg = output + ["Failures:"] - for failure in config.xpath('//*[@failure]'): - if not self.ignore_entry(failure.tag, failure.get('name')): - failures.append(failure) - msg.append("%s:%s: %s" % (failure.tag, failure.get("name"), - failure.get("failure"))) - - assert len(failures) == 0, "\n".join(msg) - - def __str__(self): - return "ClientTest(%s)" % self.client - - id = __str__ - - -def get_core(setup): - """ Get a server core, with events handled """ - core = Bcfg2.Server.Core.BaseCore(setup) - core.load_plugins() - core.block_for_fam_events(handle_events=True) - return core - - -def get_ignore(setup): - """ Given an options dict, get a dict of entry tags and names to - ignore errors from """ - ignore = dict() - for entry in setup['test_ignore']: - tag, name = entry.split(":") - try: - ignore[tag].append(name) - except KeyError: - ignore[tag] = [name] - return ignore - - -def run_child(setup, clients, queue): - """ Run tests for the given clients in a child process, returning - results via the given Queue """ - core = get_core(setup) - ignore = get_ignore(setup) - for client in clients: - try: - ClientTest(core, client, ignore).runTest() - queue.put((client, None)) - except AssertionError: - queue.put((client, str(sys.exc_info()[1]))) - except: - queue.put((client, sys.exc_info()[1])) - - core.shutdown() - - -def get_sigint_handler(core): - """ Get a function that handles SIGINT/Ctrl-C by shutting down the - core and exiting properly.""" - - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - core.shutdown() - os._exit(1) # pylint: disable=W0212 - - return hdlr - - -def parse_args(): - """ Parse command line arguments. """ - optinfo = dict(Bcfg2.Options.TEST_COMMON_OPTIONS) - - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.hm = \ - "bcfg2-test [options] [client] [client] [...]\nOptions:\n %s" % \ - setup.buildHelpMessage() - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging("bcfg2-test", - to_console=setup['verbose'] or setup['debug'], - to_syslog=False, - to_file=setup['logging'], - level=level) - logger = logging.getLogger(sys.argv[0]) - if (setup['debug'] or setup['verbose']) and "-v" not in setup['noseopts']: - setup['noseopts'].append("-v") - - if setup['children'] and not HAS_MULTIPROC: - logger.warning("Python multiprocessing library not found, running " - "with no children") - setup['children'] = 0 - - if (setup['children'] and ('--with-xunit' in setup['noseopts'] or - '--xunit-file' in setup['noseopts'])): - logger.warning("Use the --xunit option to bcfg2-test instead of the " - "--with-xunit or --xunit-file options to nosetest") - xunitfile = None - if '--with-xunit' in setup['noseopts']: - setup['noseopts'].remove('--with-xunit') - xunitfile = "nosetests.xml" - if '--xunit-file' in setup['noseopts']: - idx = setup['noseopts'].index('--xunit-file') - try: - setup['noseopts'].pop(idx) # remove --xunit-file - # remove the argument to it - xunitfile = setup['noseopts'].pop(idx) - except IndexError: - pass - if xunitfile and not setup['xunit']: - setup['xunit'] = xunitfile - return setup - - -def main(): - setup = parse_args() - logger = logging.getLogger(sys.argv[0]) - core = get_core(setup) - signal.signal(signal.SIGINT, get_sigint_handler(core)) - - if setup['args']: - clients = setup['args'] - else: - clients = core.metadata.clients - - ignore = get_ignore(setup) - - if setup['children']: - if setup['children'] > len(clients): - logger.info("Refusing to spawn more children than clients to test," - " setting children=%s" % len(clients)) - setup['children'] = len(clients) - perchild = int(ceil(len(clients) / float(setup['children'] + 1))) - queue = Queue() - for child in range(setup['children']): - start = child * perchild - end = (child + 1) * perchild - child = Process(target=run_child, - args=(setup, clients[start:end], queue)) - child.start() - - def generate_tests(): - """ Read test results for the clients """ - start = setup['children'] * perchild - for client in clients[start:]: - yield ClientTest(core, client, ignore) - - for i in range(start): # pylint: disable=W0612 - yield ClientTestFromQueue(*queue.get()) - else: - def generate_tests(): - """ Run tests for the clients """ - for client in clients: - yield ClientTest(core, client, ignore) - - result = TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], - suite=LazySuite(generate_tests), exit=False) - - # block until all children have completed -- should be - # immediate since we've already gotten all the results we - # expect - for child in active_children(): - child.join() - - core.shutdown() - if result.success: - os._exit(0) # pylint: disable=W0212 - else: - os._exit(1) # pylint: disable=W0212 - +from Bcfg2.Server.Test import CLI if __name__ == "__main__": - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index f2bdf2c97..95fb9889e 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -5,370 +5,8 @@ the right way to get around that in long-running processes it to have a short-lived helper. No, seriously -- check out the yum-updatesd code. It's pure madness. """ -import os import sys -import yum -import logging -import Bcfg2.Logger -from Bcfg2.Compat import wraps -from lockfile import FileLock, LockTimeout -from optparse import OptionParser -try: - import json - # py2.4 json library is structured differently - json.loads # pylint: disable=W0104 -except (ImportError, AttributeError): - import simplejson as json - - -def pkg_to_tuple(package): - """ json doesn't distinguish between tuples and lists, but yum - does, so we convert a package in list format to one in tuple - format """ - if isinstance(package, list): - return tuple(package) - else: - return package - - -def pkgtup_to_string(package): - """ given a package tuple, return a human-readable string - describing the package """ - if package[3] in ['auto', 'any']: - return package[0] - - rv = [package[0], "-"] - if package[2]: - rv.extend([package[2], ':']) - rv.extend([package[3], '-', package[4]]) - if package[1]: - rv.extend(['.', package[1]]) - return ''.join(str(e) for e in rv) - - -class YumHelper(object): - """ Yum helper base object """ - - def __init__(self, cfgfile, verbose=1): - self.cfgfile = cfgfile - self.yumbase = yum.YumBase() - # pylint: disable=E1121,W0212 - try: - self.yumbase.preconf.debuglevel = verbose - self.yumbase.preconf.fn = cfgfile - self.yumbase._getConfig() - except AttributeError: - self.yumbase._getConfig(cfgfile, debuglevel=verbose) - # pylint: enable=E1121,W0212 - self.logger = logging.getLogger(self.__class__.__name__) - - def setarch(self, arch): - """ Configure an arch other than the bcfg2 server arch for dep - resolution. """ - self.yumbase.arch.setup_arch(arch=arch) - - -class DepSolver(YumHelper): - """ Yum dependency solver. This is used for operations that only - read from the yum cache, and thus operates in cacheonly mode. """ - - def __init__(self, cfgfile, verbose=1): - YumHelper.__init__(self, cfgfile, verbose=verbose) - # internally, yum uses an integer, not a boolean, for conf.cache - self.yumbase.conf.cache = 1 - self._groups = None - - def get_groups(self): - """ getter for the groups property """ - if self._groups is not None: - return self._groups - else: - return ["noarch"] - - def set_groups(self, groups): - """ setter for the groups property """ - self._groups = set(groups).union(["noarch"]) - - groups = property(get_groups, set_groups) - - def get_package_object(self, pkgtup, silent=False): - """ given a package tuple, get a yum package object """ - try: - matches = yum.packageSack.packagesNewestByName( - self.yumbase.pkgSack.searchPkgTuple(pkgtup)) - except yum.Errors.PackageSackError: - if not silent: - self.logger.warning("Package '%s' not found" % - self.get_package_name(pkgtup)) - matches = [] - except yum.Errors.RepoError: - err = sys.exc_info()[1] - self.logger.error("Temporary failure loading metadata for %s: %s" % - (self.get_package_name(pkgtup), err)) - matches = [] - - pkgs = self._filter_arch(matches) - if pkgs: - return pkgs[0] - else: - return None - - def get_group(self, group, ptype="default"): - """ Resolve a package group name into a list of packages """ - if group.startswith("@"): - group = group[1:] - - try: - if self.yumbase.comps.has_group(group): - group = self.yumbase.comps.return_group(group) - else: - self.logger.error("%s is not a valid group" % group) - return [] - except yum.Errors.GroupsError: - err = sys.exc_info()[1] - self.logger.warning(err) - return [] - - if ptype == "default": - return [p - for p, d in list(group.default_packages.items()) - if d] - elif ptype == "mandatory": - return [p - for p, m in list(group.mandatory_packages.items()) - if m] - elif ptype == "optional" or ptype == "all": - return group.packages - else: - self.logger.warning("Unknown group package type '%s'" % ptype) - return [] - - def _filter_arch(self, packages): - """ filter packages in the given list that do not have an - architecture in the list of groups for this client """ - matching = [] - for pkg in packages: - if pkg.arch in self.groups: - matching.append(pkg) - else: - self.logger.debug("%s has non-matching architecture (%s)" % - (pkg, pkg.arch)) - if matching: - return matching - else: - # no packages match architecture; we'll assume that the - # user knows what s/he is doing and this is a multiarch - # box. - return packages - - def get_package_name(self, package): - """ get the name of a package or virtual package from the - internal representation used by this Collection class """ - if isinstance(package, tuple): - if len(package) == 3: - return yum.misc.prco_tuple_to_string(package) - else: - return pkgtup_to_string(package) - else: - return str(package) - - def complete(self, packagelist): - """ resolve dependencies and generate a complete package list - from the given list of initial packages """ - packages = set() - unknown = set() - for pkg in packagelist: - if isinstance(pkg, tuple): - pkgtup = pkg - else: - pkgtup = (pkg, None, None, None, None) - pkgobj = self.get_package_object(pkgtup) - if not pkgobj: - self.logger.debug("Unknown package %s" % - self.get_package_name(pkg)) - unknown.add(pkg) - else: - if self.yumbase.tsInfo.exists(pkgtup=pkgobj.pkgtup): - self.logger.debug("%s added to transaction multiple times" - % pkgobj) - else: - self.logger.debug("Adding %s to transaction" % pkgobj) - self.yumbase.tsInfo.addInstall(pkgobj) - self.yumbase.resolveDeps() - - for txmbr in self.yumbase.tsInfo: - packages.add(txmbr.pkgtup) - return list(packages), list(unknown) - - -def acquire_lock(func): - """ decorator for CacheManager methods that gets and release a - lock while the method runs """ - @wraps(func) - def inner(self, *args, **kwargs): - """ Get and release a lock while running the function this - wraps. """ - self.logger.debug("Acquiring lock at %s" % self.lockfile) - while not self.lock.i_am_locking(): - try: - self.lock.acquire(timeout=60) # wait up to 60 seconds - except LockTimeout: - self.lock.break_lock() - self.lock.acquire() - try: - func(self, *args, **kwargs) - finally: - self.lock.release() - self.logger.debug("Released lock at %s" % self.lockfile) - - return inner - - -class CacheManager(YumHelper): - """ Yum cache manager. Unlike :class:`DepSolver`, this can write - to the yum cache, and so is used for operations that muck with the - cache. (Technically, :func:`CacheManager.clean_cache` could be in - either DepSolver or CacheManager, but for consistency I've put it - here.) """ - - def __init__(self, cfgfile, verbose=1): - YumHelper.__init__(self, cfgfile, verbose=verbose) - self.lockfile = \ - os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), - "lock") - self.lock = FileLock(self.lockfile) - - @acquire_lock - def clean_cache(self): - """ clean the yum cache """ - for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", - "ExpireCache"]: - # for reasons that are entirely obvious, all of the yum - # API clean* methods return a tuple of 0 (zero, always - # zero) and a list containing a single message about how - # many files were deleted. so useful. thanks, yum. - msg = getattr(self.yumbase, "clean%s" % mdtype)()[1][0] - if not msg.startswith("0 "): - self.logger.info(msg) - - @acquire_lock - def populate_cache(self): - """ populate the yum cache """ - for repo in self.yumbase.repos.findRepos('*'): - repo.metadata_expire = 0 - repo.mdpolicy = "group:all" - self.yumbase.doRepoSetup() - self.yumbase.repos.doSetup() - for repo in self.yumbase.repos.listEnabled(): - # this populates the cache as a side effect - repo.repoXML # pylint: disable=W0104 - try: - repo.getGroups() - except yum.Errors.RepoMDError: - pass # this repo has no groups - self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) - self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) - self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) - # this does something with the groups cache as a side effect - self.yumbase.comps # pylint: disable=W0104 - - -def main(): - parser = OptionParser() - parser.add_option("-c", "--config", help="Config file") - parser.add_option("-v", "--verbose", help="Verbosity level", - action="count") - (options, args) = parser.parse_args() - - if options.verbose: - level = logging.DEBUG - clevel = logging.DEBUG - else: - level = logging.WARNING - clevel = logging.INFO - Bcfg2.Logger.setup_logging('bcfg2-yum-helper', to_syslog=True, - to_console=clevel, level=level) - logger = logging.getLogger('bcfg2-yum-helper') - - try: - cmd = args[0] - except IndexError: - logger.error("No command given") - return 1 - - if not os.path.exists(options.config): - logger.error("Config file %s not found" % options.config) - return 1 - - # pylint: disable=W0702 - rv = 0 - if cmd == "clean": - cachemgr = CacheManager(options.config, options.verbose) - try: - cachemgr.clean_cache() - print(json.dumps(True)) - except: - logger.error("Unexpected error cleaning cache: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(False)) - rv = 2 - elif cmd == "makecache": - cachemgr = CacheManager(options.config, options.verbose) - try: - # this code copied from yumcommands.py - cachemgr.populate_cache() - print(json.dumps(True)) - except yum.Errors.YumBaseError: - logger.error("Unexpected error creating cache: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(False)) - elif cmd == "complete": - depsolver = DepSolver(options.config, options.verbose) - try: - data = json.loads(sys.stdin.read()) - except: - logger.error("Unexpected error decoding JSON input: %s" % - sys.exc_info()[1]) - rv = 2 - try: - # if provided, set client arch for dependency resolution - arch = data.get('arch', None) - if arch is not None: - depsolver.setarch(arch) - - depsolver.groups = data['groups'] - (packages, unknown) = depsolver.complete( - [pkg_to_tuple(p) for p in data['packages']]) - print(json.dumps(dict(packages=list(packages), - unknown=list(unknown)))) - except: - logger.error("Unexpected error completing package set: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(dict(packages=[], unknown=data['packages']))) - rv = 2 - elif cmd == "get_groups": - depsolver = DepSolver(options.config, options.verbose) - try: - data = json.loads(sys.stdin.read()) - rv = dict() - for gdata in data: - if "type" in gdata: - packages = depsolver.get_group(gdata['group'], - ptype=gdata['type']) - else: - packages = depsolver.get_group(gdata['group']) - rv[gdata['group']] = list(packages) - print(json.dumps(rv)) - except: - logger.error("Unexpected error getting groups: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(dict())) - rv = 2 - else: - logger.error("Unknown command %s" % cmd) - print(json.dumps(None)) - rv = 2 - return rv +from Bcfg2.Server.Plugins.Packages.YumHelper import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py index b8534f5a8..500016c9d 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py @@ -48,200 +48,200 @@ test_data = """<Test> test_xdata = lxml.etree.XML(test_data) -if can_skip or HAS_AUGEAS: - class TestPOSIXAugeas(TestPOSIXTool): - test_obj = POSIXAugeas - - applied_commands = dict( - insert=lxml.etree.Element( - "Insert", label="Thing", - path='Test/Children[#attribute/identical = "true"]/Thing'), - set=lxml.etree.Element("Set", path="Test/Text/#text", - value="content with spaces"), - move=lxml.etree.Element( - "Move", source="Test/Foo", - destination='Test/Children[#attribute/identical = "false"]/Foo'), - remove=lxml.etree.Element("Remove", path="Test/Bar"), - clear=lxml.etree.Element("Clear", path="Test/Empty/#text"), - setm=lxml.etree.Element( - "SetMulti", sub="#text", value="same", - base='Test/Children[#attribute/multi = "true"]/Thing')) - - @skipUnless(HAS_AUGEAS, "Python Augeas libraries not found") - def setUp(self): - fd, self.tmpfile = tempfile.mkstemp() - os.fdopen(fd, 'w').write(test_data) - - def tearDown(self): - tmpfile = getattr(self, "tmpfile", None) - if tmpfile and os.path.exists(tmpfile): - os.unlink(tmpfile) - - def test_fully_specified(self): - ptool = self.get_obj() - - entry = lxml.etree.Element("Path", name="/test", type="augeas") - self.assertFalse(ptool.fully_specified(entry)) - - lxml.etree.SubElement(entry, "Set", path="/test", value="test") - self.assertTrue(ptool.fully_specified(entry)) - - def test_install(self): - # this is tested adequately by the other tests - pass - - def test_verify(self): - # this is tested adequately by the other tests - pass - - @patch("Bcfg2.Client.Tools.POSIX.Augeas.POSIXTool.verify") - def _verify(self, commands, mock_verify): - ptool = self.get_obj() - mock_verify.return_value = True - - entry = lxml.etree.Element("Path", name=self.tmpfile, - type="augeas", lens="Xml") - entry.extend(commands) - - modlist = [] - self.assertTrue(ptool.verify(entry, modlist)) - mock_verify.assert_called_with(ptool, entry, modlist) - self.assertXMLEqual(lxml.etree.parse(self.tmpfile).getroot(), - test_xdata) - - def test_verify_insert(self): - """ Test successfully verifying an Insert command """ - self._verify([self.applied_commands['insert']]) - - def test_verify_set(self): - """ Test successfully verifying a Set command """ - self._verify([self.applied_commands['set']]) - - def test_verify_move(self): - """ Test successfully verifying a Move command """ - self._verify([self.applied_commands['move']]) - - def test_verify_remove(self): - """ Test successfully verifying a Remove command """ - self._verify([self.applied_commands['remove']]) - - def test_verify_clear(self): - """ Test successfully verifying a Clear command """ - self._verify([self.applied_commands['clear']]) - - def test_verify_set_multi(self): - """ Test successfully verifying a SetMulti command """ - self._verify([self.applied_commands['setm']]) - - def test_verify_all(self): - """ Test successfully verifying multiple commands """ - self._verify(self.applied_commands.values()) - - @patch("Bcfg2.Client.Tools.POSIX.Augeas.POSIXTool.install") - def _install(self, commands, expected, mock_install, **attrs): - ptool = self.get_obj() - mock_install.return_value = True - - entry = lxml.etree.Element("Path", name=self.tmpfile, - type="augeas", lens="Xml") - for key, val in attrs.items(): - entry.set(key, val) - entry.extend(commands) - - self.assertTrue(ptool.install(entry)) - mock_install.assert_called_with(ptool, entry) - self.assertXMLEqual(lxml.etree.parse(self.tmpfile).getroot(), - expected) - - def test_install_set_existing(self): - """ Test setting the value of an existing node """ - expected = copy.deepcopy(test_xdata) - expected.find("Text").text = "Changed content" - self._install([lxml.etree.Element("Set", path="Test/Text/#text", - value="Changed content")], - expected) - - def test_install_set_new(self): - """ Test setting the value of an new node """ - expected = copy.deepcopy(test_xdata) - newtext = lxml.etree.SubElement(expected, "NewText") - newtext.text = "new content" - self._install([lxml.etree.Element("Set", path="Test/NewText/#text", - value="new content")], - expected) - - def test_install_remove(self): - """ Test removing a node """ - expected = copy.deepcopy(test_xdata) - expected.remove(expected.find("Attrs")) - self._install( - [lxml.etree.Element("Remove", - path='Test/*[#attribute/foo = "foo"]')], - expected) - - def test_install_move(self): - """ Test moving a node """ - expected = copy.deepcopy(test_xdata) - foo = expected.xpath("//Foo")[0] - expected.append(foo) - self._install( - [lxml.etree.Element("Move", source='Test/Children/Foo', - destination='Test/Foo')], - expected) - - def test_install_clear(self): - """ Test clearing a node """ - # TODO: clearing a node doesn't seem to work with the XML lens - # - # % augtool -b - # augtool> set /augeas/load/Xml/incl[3] "/tmp/test.xml" - # augtool> load - # augtool> clear '/files/tmp/test.xml/Test/Text/#text' - # augtool> save - # error: Failed to execute command - # saving failed (run 'print /augeas//error' for details) - # augtool> print /augeas//error - # - # The error isn't useful. - pass - - def test_install_set_multi(self): - """ Test setting multiple nodes at once """ - expected = copy.deepcopy(test_xdata) - for thing in expected.xpath("Children[@identical='true']/Thing"): - thing.text = "same" - self._install( - [lxml.etree.Element( - "SetMulti", value="same", - base='Test/Children[#attribute/identical = "true"]', - sub="Thing/#text")], - expected) - - def test_install_insert(self): - """ Test inserting a node """ - expected = copy.deepcopy(test_xdata) - children = expected.xpath("Children[@identical='true']")[0] - thing = lxml.etree.Element("Thing") - thing.text = "three" - children.append(thing) - self._install( - [lxml.etree.Element( - "Insert", - path='Test/Children[#attribute/identical = "true"]/Thing[2]', - label="Thing", where="after"), - lxml.etree.Element( - "Set", - path='Test/Children[#attribute/identical = "true"]/Thing[3]/#text', - value="three")], - expected) - - def test_install_initial(self): - """ Test creating initial content and then modifying it """ - os.unlink(self.tmpfile) - expected = copy.deepcopy(test_xdata) - expected.find("Text").text = "Changed content" - initial = lxml.etree.Element("Initial") - initial.text = test_data - modify = lxml.etree.Element("Set", path="Test/Text/#text", - value="Changed content") - self._install([initial, modify], expected, current_exists="false") +class TestPOSIXAugeas(TestPOSIXTool): + test_obj = POSIXAugeas + + applied_commands = dict( + insert=lxml.etree.Element( + "Insert", label="Thing", + path='Test/Children[#attribute/identical = "true"]/Thing'), + set=lxml.etree.Element("Set", path="Test/Text/#text", + value="content with spaces"), + move=lxml.etree.Element( + "Move", source="Test/Foo", + destination='Test/Children[#attribute/identical = "false"]/Foo'), + remove=lxml.etree.Element("Remove", path="Test/Bar"), + clear=lxml.etree.Element("Clear", path="Test/Empty/#text"), + setm=lxml.etree.Element( + "SetMulti", sub="#text", value="same", + base='Test/Children[#attribute/multi = "true"]/Thing')) + + @skipUnless(HAS_AUGEAS, "Python Augeas libraries not found") + def setUp(self): + TestPOSIXTool.setUp(self) + fd, self.tmpfile = tempfile.mkstemp() + os.fdopen(fd, 'w').write(test_data) + + def tearDown(self): + tmpfile = getattr(self, "tmpfile", None) + if tmpfile and os.path.exists(tmpfile): + os.unlink(tmpfile) + + def test_fully_specified(self): + ptool = self.get_obj() + + entry = lxml.etree.Element("Path", name="/test", type="augeas") + self.assertFalse(ptool.fully_specified(entry)) + + lxml.etree.SubElement(entry, "Set", path="/test", value="test") + self.assertTrue(ptool.fully_specified(entry)) + + def test_install(self): + # this is tested adequately by the other tests + pass + + def test_verify(self): + # this is tested adequately by the other tests + pass + + @patch("Bcfg2.Client.Tools.POSIX.Augeas.POSIXTool.verify") + def _verify(self, commands, mock_verify): + ptool = self.get_obj() + mock_verify.return_value = True + + entry = lxml.etree.Element("Path", name=self.tmpfile, + type="augeas", lens="Xml") + entry.extend(commands) + + modlist = [] + self.assertTrue(ptool.verify(entry, modlist)) + mock_verify.assert_called_with(ptool, entry, modlist) + self.assertXMLEqual(lxml.etree.parse(self.tmpfile).getroot(), + test_xdata) + + def test_verify_insert(self): + """ Test successfully verifying an Insert command """ + self._verify([self.applied_commands['insert']]) + + def test_verify_set(self): + """ Test successfully verifying a Set command """ + self._verify([self.applied_commands['set']]) + + def test_verify_move(self): + """ Test successfully verifying a Move command """ + self._verify([self.applied_commands['move']]) + + def test_verify_remove(self): + """ Test successfully verifying a Remove command """ + self._verify([self.applied_commands['remove']]) + + def test_verify_clear(self): + """ Test successfully verifying a Clear command """ + self._verify([self.applied_commands['clear']]) + + def test_verify_set_multi(self): + """ Test successfully verifying a SetMulti command """ + self._verify([self.applied_commands['setm']]) + + def test_verify_all(self): + """ Test successfully verifying multiple commands """ + self._verify(self.applied_commands.values()) + + @patch("Bcfg2.Client.Tools.POSIX.Augeas.POSIXTool.install") + def _install(self, commands, expected, mock_install, **attrs): + ptool = self.get_obj() + mock_install.return_value = True + + entry = lxml.etree.Element("Path", name=self.tmpfile, + type="augeas", lens="Xml") + for key, val in attrs.items(): + entry.set(key, val) + entry.extend(commands) + + self.assertTrue(ptool.install(entry)) + mock_install.assert_called_with(ptool, entry) + self.assertXMLEqual(lxml.etree.parse(self.tmpfile).getroot(), + expected) + + def test_install_set_existing(self): + """ Test setting the value of an existing node """ + expected = copy.deepcopy(test_xdata) + expected.find("Text").text = "Changed content" + self._install([lxml.etree.Element("Set", path="Test/Text/#text", + value="Changed content")], + expected) + + def test_install_set_new(self): + """ Test setting the value of an new node """ + expected = copy.deepcopy(test_xdata) + newtext = lxml.etree.SubElement(expected, "NewText") + newtext.text = "new content" + self._install([lxml.etree.Element("Set", path="Test/NewText/#text", + value="new content")], + expected) + + def test_install_remove(self): + """ Test removing a node """ + expected = copy.deepcopy(test_xdata) + expected.remove(expected.find("Attrs")) + self._install( + [lxml.etree.Element("Remove", + path='Test/*[#attribute/foo = "foo"]')], + expected) + + def test_install_move(self): + """ Test moving a node """ + expected = copy.deepcopy(test_xdata) + foo = expected.xpath("//Foo")[0] + expected.append(foo) + self._install( + [lxml.etree.Element("Move", source='Test/Children/Foo', + destination='Test/Foo')], + expected) + + def test_install_clear(self): + """ Test clearing a node """ + # TODO: clearing a node doesn't seem to work with the XML lens + # + # % augtool -b + # augtool> set /augeas/load/Xml/incl[3] "/tmp/test.xml" + # augtool> load + # augtool> clear '/files/tmp/test.xml/Test/Text/#text' + # augtool> save + # error: Failed to execute command + # saving failed (run 'print /augeas//error' for details) + # augtool> print /augeas//error + # + # The error isn't useful. + pass + + def test_install_set_multi(self): + """ Test setting multiple nodes at once """ + expected = copy.deepcopy(test_xdata) + for thing in expected.xpath("Children[@identical='true']/Thing"): + thing.text = "same" + self._install( + [lxml.etree.Element( + "SetMulti", value="same", + base='Test/Children[#attribute/identical = "true"]', + sub="Thing/#text")], + expected) + + def test_install_insert(self): + """ Test inserting a node """ + expected = copy.deepcopy(test_xdata) + children = expected.xpath("Children[@identical='true']")[0] + thing = lxml.etree.Element("Thing") + thing.text = "three" + children.append(thing) + self._install( + [lxml.etree.Element( + "Insert", + path='Test/Children[#attribute/identical = "true"]/Thing[2]', + label="Thing", where="after"), + lxml.etree.Element( + "Set", + path='Test/Children[#attribute/identical = "true"]/Thing[3]/#text', + value="three")], + expected) + + def test_install_initial(self): + """ Test creating initial content and then modifying it """ + os.unlink(self.tmpfile) + expected = copy.deepcopy(test_xdata) + expected.find("Text").text = "Changed content" + initial = lxml.etree.Element("Initial") + initial.text = test_data + modify = lxml.etree.Element("Set", path="Test/Text/#text", + value="Changed content") + self._install([initial, modify], expected, current_exists="false") diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py index 8f933e08f..69dd562be 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py @@ -55,8 +55,8 @@ class TestPOSIXFile(TestPOSIXTool): def test_get_data(self): orig_entry = lxml.etree.Element("Path", name="/test", type="file") - ptool = self.get_obj(setup=dict(encoding="ascii", ppath='/', - max_copies=5)) + Bcfg2.Options.setup.encoding = "ascii" + ptool = self.get_obj() entry = copy.deepcopy(orig_entry) entry.text = b64encode("test") @@ -91,8 +91,7 @@ class TestPOSIXFile(TestPOSIXTool): @patch("Bcfg2.Client.Tools.POSIX.base.POSIXTool.verify") def test_verify(self, mock_verify, mock_open): entry = lxml.etree.Element("Path", name="/test", type="file") - ptool = self.get_obj(setup=dict(interactive=False, ppath='/', - max_copies=5)) + ptool = self.get_obj() ptool._exists = Mock() ptool._get_data = Mock() ptool._get_diffs = Mock() @@ -223,8 +222,8 @@ class TestPOSIXFile(TestPOSIXTool): group='root') orig_entry.text = "test" ondisk = "test2" - ptool = self.get_obj(setup=dict(encoding="utf-8", ppath='/', - max_copies=5)) + Bcfg2.Options.setup.encoding = "utf-8" + ptool = self.get_obj() ptool._get_data = Mock() ptool._diff = Mock() ptool._is_string = Mock() @@ -271,7 +270,6 @@ class TestPOSIXFile(TestPOSIXTool): mock_open.assert_called_with(entry.get("name")) mock_open.return_value.read.assert_any_call() ptool._diff.assert_called_with(ondisk, entry.text, - difflib.unified_diff, filename=entry.get("name")) self.assertIsNotNone(entry.get("qtext")) del entry.attrib['qtext'] @@ -281,8 +279,8 @@ class TestPOSIXFile(TestPOSIXTool): entry = reset() ptool._get_diffs(entry, content=ondisk) self.assertFalse(mock_open.called) - ptool._diff.assert_called_with(ondisk, entry.text, difflib.ndiff, - filename=entry.get("name")) + ptool._diff.assert_called_with(ondisk, entry.text, + filename=entry.get("name")) self.assertIsNone(entry.get("qtext")) self.assertEqual(entry.get("current_bdiff"), b64encode("\n".join(ptool._diff.return_value))) @@ -297,9 +295,7 @@ class TestPOSIXFile(TestPOSIXTool): mock_open.assert_called_with(entry.get("name")) mock_open.return_value.read.assert_any_call() self.assertItemsEqual(ptool._diff.call_args_list, - [call(ondisk, entry.text, difflib.unified_diff, - filename=entry.get("name")), - call(ondisk, entry.text, difflib.ndiff, + [call(ondisk, entry.text, filename=entry.get("name"))]) self.assertIsNotNone(entry.get("qtext")) self.assertTrue(entry.get("qtext").startswith("test\n")) @@ -312,16 +308,14 @@ class TestPOSIXFile(TestPOSIXTool): # non-sensitive, interactive with unicode data entry = reset() entry.text = u("tëst") - encoded = entry.text.encode(ptool.setup['encoding']) + encoded = entry.text.encode(Bcfg2.Options.setup.encoding) ptool._diff.return_value = ["-test2", "+tëst"] ptool._get_data.return_value = (encoded, False) ptool._get_diffs(entry, interactive=True) mock_open.assert_called_with(entry.get("name")) mock_open.return_value.read.assert_any_call() self.assertItemsEqual(ptool._diff.call_args_list, - [call(ondisk, encoded, difflib.unified_diff, - filename=entry.get("name")), - call(ondisk, encoded, difflib.ndiff, + [call(ondisk, encoded, filename=entry.get("name"))]) self.assertIsNotNone(entry.get("qtext")) self.assertEqual(entry.get("current_bdiff"), @@ -416,35 +410,23 @@ class TestPOSIXFile(TestPOSIXTool): ptool._rename_tmpfile.assert_called_with(newfile, entry) mock_install.assert_called_with(ptool, entry) - @patch("time.time") - def test_diff(self, mock_time): + @patch("difflib.unified_diff") + def test_diff(self, mock_diff): ptool = self.get_obj() + filename = "/test" content1 = "line1\nline2" content2 = "line3" - self.now = 1345640723 - - def time_rv(): - self.now += 1 - return self.now - mock_time.side_effect = time_rv - rv = ["line1", "line2", "line3"] - func = Mock() - func.return_value = rv - self.assertItemsEqual(ptool._diff(content1, content2, func), rv) - func.assert_called_with(["line1", "line2"], ["line3"]) - - func.reset_mock() - mock_time.reset_mock() - def time_rv(): - self.now += 5 - return self.now - mock_time.side_effect = time_rv - - def slow_diff(content1, content2): - for i in range(1, 10): - yield "line%s" % i - func.side_effect = slow_diff - self.assertFalse(ptool._diff(content1, content2, func), rv) - func.assert_called_with(["line1", "line2"], ["line3"]) + mock_diff.return_value = rv + self.assertItemsEqual(ptool._diff(content1, content2), rv) + mock_diff.assert_called_with(["line1", "line2"], ["line3"], + fromfile='', tofile='') + + mock_diff.reset_mock() + self.assertItemsEqual(ptool._diff(content1, content2, + filename=filename), + rv) + mock_diff.assert_called_with(["line1", "line2"], ["line3"], + fromfile='/test (on disk)', + tofile='/test (from bcfg2)') diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Test__init.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Test__init.py index f01082e86..adc2032b7 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Test__init.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Test__init.py @@ -49,7 +49,6 @@ class TestPOSIX(TestTool): mock_canVerify.assert_called_with(posix, entry) # next, test fully_specified failure - posix.logger.error.reset_mock() mock_canVerify.reset_mock() mock_canVerify.return_value = True mock_fully_spec = Mock() @@ -59,17 +58,14 @@ class TestPOSIX(TestTool): self.assertFalse(posix.canVerify(entry)) mock_canVerify.assert_called_with(posix, entry) mock_fully_spec.assert_called_with(entry) - self.assertTrue(posix.logger.error.called) # finally, test success - posix.logger.error.reset_mock() mock_canVerify.reset_mock() mock_fully_spec.reset_mock() mock_fully_spec.return_value = True self.assertTrue(posix.canVerify(entry)) mock_canVerify.assert_called_with(posix, entry) mock_fully_spec.assert_called_with(entry) - self.assertFalse(posix.logger.error.called) @patch("Bcfg2.Client.Tools.Tool.canInstall") def test_canInstall(self, mock_canInstall): @@ -82,7 +78,6 @@ class TestPOSIX(TestTool): mock_canInstall.assert_called_with(posix, entry) # next, test fully_specified failure - posix.logger.error.reset_mock() mock_canInstall.reset_mock() mock_canInstall.return_value = True mock_fully_spec = Mock() @@ -92,17 +87,14 @@ class TestPOSIX(TestTool): self.assertFalse(posix.canInstall(entry)) mock_canInstall.assert_called_with(posix, entry) mock_fully_spec.assert_called_with(entry) - self.assertTrue(posix.logger.error.called) # finally, test success - posix.logger.error.reset_mock() mock_canInstall.reset_mock() mock_fully_spec.reset_mock() mock_fully_spec.return_value = True self.assertTrue(posix.canInstall(entry)) mock_canInstall.assert_called_with(posix, entry) mock_fully_spec.assert_called_with(entry) - self.assertFalse(posix.logger.error.called) def test_InstallPath(self): posix = self.get_obj() @@ -127,15 +119,17 @@ class TestPOSIX(TestTool): mock_verify.reset_mock() mock_verify.return_value = False - posix.setup.__getitem__.return_value = True + Bcfg2.Options.setup.interactive = True self.assertFalse(posix.VerifyPath(entry, modlist)) self.assertIsNotNone(entry.get('qtext')) @patch('os.remove') def test_prune_old_backups(self, mock_remove): entry = lxml.etree.Element("Path", name="/etc/foo", type="file") - setup = dict(ppath='/', max_copies=5, paranoid=True) - posix = self.get_obj(setup=setup) + Bcfg2.Options.setup.paranoid_path = '/' + Bcfg2.Options.setup.paranoid_copies = 5 + Bcfg2.Options.setup.paranoid = True + posix = self.get_obj() remove = ["_etc_foo_2012-07-20T04:13:22.364989", "_etc_foo_2012-07-31T04:13:23.894958", @@ -152,9 +146,8 @@ class TestPOSIX(TestTool): def inner(mock_listdir): mock_listdir.side_effect = OSError posix._prune_old_backups(entry) - self.assertTrue(posix.logger.error.called) self.assertFalse(mock_remove.called) - mock_listdir.assert_called_with(setup['ppath']) + mock_listdir.assert_called_with(Bcfg2.Options.setup.paranoid_path) mock_listdir.reset_mock() mock_remove.reset_mock() @@ -162,23 +155,23 @@ class TestPOSIX(TestTool): mock_listdir.return_value = keep + remove posix._prune_old_backups(entry) - mock_listdir.assert_called_with(setup['ppath']) + mock_listdir.assert_called_with(Bcfg2.Options.setup.paranoid_path) self.assertItemsEqual(mock_remove.call_args_list, - [call(os.path.join(setup['ppath'], p)) + [call(os.path.join(Bcfg2.Options.setup.paranoid_path, + p)) for p in remove]) mock_listdir.reset_mock() mock_remove.reset_mock() mock_remove.side_effect = OSError - posix.logger.error.reset_mock() # test to ensure that we call os.remove() for all files that # need to be removed even if we get an error posix._prune_old_backups(entry) - mock_listdir.assert_called_with(setup['ppath']) + mock_listdir.assert_called_with(Bcfg2.Options.setup.paranoid_path) self.assertItemsEqual(mock_remove.call_args_list, - [call(os.path.join(setup['ppath'], p)) + [call(os.path.join(Bcfg2.Options.setup.paranoid_path, + p)) for p in remove]) - self.assertTrue(posix.logger.error.called) inner() @@ -186,8 +179,10 @@ class TestPOSIX(TestTool): @patch("os.path.isdir") def test_paranoid_backup(self, mock_isdir, mock_copy): entry = lxml.etree.Element("Path", name="/etc/foo", type="file") - setup = dict(ppath='/', max_copies=5, paranoid=False) - posix = self.get_obj(setup=setup) + Bcfg2.Options.setup.paranoid_path = '/' + Bcfg2.Options.setup.paranoid_copies = 5 + Bcfg2.Options.setup.paranoid = False + posix = self.get_obj() posix._prune_old_backups = Mock() # paranoid false globally @@ -196,9 +191,7 @@ class TestPOSIX(TestTool): self.assertFalse(mock_copy.called) # paranoid false on the entry - setup['paranoid'] = True - posix = self.get_obj(setup=setup) - posix._prune_old_backups = Mock() + Bcfg2.Options.setup.paranoid = True def reset(): mock_isdir.reset_mock() @@ -238,6 +231,6 @@ class TestPOSIX(TestTool): # just test it good enough self.assertEqual(mock_copy.call_args[0][0], entry.get("name")) - bkupnam = os.path.join(setup['ppath'], + bkupnam = os.path.join(Bcfg2.Options.setup.paranoid_path, entry.get('name').replace('/', '_')) + '_' self.assertEqual(bkupnam, mock_copy.call_args[0][1][:len(bkupnam)]) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py index ab6e2fe54..ea4ca3f5f 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py @@ -197,7 +197,8 @@ class TestPOSIXTool(TestTool): @patch("os.chown") @patch("os.chmod") @patch("os.utime") - def test_set_perms(self, mock_utime, mock_chmod, mock_chown): + @patch("os.geteuid") + def test_set_perms(self, mock_geteuid, mock_utime, mock_chmod, mock_chown): ptool = self.get_obj() ptool._norm_entry_uid = Mock() ptool._norm_entry_gid = Mock() @@ -211,7 +212,12 @@ class TestPOSIXTool(TestTool): mock_chmod.reset_mock() mock_chown.reset_mock() mock_utime.reset_mock() + mock_geteuid.reset_mock() + # pretend to run as root + mock_geteuid.return_value = 0 + + # test symlink -- no owner, group, permissions entry = lxml.etree.Element("Path", name="/etc/foo", to="/etc/bar", type="symlink") ptool._set_acls.return_value = True @@ -220,12 +226,12 @@ class TestPOSIXTool(TestTool): ptool._set_secontext.assert_called_with(entry, path=entry.get("name")) ptool._set_acls.assert_called_with(entry, path=entry.get("name")) + # test file with owner, group, permissions + reset() entry = lxml.etree.Element("Path", name="/etc/foo", owner="owner", group="group", mode="644", type="file") ptool._norm_entry_uid.return_value = 10 ptool._norm_entry_gid.return_value = 100 - - reset() self.assertTrue(ptool._set_perms(entry)) ptool._norm_entry_uid.assert_called_with(entry) ptool._norm_entry_gid.assert_called_with(entry) @@ -236,6 +242,23 @@ class TestPOSIXTool(TestTool): ptool._set_secontext.assert_called_with(entry, path=entry.get("name")) ptool._set_acls.assert_called_with(entry, path=entry.get("name")) + # test file with owner, group, permissions, run as non-root + mock_geteuid.return_value = 1000 + reset() + entry = lxml.etree.Element("Path", name="/etc/foo", owner="owner", + group="group", mode="644", type="file") + self.assertTrue(ptool._set_perms(entry)) + self.assertFalse(ptool._norm_entry_uid.called) + self.assertFalse(ptool._norm_entry_gid.called) + self.assertFalse(mock_chown.called) + mock_chmod.assert_called_with(entry.get("name"), + int(entry.get("mode"), 8)) + self.assertFalse(mock_utime.called) + ptool._set_secontext.assert_called_with(entry, path=entry.get("name")) + ptool._set_acls.assert_called_with(entry, path=entry.get("name")) + mock_geteuid.return_value = 0 + + # test with mtime reset() mtime = 1344459042 entry.set("mtime", str(mtime)) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py index c207900f1..cc1ea6fd7 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py @@ -24,18 +24,16 @@ from TestTools.Test_init import TestTool class TestPOSIXUsers(TestTool): test_obj = POSIXUsers - def get_obj(self, logger=None, setup=None, config=None): - if setup is None: - setup = MagicMock() - def getitem(key): - if key == 'encoding': - return 'UTF-8' - else: - return [] - - setup.__getitem__.side_effect = getitem + def setUp(self): + TestTool.setUp(self) + set_setup_default('uid_whitelist', []) + set_setup_default('uid_blacklist', []) + set_setup_default('gid_whitelist', []) + set_setup_default('gid_blacklist', []) + set_setup_default('encoding', 'UTF-8') - return TestTool.get_obj(self, logger, setup, config) + def get_obj(self, config=None): + return TestTool.get_obj(self, config) @patch("pwd.getpwall") @patch("grp.getgrall") @@ -141,10 +139,9 @@ class TestPOSIXUsers(TestTool): users.set_defaults['POSIXUser'] = Mock() users.set_defaults['POSIXUser'].side_effect = lambda e: e - states = dict() - self.assertEqual(users.Inventory(states), + self.assertEqual(users.Inventory(), mock_Inventory.return_value) - mock_Inventory.assert_called_with(users, states, config.getchildren()) + mock_Inventory.assert_called_with(users, config.getchildren()) lxml.etree.SubElement(orig_bundle, "POSIXGroup", name="test") self.assertXMLEqual(orig_bundle, bundle) @@ -306,9 +303,8 @@ class TestPOSIXUsers(TestTool): entries = [lxml.etree.Element("POSIXUser", name="test"), lxml.etree.Element("POSIXGroup", name="test"), lxml.etree.Element("POSIXUser", name="test2")] - states = dict() - users.Install(entries, states) + states = users.Install(entries) self.assertItemsEqual(entries, states.keys()) for state in states.values(): self.assertEqual(state, users._install.return_value) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py index e0dacc1ce..740b90251 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py @@ -17,30 +17,29 @@ while path != "/": path = os.path.dirname(path) from common import * +# try to find true +if os.path.exists("/bin/true"): + TRUE = "/bin/true" +elif os.path.exists("/usr/bin/true"): + TRUE = "/usr/bin/true" +else: + TRUE = None + class TestTool(Bcfg2TestCase): test_obj = Tool - def get_obj(self, logger=None, setup=None, config=None): + def setUp(self): + set_setup_default('command_timeout') + set_setup_default('interactive', False) + set_setup_default('decision') + + def get_obj(self, config=None): if config is None: config = lxml.etree.Element("Configuration") - if not logger: - def print_msg(msg): - print(msg) - logger = Mock() - logger.error = Mock(side_effect=print_msg) - logger.warning = Mock(side_effect=print_msg) - logger.info = Mock(side_effect=print_msg) - logger.debug = Mock(side_effect=print_msg) - if not setup: - setup = MagicMock() - if 'command_timeout' not in setup: - setup['command_timeout'] = None - if 'decision' not in setup: - setup['decision'] = None execs = self.test_obj.__execs__ self.test_obj.__execs__ = [] - rv = self.test_obj(logger, setup, config) + rv = self.test_obj(config) self.test_obj.__execs__ = execs return rv @@ -50,10 +49,12 @@ class TestTool(Bcfg2TestCase): @patch("%s.%s._analyze_config" % (self.test_obj.__module__, self.test_obj.__name__)) def inner(mock_analyze_config, mock_check_execs): - t = self.get_obj() + self.get_obj() mock_analyze_config.assert_called_with() mock_check_execs.assert_called_with() + inner() + def test__analyze_config(self): t = self.get_obj() t.getSupportedEntries = Mock() @@ -76,10 +77,11 @@ class TestTool(Bcfg2TestCase): ["/test"] + [e.get("name") for e in important]) t.getSupportedEntries.assert_called_with() + @skipIf(TRUE is None, "/bin/true or equivalent not found") def test__check_execs(self): t = self.get_obj() if t.__execs__ == []: - t.__execs__.append("/bin/true") + t.__execs__.append(TRUE) @patch("os.stat") def inner(mock_stat): @@ -168,14 +170,12 @@ class TestTool(Bcfg2TestCase): self.assertItemsEqual(states, expected_states) self.assertEqual(t.extra, t.FindExtra.return_value) - actual_states = dict() - t.Inventory(actual_states, structures=[bundle1, bundle2]) + actual_states = t.Inventory(structures=[bundle1, bundle2]) perform_assertions(actual_states) reset() - actual_states = dict() t.config = config - t.Inventory(actual_states) + actual_states = t.Inventory() perform_assertions(actual_states) def test_Install(self): @@ -201,9 +201,8 @@ class TestTool(Bcfg2TestCase): expected_states.update(dict([(e, t.InstallService.return_value) for e in entries if e.tag == "Service"])) - actual_states = dict() t.modified = [] - t.Install(entries, actual_states) + actual_states = t.Install(entries) self.assertItemsEqual(t.InstallPath.call_args_list, [call(e) for e in entries if e.tag == "Path"]) self.assertItemsEqual(t.InstallPackage.call_args_list, @@ -387,8 +386,7 @@ class TestPkgTool(TestTool): # test single-pass install success reset() pt.cmd.run.return_value = True - states = dict([(p, False) for p in packages]) - pt.Install(packages, states) + states = pt.Install(packages) pt._get_package_command.assert_called_with(packages) pt.cmd.run.assert_called_with([p.get("name") for p in packages]) self.assertItemsEqual(states, @@ -408,8 +406,7 @@ class TestPkgTool(TestTool): pt.VerifyPackage.side_effect = lambda p, m: p.get("name") == "bar" pt.cmd.run.side_effect = run - states = dict([(p, False) for p in packages]) - pt.Install(packages, states) + states = pt.Install(packages) pt._get_package_command.assert_any_call(packages) for pkg in packages: pt.VerifyPackage.assert_any_call(pkg, []) @@ -553,15 +550,15 @@ class TestSvcTool(TestTool): @patch("Bcfg2.Client.prompt") def test_BundleUpdated(self, mock_prompt): - st = self.get_obj(setup=dict(interactive=False, - servicemode='default')) + Bcfg2.Options.setup.service_mode = 'default' + Bcfg2.Options.setup.interactive = False + st = self.get_obj() st.handlesEntry = Mock() st.handlesEntry.side_effect = lambda e: e.tag == "Service" st.stop_service = Mock() - st.stop_service.return_value = 0 + st.stop_service.return_value = True st.restart_service = Mock() - st.restart_service.side_effect = lambda e: \ - int(e.get("name") != "failed") + st.restart_service.side_effect = lambda e: e.get("name") != "failed" def reset(): st.handlesEntry.reset_mock() @@ -595,8 +592,7 @@ class TestSvcTool(TestTool): # test in non-interactive mode reset() - states = dict() - st.BundleUpdated(bundle, states) + states = st.BundleUpdated(bundle) self.assertItemsEqual(st.handlesEntry.call_args_list, [call(e) for e in entries]) st.stop_service.assert_called_with(stop) @@ -608,9 +604,8 @@ class TestSvcTool(TestTool): # test in interactive mode reset() mock_prompt.side_effect = lambda p: "interactive2" not in p - st.setup['interactive'] = True - states = dict() - st.BundleUpdated(bundle, states) + Bcfg2.Options.setup.interactive = True + states = st.BundleUpdated(bundle) self.assertItemsEqual(st.handlesEntry.call_args_list, [call(e) for e in entries]) st.stop_service.assert_called_with(stop) @@ -622,10 +617,9 @@ class TestSvcTool(TestTool): # test in build mode reset() - st.setup['interactive'] = False - st.setup['servicemode'] = 'build' - states = dict() - st.BundleUpdated(bundle, states) + Bcfg2.Options.setup.interactive = False + Bcfg2.Options.setup.service_mode = 'build' + states = st.BundleUpdated(bundle) self.assertItemsEqual(st.handlesEntry.call_args_list, [call(e) for e in entries]) self.assertItemsEqual(st.stop_service.call_args_list, @@ -641,10 +635,9 @@ class TestSvcTool(TestTool): services = install + [lxml.etree.Element("Service", type="test", name="bar", install="false")] st = self.get_obj() - states = Mock() - self.assertEqual(st.Install(services, states), + self.assertEqual(st.Install(services), mock_Install.return_value) - mock_Install.assert_called_with(st, install, states) + mock_Install.assert_called_with(st, install) def test_InstallService(self): st = self.get_obj() diff --git a/testsuite/Testsrc/Testlib/TestEncryption.py b/testsuite/Testsrc/Testlib/TestEncryption.py deleted file mode 100644 index c03aa66e1..000000000 --- a/testsuite/Testsrc/Testlib/TestEncryption.py +++ /dev/null @@ -1,199 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import sys -from Bcfg2.Compat import b64decode -from mock import Mock, MagicMock, patch - -# add all parent testsuite directories to sys.path to allow (most) -# relative imports in python 2.4 -path = os.path.dirname(__file__) -while path != "/": - if os.path.basename(path).lower().startswith("test"): - sys.path.append(path) - if os.path.basename(path) == "testsuite": - break - path = os.path.dirname(path) -from common import * - -try: - from Bcfg2.Encryption import * - HAS_CRYPTO = True -except ImportError: - HAS_CRYPTO = False - - -if can_skip or HAS_CRYPTO: - class TestEncryption(Bcfg2TestCase): - plaintext = """foo bar -baz -ö -\t\tquux -""" + "a" * 16384 # 16K is completely arbitrary - iv = "0123456789ABCDEF" - salt = "01234567" - algo = "des_cbc" - - @skipUnless(HAS_CRYPTO, "Encryption libraries not found") - def setUp(self): - pass - - def test_str_crypt(self): - """ test str_encrypt/str_decrypt """ - key = "a simple key" - - # simple symmetrical test with no options - crypted = str_encrypt(self.plaintext, key) - self.assertEqual(self.plaintext, str_decrypt(crypted, key)) - - # symmetrical test with lots of options - crypted = str_encrypt(self.plaintext, key, - iv=self.iv, salt=self.salt, - algorithm=self.algo) - self.assertEqual(self.plaintext, - str_decrypt(crypted, key, iv=self.iv, - algorithm=self.algo)) - - # test that different algorithms are actually used - self.assertNotEqual(str_encrypt(self.plaintext, key), - str_encrypt(self.plaintext, key, - algorithm=self.algo)) - - # test that different keys are actually used - self.assertNotEqual(str_encrypt(self.plaintext, key), - str_encrypt(self.plaintext, "different key")) - - # test that different IVs are actually used - self.assertNotEqual(str_encrypt(self.plaintext, key, iv=self.iv), - str_encrypt(self.plaintext, key)) - - # test that errors are raised on bad decrypts - crypted = str_encrypt(self.plaintext, key, algorithm=self.algo) - self.assertRaises(EVPError, str_decrypt, - crypted, "bogus key", algorithm=self.algo) - self.assertRaises(EVPError, str_decrypt, - crypted, key) # bogus algorithm - - def test_ssl_crypt(self): - """ test ssl_encrypt/ssl_decrypt """ - passwd = "a simple passphrase" - - # simple symmetrical test - crypted = ssl_encrypt(self.plaintext, passwd) - self.assertEqual(self.plaintext, ssl_decrypt(crypted, passwd)) - - # more complex symmetrical test - crypted = ssl_encrypt(self.plaintext, passwd, algorithm=self.algo, - salt=self.salt) - self.assertEqual(self.plaintext, - ssl_decrypt(crypted, passwd, algorithm=self.algo)) - - # test that different algorithms are actually used - self.assertNotEqual(ssl_encrypt(self.plaintext, passwd), - ssl_encrypt(self.plaintext, passwd, - algorithm=self.algo)) - - # test that different passwords are actually used - self.assertNotEqual(ssl_encrypt(self.plaintext, passwd), - ssl_encrypt(self.plaintext, "different pass")) - - # there's no reasonable test we can do to see if the - # output is base64-encoded, unfortunately, but if it's - # obviously not we fail - crypted = ssl_encrypt(self.plaintext, passwd) - self.assertRegexpMatches(crypted, r'^[A-Za-z0-9+/]+[=]{0,2}$') - - # test that errors are raised on bad decrypts - crypted = ssl_encrypt(self.plaintext, passwd, - algorithm=self.algo) - self.assertRaises(EVPError, ssl_decrypt, - crypted, "bogus passwd", algorithm=self.algo) - self.assertRaises(EVPError, ssl_decrypt, - crypted, passwd) # bogus algorithm - - def test_get_algorithm(self): - setup = Mock() - # we don't care what the default is, as long as there is - # one - setup.cfp.get.return_value = ALGORITHM - self.assertRegexpMatches(get_algorithm(setup), - r'^[a-z0-9]+_[a-z0-9_]+$') - setup.cfp.get.assert_called_with(CFG_SECTION, CFG_ALGORITHM, - default=ALGORITHM) - - setup.cfp.get.return_value = self.algo - self.assertEqual(get_algorithm(setup), self.algo) - setup.cfp.get.assert_called_with(CFG_SECTION, CFG_ALGORITHM, - default=ALGORITHM) - - # test that get_algorithm converts algorithms given in - # OpenSSL style to M2Crypto style - setup.cfp.get.return_value = "DES-EDE3-CFB8" - self.assertEqual(get_algorithm(setup), "des_ede3_cfb8") - setup.cfp.get.assert_called_with(CFG_SECTION, CFG_ALGORITHM, - default=ALGORITHM) - - def test_get_passphrases(self): - setup = Mock() - setup.cfp.has_section.return_value = False - self.assertEqual(get_passphrases(setup), dict()) - - setup.cfp.has_section.return_value = True - setup.cfp.options.return_value = ["foo", "bar", CFG_ALGORITHM] - setup.cfp.get.return_value = "passphrase" - self.assertItemsEqual(get_passphrases(setup), - dict(foo="passphrase", - bar="passphrase")) - - @patch("Bcfg2.Encryption.get_passphrases") - def test_bruteforce_decrypt(self, mock_passphrases): - passwd = "a simple passphrase" - crypted = ssl_encrypt(self.plaintext, passwd) - setup = Mock() - - # test with no passphrases given nor in config - mock_passphrases.return_value = dict() - self.assertRaises(EVPError, - bruteforce_decrypt, - crypted, setup=setup) - mock_passphrases.assert_called_with(setup) - - # test with good passphrase given in function call - mock_passphrases.reset_mock() - self.assertEqual(self.plaintext, - bruteforce_decrypt(crypted, - passphrases=["bogus pass", - passwd, - "also bogus"])) - self.assertFalse(mock_passphrases.called) - - # test with no good passphrase given nor in config - mock_passphrases.reset_mock() - self.assertRaises(EVPError, - bruteforce_decrypt, - crypted, passphrases=["bogus", "also bogus"]) - self.assertFalse(mock_passphrases.called) - - # test with good passphrase in config file - mock_passphrases.reset_mock() - mock_passphrases.return_value = dict(bogus="bogus", - real=passwd, - bogus2="also bogus") - self.assertEqual(self.plaintext, - bruteforce_decrypt(crypted, setup=setup)) - mock_passphrases.assert_called_with(setup) - - # test that passphrases given in function call take - # precedence over config - mock_passphrases.reset_mock() - self.assertRaises(EVPError, - bruteforce_decrypt, - crypted, setup=setup, - passphrases=["bogus", "also bogus"]) - self.assertFalse(mock_passphrases.called) - - # test that different algorithms are used - mock_passphrases.reset_mock() - crypted = ssl_encrypt(self.plaintext, passwd, algorithm=self.algo) - self.assertEqual(self.plaintext, - bruteforce_decrypt(crypted, setup=setup, - algorithm=self.algo)) diff --git a/testsuite/Testsrc/Testlib/TestLogger.py b/testsuite/Testsrc/Testlib/TestLogger.py new file mode 100644 index 000000000..1baea2f35 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestLogger.py @@ -0,0 +1,63 @@ +import os +import sys +import logging +from mock import Mock +from Bcfg2.Logger import * + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != '/': + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * + + +class TestDebuggable(Bcfg2TestCase): + test_obj = Debuggable + + def setUp(self): + set_setup_default('debug', False) + + def get_obj(self): + return self.test_obj() + + def test__init(self): + d = self.get_obj() + self.assertIsInstance(d.logger, logging.Logger) + self.assertFalse(d.debug_flag) + + def test_set_debug(self): + d = self.get_obj() + self.assertEqual(True, d.set_debug(True)) + self.assertEqual(d.debug_flag, True) + + self.assertEqual(False, d.set_debug(False)) + self.assertEqual(d.debug_flag, False) + + def test_toggle_debug(self): + d = self.get_obj() + d.set_debug = Mock() + orig = d.debug_flag + self.assertEqual(d.toggle_debug(), + d.set_debug.return_value) + d.set_debug.assert_called_with(not orig) + + def test_debug_log(self): + d = self.get_obj() + d.logger = Mock() + d.debug_flag = False + d.debug_log("test") + self.assertFalse(d.logger.error.called) + + d.logger.reset_mock() + d.debug_log("test", flag=True) + self.assertTrue(d.logger.error.called) + + d.logger.reset_mock() + d.debug_flag = True + d.debug_log("test") + self.assertTrue(d.logger.error.called) diff --git a/testsuite/Testsrc/Testlib/TestOptions.py b/testsuite/Testsrc/Testlib/TestOptions.py deleted file mode 100644 index dc91a499b..000000000 --- a/testsuite/Testsrc/Testlib/TestOptions.py +++ /dev/null @@ -1,236 +0,0 @@ -import os -import sys -from mock import Mock, MagicMock, patch -from Bcfg2.Options import * -from Bcfg2.Compat import ConfigParser - -# add all parent testsuite directories to sys.path to allow (most) -# relative imports in python 2.4 -path = os.path.dirname(__file__) -while path != '/': - if os.path.basename(path).lower().startswith("test"): - sys.path.append(path) - if os.path.basename(path) == "testsuite": - break - path = os.path.dirname(path) -from common import * - -class TestDefaultConfigParser(Bcfg2TestCase): - @patch("%s.ConfigParser.get" % ConfigParser.__name__) - def test_get(self, mock_get): - dcp = DefaultConfigParser() - mock_get.return_value = "foo" - self.assertEqual(dcp.get("section", "option"), "foo") - mock_get.assert_called_with(dcp, "section", "option") - - mock_get.reset_mock() - self.assertEqual(dcp.get("section", "option", - default="bar", other="test"), "foo") - mock_get.assert_called_with(dcp, "section", "option", other="test") - - for etype, err in [(ConfigParser.NoOptionError, - ConfigParser.NoOptionError(None, None)), - (ConfigParser.NoSectionError, - ConfigParser.NoSectionError(None))]: - mock_get.side_effect = err - mock_get.reset_mock() - self.assertEqual(dcp.get("section", "option", default="bar"), "bar") - mock_get.assert_called_with(dcp, "section", "option") - - mock_get.reset_mock() - self.assertRaises(etype, dcp.get, "section", "option") - mock_get.assert_called_with(dcp, "section", "option") - - @patch("%s.ConfigParser.getboolean" % ConfigParser.__name__) - def test_getboolean(self, mock_getboolean): - dcp = DefaultConfigParser() - mock_getboolean.return_value = True - self.assertEqual(dcp.getboolean("section", "option"), True) - mock_getboolean.assert_called_with(dcp, "section", "option") - - mock_getboolean.reset_mock() - self.assertEqual(dcp.getboolean("section", "option", - default=False, other="test"), True) - mock_getboolean.assert_called_with(dcp, "section", "option", - other="test") - - for etype, err in [(ConfigParser.NoOptionError, - ConfigParser.NoOptionError(None, None)), - (ConfigParser.NoSectionError, - ConfigParser.NoSectionError(None))]: - mock_getboolean.side_effect = err - mock_getboolean.reset_mock() - self.assertEqual(dcp.getboolean("section", "option", default=False), - False) - mock_getboolean.assert_called_with(dcp, "section", "option") - - mock_getboolean.reset_mock() - self.assertRaises(etype, dcp.getboolean, "section", "option") - mock_getboolean.assert_called_with(dcp, "section", "option") - - -class TestOption(Bcfg2TestCase): - def test__init(self): - self.assertRaises(OptionFailure, - Option, - 'foo', False, cmd='f') - self.assertRaises(OptionFailure, - Option, - 'foo', False, cmd='--f') - self.assertRaises(OptionFailure, - Option, - 'foo', False, cmd='-foo') - self.assertRaises(OptionFailure, - Option, - 'foo', False, cmd='-foo', long_arg=True) - opt = Option('foo', False) - self.assertTrue(opt.boolean) - opt = Option('foo', False, odesc='<val>') - self.assertFalse(opt.boolean) - opt = Option('foo', False, cook=get_bool) - self.assertFalse(opt.boolean) - opt = Option('foo', "foo") - self.assertFalse(opt.boolean) - - def test_get_cooked_value(self): - opt = Option('foo', False) - opt.boolean = True - self.assertTrue(opt.get_cooked_value("anything")) - - opt = Option('foo', 'foo') - opt.boolean = False - opt.cook = False - self.assertEqual("foo", opt.get_cooked_value("foo")) - - opt = Option('foo', 'foo') - opt.boolean = False - opt.cook = Mock() - self.assertEqual(opt.cook.return_value, opt.get_cooked_value("foo")) - opt.cook.assert_called_with("foo") - - def test_buildHelpMessage(self): - opt = Option('foo', False) - self.assertEqual(opt.buildHelpMessage(), '') - - opt = Option('foo', False, '-f') - self.assertEqual(opt.buildHelpMessage().split(), - ["-f", "foo"]) - - opt = Option('foo', False, cmd="--foo", long_arg=True) - self.assertEqual(opt.buildHelpMessage().split(), - ["--foo", "foo"]) - - opt = Option('foo', False, cmd="-f", odesc='<val>') - self.assertEqual(opt.buildHelpMessage().split(), - ["-f", "<val>", "foo"]) - - opt = Option('foo', False, cmd="--foo", long_arg=True, odesc='<val>') - self.assertEqual(opt.buildHelpMessage().split(), - ["--foo=<val>", "foo"]) - - def test_buildGetopt(self): - opt = Option('foo', False) - self.assertEqual(opt.buildGetopt(), '') - - opt = Option('foo', False, '-f') - self.assertEqual(opt.buildGetopt(), "f") - - opt = Option('foo', False, cmd="--foo", long_arg=True) - self.assertEqual(opt.buildGetopt(), '') - - opt = Option('foo', False, cmd="-f", odesc='<val>') - self.assertEqual(opt.buildGetopt(), 'f:') - - opt = Option('foo', False, cmd="--foo", long_arg=True, odesc='<val>') - self.assertEqual(opt.buildGetopt(), '') - - def test_buildLongGetopt(self): - opt = Option('foo', False, cmd="--foo", long_arg=True) - self.assertEqual(opt.buildLongGetopt(), 'foo') - - opt = Option('foo', False, cmd="--foo", long_arg=True, odesc='<val>') - self.assertEqual(opt.buildLongGetopt(), 'foo=') - - def test_parse(self): - cf = ('communication', 'password') - o = Option('foo', default='test4', cmd='-F', env='TEST2', - odesc='bar', cf=cf) - o.parse([], ['-F', 'test']) - self.assertEqual(o.value, 'test') - o.parse([('-F', 'test2')], []) - self.assertEqual(o.value, 'test2') - - os.environ['TEST2'] = 'test3' - o.parse([], []) - self.assertEqual(o.value, 'test3') - del os.environ['TEST2'] - - cfp = DefaultConfigParser() - cfp.get = Mock() - cfp.get.return_value = 'test5' - o.parse([], [], configparser=cfp) - cfp.get.assert_any_call(*cf) - self.assertEqual(o.value, 'test5') - - o.cf = False - o.parse([], []) - assert o.value == 'test4' - - -class TestOptionSet(Bcfg2TestCase): - def test_buildGetopt(self): - opts = [('foo', Option('foo', 'test1', cmd='-G')), - ('bar', Option('foo', 'test2')), - ('baz', Option('foo', 'test1', cmd='-H', - odesc='1'))] - oset = OptionSet(opts) - res = oset.buildGetopt() - self.assertIn('H:', res) - self.assertIn('G', res) - self.assertEqual(len(res), 3) - - def test_buildLongGetopt(self): - opts = [('foo', Option('foo', 'test1', cmd='-G')), - ('bar', Option('foo', 'test2')), - ('baz', Option('foo', 'test1', cmd='--H', - odesc='1', long_arg=True))] - oset = OptionSet(opts) - res = oset.buildLongGetopt() - self.assertIn('H=', res) - self.assertEqual(len(res), 1) - - def test_parse(self): - opts = [('foo', Option('foo', 'test1', cmd='-G')), - ('bar', Option('foo', 'test2')), - ('baz', Option('foo', 'test1', cmd='-H', - odesc='1'))] - oset = OptionSet(opts) - self.assertRaises(SystemExit, - oset.parse, - ['-G', '-H']) - oset2 = OptionSet(opts) - self.assertRaises(SystemExit, - oset2.parse, - ['-h']) - oset3 = OptionSet(opts) - oset3.parse(['-G']) - self.assertTrue(oset3['foo']) - - -class TestOptionParser(Bcfg2TestCase): - def test__init(self): - opts = [('foo', Option('foo', 'test1', cmd='-h')), - ('bar', Option('foo', 'test2')), - ('baz', Option('foo', 'test1', cmd='-H', - odesc='1'))] - oset1 = OptionParser(opts) - self.assertEqual(oset1.cfile, - DEFAULT_CONFIG_LOCATION) - sys.argv = ['foo', '-C', '/usr/local/etc/bcfg2.conf'] - oset2 = OptionParser(opts) - self.assertEqual(oset2.cfile, - '/usr/local/etc/bcfg2.conf') - sys.argv = [] - oset3 = OptionParser(opts) - self.assertEqual(oset3.cfile, - DEFAULT_CONFIG_LOCATION) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestCache.py b/testsuite/Testsrc/Testlib/TestServer/TestCache.py new file mode 100644 index 000000000..7c26e52b8 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestCache.py @@ -0,0 +1,54 @@ +import os +import sys + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * + +from Bcfg2.Server.Cache import * + + +class TestCache(Bcfg2TestCase): + def test_cache(self): + md_cache = Cache("Metadata") + md_cache['foo.example.com'] = 'foo metadata' + md_cache['bar.example.com'] = 'bar metadata' + self.assertItemsEqual(list(iter(md_cache)), + ["foo.example.com", "bar.example.com"]) + + probe_cache = Cache("Probes", "data") + probe_cache['foo.example.com'] = 'foo probe data' + probe_cache['bar.example.com'] = 'bar probe data' + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com", "bar.example.com"]) + + md_cache.expire("foo.example.com") + self.assertItemsEqual(list(iter(md_cache)), ["bar.example.com"]) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com", "bar.example.com"]) + + probe_cache.expire("bar.example.com") + self.assertItemsEqual(list(iter(md_cache)), ["bar.example.com"]) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com"]) + + probe_cache['bar.example.com'] = 'bar probe data' + self.assertItemsEqual(list(iter(md_cache)), ["bar.example.com"]) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com", "bar.example.com"]) + + expire("bar.example.com") + self.assertEqual(len(md_cache), 0) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com"]) + + probe_cache2 = Cache("Probes", "data") + self.assertItemsEqual(list(iter(probe_cache)), + list(iter(probe_cache2))) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py b/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py new file mode 100644 index 000000000..3da323262 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +import os +import sys +from Bcfg2.Compat import b64decode, b64encode +from mock import Mock, MagicMock, patch + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * + +try: + from Bcfg2.Server.Encryption import * + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False + + +class TestEncryption(Bcfg2TestCase): + plaintext = """foo bar +baz +ö +\t\tquux +""" + "a" * 16384 # 16K is completely arbitrary + iv = "0123456789ABCDEF" + salt = "01234567" + algo = "des_cbc" + + @skipUnless(HAS_CRYPTO, "Encryption libraries not found") + def setUp(self): + Bcfg2.Options.setup.algorithm = "aes_256_cbc" + + def test_str_crypt(self): + """ test str_encrypt/str_decrypt """ + key = "a simple key" + + # simple symmetrical test with no options + crypted = str_encrypt(self.plaintext, key) + self.assertEqual(self.plaintext, str_decrypt(crypted, key)) + + # symmetrical test with lots of options + crypted = str_encrypt(self.plaintext, key, + iv=self.iv, salt=self.salt, + algorithm=self.algo) + self.assertEqual(self.plaintext, + str_decrypt(crypted, key, iv=self.iv, + algorithm=self.algo)) + + # test that different algorithms are actually used + self.assertNotEqual(str_encrypt(self.plaintext, key), + str_encrypt(self.plaintext, key, + algorithm=self.algo)) + + # test that different keys are actually used + self.assertNotEqual(str_encrypt(self.plaintext, key), + str_encrypt(self.plaintext, "different key")) + + # test that different IVs are actually used + self.assertNotEqual(str_encrypt(self.plaintext, key, iv=self.iv), + str_encrypt(self.plaintext, key)) + + # test that errors are raised on bad decrypts + crypted = str_encrypt(self.plaintext, key, algorithm=self.algo) + self.assertRaises(EVPError, str_decrypt, + crypted, "bogus key", algorithm=self.algo) + self.assertRaises(EVPError, str_decrypt, + crypted, key) # bogus algorithm + + def test_ssl_crypt(self): + """ test ssl_encrypt/ssl_decrypt """ + passwd = "a simple passphrase" + + # simple symmetrical test + crypted = ssl_encrypt(self.plaintext, passwd) + self.assertEqual(self.plaintext, ssl_decrypt(crypted, passwd)) + + # more complex symmetrical test + crypted = ssl_encrypt(self.plaintext, passwd, algorithm=self.algo, + salt=self.salt) + self.assertEqual(self.plaintext, + ssl_decrypt(crypted, passwd, algorithm=self.algo)) + + # test that different algorithms are actually used + self.assertNotEqual(ssl_encrypt(self.plaintext, passwd), + ssl_encrypt(self.plaintext, passwd, + algorithm=self.algo)) + + # test that different passwords are actually used + self.assertNotEqual(ssl_encrypt(self.plaintext, passwd), + ssl_encrypt(self.plaintext, "different pass")) + + # there's no reasonable test we can do to see if the + # output is base64-encoded, unfortunately, but if it's + # obviously not we fail + crypted = ssl_encrypt(self.plaintext, passwd) + self.assertRegexpMatches(crypted, r'^[A-Za-z0-9+/]+[=]{0,2}$') + + # test that errors are raised on bad decrypts + crypted = ssl_encrypt(self.plaintext, passwd, + algorithm=self.algo) + self.assertRaises(EVPError, ssl_decrypt, + crypted, "bogus passwd", algorithm=self.algo) + self.assertRaises(EVPError, ssl_decrypt, + crypted, passwd) # bogus algorithm + + def test_bruteforce_decrypt(self): + passwd = "a simple passphrase" + crypted = ssl_encrypt(self.plaintext, passwd) + + # test with no passphrases given nor in config + Bcfg2.Options.setup.passphrases = dict() + self.assertRaises(EVPError, + bruteforce_decrypt, crypted) + + # test with good passphrase given in function call + self.assertEqual(self.plaintext, + bruteforce_decrypt(crypted, + passphrases=["bogus pass", + passwd, + "also bogus"])) + + # test with no good passphrase given nor in config. we use + # something that isn't a valid ciphertext here since a + # ciphertext encrypted with one key may be technically + # decryptable with a different key, although it will decrypt + # to gibberish. nonetheless, it doesn't raise the requisite + # EVPError, so the test fails. + self.assertRaises(EVPError, + bruteforce_decrypt, + b64encode("not an actual ciphertext!"), + passphrases=["bogus", "also bogus"]) + + # test with no good passphrase given nor in config. this + # version of the test uses a valid ciphertext, and looks for + # *either* EVPError or a failed decrypt. + try: + plaintext = bruteforce_decrypt(crypted, + passphrases=["bogus", "also bogus"]) + if plaintext == passwd: + self.fail("Successfully decrypted ciphertext with wrong key") + except EVPError: + # success! + pass + + # test with good passphrase in config file + Bcfg2.Options.setup.passphrases = dict(bogus="bogus", + real=passwd, + bogus2="also bogus") + self.assertEqual(self.plaintext, + bruteforce_decrypt(crypted)) + + # test that passphrases given in function call take + # precedence over config + self.assertRaises(EVPError, + bruteforce_decrypt, crypted, + passphrases=["bogus", "also bogus"]) + + # test that different algorithms are used + crypted = ssl_encrypt(self.plaintext, passwd, algorithm=self.algo) + self.assertEqual(self.plaintext, + bruteforce_decrypt(crypted, algorithm=self.algo)) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py index 870983f60..f135a0197 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py @@ -1,6 +1,5 @@ import os import sys -import logging from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugin.base import * @@ -14,62 +13,23 @@ while path != '/': break path = os.path.dirname(path) from common import * - - -class TestDebuggable(Bcfg2TestCase): - test_obj = Debuggable - - def get_obj(self): - return self.test_obj() - - def test__init(self): - d = self.get_obj() - self.assertIsInstance(d.logger, logging.Logger) - self.assertFalse(d.debug_flag) - - def test_set_debug(self): - d = self.get_obj() - self.assertEqual(True, d.set_debug(True)) - self.assertEqual(d.debug_flag, True) - - self.assertEqual(False, d.set_debug(False)) - self.assertEqual(d.debug_flag, False) - - def test_toggle_debug(self): - d = self.get_obj() - d.set_debug = Mock() - orig = d.debug_flag - self.assertEqual(d.toggle_debug(), - d.set_debug.return_value) - d.set_debug.assert_called_with(not orig) - - def test_debug_log(self): - d = self.get_obj() - d.logger = Mock() - d.debug_flag = False - d.debug_log("test") - self.assertFalse(d.logger.error.called) - - d.logger.reset_mock() - d.debug_log("test", flag=True) - self.assertTrue(d.logger.error.called) - - d.logger.reset_mock() - d.debug_flag = True - d.debug_log("test") - self.assertTrue(d.logger.error.called) +from TestLogger import TestDebuggable class TestPlugin(TestDebuggable): test_obj = Plugin + def setUp(self): + TestDebuggable.setUp(self) + set_setup_default("filemonitor", MagicMock()) + def get_obj(self, core=None): if core is None: core = Mock() - core.setup = MagicMock() + @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock()) def inner(): - return self.test_obj(core, datastore) + return self.test_obj(core) return inner() @patch("os.makedirs") diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index ce17cb076..37beaa26c 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -1,8 +1,10 @@ import os import sys import copy +import genshi import lxml.etree import Bcfg2.Server +import genshi.core from Bcfg2.Compat import reduce from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugin.helpers import * @@ -21,6 +23,11 @@ from common import * from TestServer.TestPlugin.Testbase import TestPlugin, TestDebuggable from TestServer.TestPlugin.Testinterfaces import TestGenerator +try: + from Bcfg2.Server.Encryption import EVPError +except: + pass + def tostring(el): return lxml.etree.tostring(el, xml_declaration=False).decode('UTF-8') @@ -32,81 +39,41 @@ class FakeElementTree(lxml.etree._ElementTree): class TestFunctions(Bcfg2TestCase): - def test_bind_info(self): - entry = lxml.etree.Element("Path", name="/test") - metadata = Mock() - default = dict(test1="test1", test2="test2") - # test without infoxml - bind_info(entry, metadata, default=default) - self.assertItemsEqual(entry.attrib, - dict(test1="test1", - test2="test2", - name="/test")) - - # test with bogus infoxml - entry = lxml.etree.Element("Path", name="/test") - infoxml = Mock() - self.assertRaises(PluginExecutionError, - bind_info, - entry, metadata, infoxml=infoxml) - infoxml.pnode.Match.assert_called_with(metadata, dict(), entry=entry) - - # test with valid infoxml - entry = lxml.etree.Element("Path", name="/test") - infoxml.reset_mock() - infodata = {None: {"test3": "test3", "test4": "test4"}} - def infoxml_rv(metadata, rv, entry=None): - rv['Info'] = infodata - infoxml.pnode.Match.side_effect = infoxml_rv - bind_info(entry, metadata, infoxml=infoxml, default=default) - # mock objects don't properly track the called-with value of - # arguments whose value is changed by the function, so it - # thinks Match() was called with the final value of the mdata - # arg, not the initial value. makes this test a little less - # worthwhile, TBH. - infoxml.pnode.Match.assert_called_with(metadata, dict(Info=infodata), - entry=entry) - self.assertItemsEqual(entry.attrib, - dict(test1="test1", - test2="test2", - test3="test3", - test4="test4", - name="/test")) + def test_removecomment(self): + data = [(None, "test", 1), + (None, "test2", 2)] + stream = [(genshi.core.COMMENT, "test", 0), + data[0], + (genshi.core.COMMENT, "test3", 0), + data[1]] + self.assertItemsEqual(list(removecomment(stream)), data) class TestDatabaseBacked(TestPlugin): test_obj = DatabaseBacked - def get_obj(self, core=None): - if not HAS_DJANGO: - if core is None: - core = MagicMock() - # disable the database - core.setup.cfp.getboolean.return_value = False - return TestPlugin.get_obj(self, core=core) + def setUp(self): + TestPlugin.setUp(self) + set_setup_default("%s_db" % self.test_obj.__name__.lower(), False) @skipUnless(HAS_DJANGO, "Django not found") def test__use_db(self): core = Mock() - core.setup.cfp.getboolean.return_value = True - db = self.get_obj(core) + db = self.get_obj(core=core) + attr = "%s_db" % self.test_obj.__name__.lower() + + db.core.database_available = True + setattr(Bcfg2.Options.setup, attr, True) self.assertTrue(db._use_db) - core = Mock() - core.setup.cfp.getboolean.return_value = False - db = self.get_obj(core) + setattr(Bcfg2.Options.setup, attr, False) self.assertFalse(db._use_db) - Bcfg2.Server.Plugin.helpers.HAS_DJANGO = False - core = Mock() - core.setup.cfp.getboolean.return_value = False - db = self.get_obj(core) + db.core.database_available = False self.assertFalse(db._use_db) - core = Mock() - core.setup.cfp.getboolean.return_value = True + setattr(Bcfg2.Options.setup, attr, True) self.assertRaises(PluginInitError, self.get_obj, core) - Bcfg2.Server.Plugin.helpers.HAS_DJANGO = True class TestPluginDatabaseModel(Bcfg2TestCase): @@ -114,14 +81,18 @@ class TestPluginDatabaseModel(Bcfg2TestCase): pass -class TestFileBacked(Bcfg2TestCase): +class TestFileBacked(TestDebuggable): test_obj = FileBacked path = os.path.join(datastore, "test") - def get_obj(self, path=None, fam=None): + def setUp(self): + TestDebuggable.setUp(self) + set_setup_default("filemonitor", MagicMock()) + + def get_obj(self, path=None): if path is None: path = self.path - return self.test_obj(path, fam=fam) + return self.test_obj(path) @patch("%s.open" % builtins) def test_HandleEvent(self, mock_open): @@ -149,7 +120,7 @@ class TestFileBacked(Bcfg2TestCase): self.assertFalse(fb.Index.called) -class TestDirectoryBacked(Bcfg2TestCase): +class TestDirectoryBacked(TestDebuggable): test_obj = DirectoryBacked testpaths = {1: '', 2: '/foo', @@ -163,6 +134,10 @@ class TestDirectoryBacked(Bcfg2TestCase): badevents = [] # DirectoryBacked handles all files, so there's no # such thing as a bad event + def setUp(self): + TestDebuggable.setUp(self) + set_setup_default("filemonitor", MagicMock()) + def test_child_interface(self): """ ensure that the child object has the correct interface """ self.assertTrue(hasattr(self.test_obj.__child__, "HandleEvent")) @@ -177,8 +152,7 @@ class TestDirectoryBacked(Bcfg2TestCase): Mock()) def inner(): return self.test_obj(os.path.join(datastore, - self.test_obj.__name__), - fam) + self.test_obj.__name__)) return inner() @patch("os.makedirs") @@ -187,8 +161,8 @@ class TestDirectoryBacked(Bcfg2TestCase): @patch("%s.%s.add_directory_monitor" % (self.test_obj.__module__, self.test_obj.__name__)) def inner(mock_add_monitor): + db = self.test_obj(datastore) mock_exists.return_value = True - db = self.test_obj(datastore, Mock()) mock_add_monitor.assert_called_with('') mock_exists.assert_called_with(db.data) self.assertFalse(mock_makedirs.called) @@ -197,7 +171,7 @@ class TestDirectoryBacked(Bcfg2TestCase): mock_exists.reset_mock() mock_makedirs.reset_mock() mock_exists.return_value = False - db = self.test_obj(datastore, Mock()) + db = self.test_obj(datastore) mock_add_monitor.assert_called_with('') mock_exists.assert_called_with(db.data) mock_makedirs.assert_called_with(db.data) @@ -268,10 +242,9 @@ class TestDirectoryBacked(Bcfg2TestCase): db.fam = Mock() class MockChild(Mock): - def __init__(self, path, fam, **kwargs): + def __init__(self, path, **kwargs): Mock.__init__(self, **kwargs) self.path = path - self.fam = fam self.HandleEvent = Mock() db.__child__ = MockChild @@ -281,7 +254,6 @@ class TestDirectoryBacked(Bcfg2TestCase): self.assertIn(path, db.entries) self.assertEqual(db.entries[path].path, os.path.join(db.data, path)) - self.assertEqual(db.entries[path].fam, db.fam) db.entries[path].HandleEvent.assert_called_with(event) @patch("os.path.isdir") @@ -419,28 +391,31 @@ class TestXMLFileBacked(TestFileBacked): should_monitor = None path = os.path.join(datastore, "test", "test1.xml") - def get_obj(self, path=None, fam=None, should_monitor=False): + def setUp(self): + TestFileBacked.setUp(self) + set_setup_default("encoding", 'utf-8') + + def get_obj(self, path=None, should_monitor=False): if path is None: path = self.path @patchIf(not isinstance(os.path.exists, Mock), "os.path.exists", Mock()) def inner(): - return self.test_obj(path, fam=fam, should_monitor=should_monitor) + return self.test_obj(path, should_monitor=should_monitor) return inner() - def test__init(self): - fam = Mock() + @patch("Bcfg2.Server.FileMonitor.get_fam") + def test__init(self, mock_get_fam): xfb = self.get_obj() + self.assertEqual(xfb.fam, mock_get_fam.return_value) + if self.should_monitor: - self.assertIsNotNone(xfb.fam) - fam.reset_mock() - xfb = self.get_obj(fam=fam, should_monitor=True) - fam.AddMonitor.assert_called_with(self.path, xfb) + xfb = self.get_obj(should_monitor=True) + xfb.fam.AddMonitor.assert_called_with(self.path, xfb) else: - self.assertIsNone(xfb.fam) - xfb = self.get_obj(fam=fam) - self.assertFalse(fam.AddMonitor.called) + xfb = self.get_obj() + self.assertFalse(xfb.fam.AddMonitor.called) @patch("glob.glob") @patch("lxml.etree.parse") @@ -609,6 +584,7 @@ class TestXMLFileBacked(TestFileBacked): test3 = lxml.etree.Element("Test", name="test3") replacements = {"/test/test2.xml": test2, "/test/test_dir/test3.xml": test3} + def xinclude(): for el in xfb.xdata.findall('//%sinclude' % Bcfg2.Server.XI_NAMESPACE): @@ -626,23 +602,26 @@ class TestXMLFileBacked(TestFileBacked): self.assertItemsEqual([tostring(e) for e in xfb.entries], [tostring(e) for e in children]) + @patch("Bcfg2.Server.FileMonitor.get_fam", Mock()) def test_add_monitor(self): xfb = self.get_obj() xfb.add_monitor("/test/test2.xml") self.assertIn("/test/test2.xml", xfb.extra_monitors) - fam = Mock() - fam.reset_mock() - xfb = self.get_obj(fam=fam) - if xfb.fam: - xfb.add_monitor("/test/test4.xml") - fam.AddMonitor.assert_called_with("/test/test4.xml", xfb) - self.assertIn("/test/test4.xml", xfb.extra_monitors) + xfb = self.get_obj() + xfb.fam = Mock() + xfb.add_monitor("/test/test4.xml") + xfb.fam.AddMonitor.assert_called_with("/test/test4.xml", xfb) + self.assertIn("/test/test4.xml", xfb.extra_monitors) class TestStructFile(TestXMLFileBacked): test_obj = StructFile + def setUp(self): + TestXMLFileBacked.setUp(self) + set_setup_default("lax_decryption", False) + def _get_test_data(self): """ build a very complex set of test data """ # top-level group and client elements @@ -684,7 +663,8 @@ class TestStructFile(TestXMLFileBacked): lxml.etree.SubElement(groups[1], "Child", name="c3") lxml.etree.SubElement(groups[1], "Child", name="c4") - standalone.append(lxml.etree.SubElement(xdata, "Standalone", name="s1")) + standalone.append(lxml.etree.SubElement(xdata, + "Standalone", name="s1")) groups[2] = lxml.etree.SubElement(xdata, "Client", name="client2", include="false") @@ -706,12 +686,124 @@ class TestStructFile(TestXMLFileBacked): subchildren[3] = [] lxml.etree.SubElement(children[3][-1], "SubChild", name="subchild") - standalone.append(lxml.etree.SubElement(xdata, "Standalone", name="s3")) + standalone.append(lxml.etree.SubElement(xdata, + "Standalone", name="s3")) lxml.etree.SubElement(standalone[-1], "SubStandalone", name="sub1") - children[4] = standalone return (xdata, groups, subgroups, children, subchildren, standalone) + def _get_template_test_data(self): + (xdata, groups, subgroups, children, subchildren, standalone) = \ + self._get_test_data() + template_xdata = \ + lxml.etree.Element("Test", name="test", + nsmap=dict(py='http://genshi.edgewall.org/')) + template_xdata.extend(xdata.getchildren()) + return (template_xdata, groups, subgroups, children, subchildren, + standalone) + + @patch("genshi.template.TemplateLoader") + def test_Index(self, mock_TemplateLoader): + TestXMLFileBacked.test_Index(self) + + sf = self.get_obj() + sf.encryption = False + sf.encoding = Mock() + (xdata, groups, subgroups, children, subchildren, standalone) = \ + self._get_test_data() + sf.data = lxml.etree.tostring(xdata) + + mock_TemplateLoader.reset_mock() + sf.Index() + self.assertFalse(mock_TemplateLoader.called) + + mock_TemplateLoader.reset_mock() + template_xdata = \ + lxml.etree.Element("Test", name="test", + nsmap=dict(py='http://genshi.edgewall.org/')) + template_xdata.extend(xdata.getchildren()) + sf.data = lxml.etree.tostring(template_xdata) + sf.Index() + mock_TemplateLoader.assert_called_with() + loader = mock_TemplateLoader.return_value + loader.load.assert_called_with(sf.name, + cls=genshi.template.MarkupTemplate, + encoding=Bcfg2.Options.setup.encoding) + self.assertEqual(sf.template, + loader.load.return_value) + + @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping") + def test_Index_crypto(self): + if not self.test_obj.encryption: + return + Bcfg2.Options.setup.lax_decryption = False + sf = self.get_obj() + sf._decrypt = Mock() + sf._decrypt.return_value = 'plaintext' + sf.data = ''' +<EncryptedData> + <Group name="test"> + <Datum encrypted="foo">crypted</Datum> + </Group> + <Group name="test" negate="true"> + <Datum>plain</Datum> + </Group> +</EncryptedData>''' + + # test successful decryption + sf.Index() + self.assertItemsEqual( + sf._decrypt.call_args_list, + [call(el) for el in sf.xdata.xpath("//*[@encrypted]")]) + for el in sf.xdata.xpath("//*[@encrypted]"): + self.assertEqual(el.text, sf._decrypt.return_value) + + # test failed decryption, strict + sf._decrypt.reset_mock() + sf._decrypt.side_effect = EVPError + self.assertRaises(PluginExecutionError, sf.Index) + + # test failed decryption, lax + Bcfg2.Options.setup.lax_decryption = True + sf._decrypt.reset_mock() + sf.Index() + self.assertItemsEqual( + sf._decrypt.call_args_list, + [call(el) for el in sf.xdata.xpath("//*[@encrypted]")]) + + @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping") + @patchIf(HAS_CRYPTO, "Bcfg2.Server.Encryption.ssl_decrypt") + def test_decrypt(self, mock_ssl): + sf = self.get_obj() + + def reset(): + mock_ssl.reset_mock() + + # test element without text contents + Bcfg2.Options.setup.passphrases = dict() + self.assertIsNone(sf._decrypt(lxml.etree.Element("Test"))) + self.assertFalse(mock_ssl.called) + + # test element with a passphrase in the config file + reset() + el = lxml.etree.Element("Test", encrypted="foo") + el.text = "crypted" + Bcfg2.Options.setup.passphrases = dict(foo="foopass", bar="barpass") + mock_ssl.return_value = "decrypted with ssl" + self.assertEqual(sf._decrypt(el), mock_ssl.return_value) + mock_ssl.assert_called_with(el.text, "foopass") + + # test element without valid passphrase + reset() + el.set("encrypted", "true") + self.assertRaises(EVPError, sf._decrypt, el) + self.assertFalse(mock_ssl.called) + + # test failure to decrypt element with a passphrase in the config + reset() + mock_ssl.side_effect = EVPError + self.assertRaises(EVPError, sf._decrypt, el) + def test_include_element(self): sf = self.get_obj() metadata = Mock() @@ -744,22 +836,63 @@ class TestStructFile(TestXMLFileBacked): self.assertTrue(inc("Other")) - @patch("Bcfg2.Server.Plugin.helpers.%s._include_element" % - test_obj.__name__) - def test__match(self, mock_include): + def test__match(self): sf = self.get_obj() + sf._include_element = Mock() metadata = Mock() - (xdata, groups, subgroups, children, subchildren, standalone) = \ - self._get_test_data() - - mock_include.side_effect = \ - lambda x, _: (x.tag not in ['Client', 'Group'] or + sf._include_element.side_effect = \ + lambda x, _: (x.tag not in sf._include_tests.keys() or x.get("include") == "true") - for i, group in groups.items(): - actual = sf._match(group, metadata) - expected = children[i] + subchildren[i] + for test_data in [self._get_test_data(), + self._get_template_test_data()]: + (xdata, groups, subgroups, children, subchildren, standalone) = \ + test_data + + for i, group in groups.items(): + actual = sf._match(group, metadata) + expected = children[i] + subchildren[i] + self.assertEqual(len(actual), len(expected)) + # easiest way to compare the values is actually to make + # them into an XML document and let assertXMLEqual compare + # them + xactual = lxml.etree.Element("Container") + xactual.extend(actual) + xexpected = lxml.etree.Element("Container") + xexpected.extend(expected) + self.assertXMLEqual(xactual, xexpected) + + for el in standalone: + self.assertXMLEqual(el, sf._match(el, metadata)[0]) + + def test_do_match(self): + Bcfg2.Options.setup.lax_decryption = True + sf = self.get_obj() + sf._match = Mock() + + def match_rv(el, _): + if el.tag not in sf._include_tests.keys(): + return [el] + elif el.get("include") == "true": + return el.getchildren() + else: + return [] + sf._match.side_effect = match_rv + + metadata = Mock() + + for test_data in [self._get_test_data(), + self._get_template_test_data()]: + (xdata, groups, subgroups, children, subchildren, standalone) = \ + test_data + sf.data = lxml.etree.tostring(xdata) + sf.Index() + + actual = sf._do_match(metadata) + expected = reduce(lambda x, y: x + y, + list(children.values()) + \ + list(subgroups.values())) + standalone self.assertEqual(len(actual), len(expected)) # easiest way to compare the values is actually to make # them into an XML document and let assertXMLEqual compare @@ -770,428 +903,263 @@ class TestStructFile(TestXMLFileBacked): xexpected.extend(expected) self.assertXMLEqual(xactual, xexpected) - for el in standalone: - self.assertXMLEqual(el, sf._match(el, metadata)[0]) + def test__xml_match(self): + sf = self.get_obj() + sf._include_element = Mock() + metadata = Mock() + + sf._include_element.side_effect = \ + lambda x, _: (x.tag not in sf._include_tests.keys() or + x.get("include") == "true") - @patch("Bcfg2.Server.Plugin.helpers.%s._match" % test_obj.__name__) - def test_Match(self, mock_match): + for test_data in [self._get_test_data(), + self._get_template_test_data()]: + (xdata, groups, subgroups, children, subchildren, standalone) = \ + test_data + + actual = copy.deepcopy(xdata) + for el in actual.getchildren(): + sf._xml_match(el, metadata) + expected = lxml.etree.Element(xdata.tag, **dict(xdata.attrib)) + expected.text = xdata.text + expected.extend(reduce(lambda x, y: x + y, + list(children.values()) + \ + list(subchildren.values()))) + expected.extend(standalone) + self.assertXMLEqual(actual, expected) + + def test_do_xmlmatch(self): sf = self.get_obj() + sf._xml_match = Mock() metadata = Mock() - (xdata, groups, subgroups, children, subchildren, standalone) = \ - self._get_test_data() - sf.entries.extend(copy.deepcopy(xdata).getchildren()) + for data_type, test_data in \ + [("", self._get_test_data()), + ("templated ", self._get_template_test_data())]: + (xdata, groups, subgroups, children, subchildren, standalone) = \ + test_data + sf.xdata = xdata + sf._xml_match.reset_mock() + + sf._do_xmlmatch(metadata) + actual = [] + for call in sf._xml_match.call_args_list: + actual.append(call[0][0]) + self.assertEqual(call[0][1], metadata) + expected = list(groups.values()) + standalone + # easiest way to compare the values is actually to make + # them into an XML document and let assertXMLEqual compare + # them + xactual = lxml.etree.Element("Container") + xactual.extend(actual) + xexpected = lxml.etree.Element("Container") + xexpected.extend(expected) + self.assertXMLEqual(xactual, xexpected, + "XMLMatch() calls were incorrect for " + "%stest data" % data_type) + + def test_match_ordering(self): + """ Match() returns elements in document order """ + Bcfg2.Options.setup.lax_decryption = True + sf = self.get_obj() + sf._match = Mock() def match_rv(el, _): - if el.tag not in ['Client', 'Group']: + if el.tag not in sf._include_tests.keys(): return [el] elif el.get("include") == "true": return el.getchildren() else: return [] - mock_match.side_effect = match_rv - actual = sf.Match(metadata) - expected = reduce(lambda x, y: x + y, - list(children.values()) + list(subgroups.values())) - self.assertEqual(len(actual), len(expected)) - # easiest way to compare the values is actually to make - # them into an XML document and let assertXMLEqual compare - # them - xactual = lxml.etree.Element("Container") - xactual.extend(actual) - xexpected = lxml.etree.Element("Container") - xexpected.extend(expected) - self.assertXMLEqual(xactual, xexpected) - - @patch("Bcfg2.Server.Plugin.helpers.%s._include_element" % - test_obj.__name__) - def test__xml_match(self, mock_include): - sf = self.get_obj() + sf._match.side_effect = match_rv + metadata = Mock() + test_data = lxml.etree.Element("Test") + group = lxml.etree.SubElement(test_data, "Group", name="group", + include="true") + first = lxml.etree.SubElement(group, "Element", name="first") + second = lxml.etree.SubElement(test_data, "Element", name="second") + + # sanity check to ensure that first and second are in the + # correct document order + if test_data.xpath("//Element") != [first, second]: + skip("lxml.etree does not construct documents in a reliable order") + + sf.data = lxml.etree.tostring(test_data) + sf.Index() + rv = sf._do_match(metadata) + self.assertEqual(len(rv), 2, + "Match() seems to be broken, cannot test ordering") + msg = "Match() does not return elements in document order:\n" + \ + "Expected: [%s, %s]\n" % (first, second) + \ + "Actual: %s" % rv + self.assertXMLEqual(rv[0], first, msg) + self.assertXMLEqual(rv[1], second, msg) + + # TODO: add tests to ensure that XMLMatch() returns elements + # in document order + + +class TestInfoXML(TestStructFile): + test_obj = InfoXML + + def _get_test_data(self): (xdata, groups, subgroups, children, subchildren, standalone) = \ - self._get_test_data() + TestStructFile._get_test_data(self) + idx = max(groups.keys()) + 1 + groups[idx] = lxml.etree.SubElement( + xdata, "Path", name="path1", include="true") + children[idx] = [lxml.etree.SubElement(groups[idx], "Child", + name="pc1")] + subgroups[idx] = [lxml.etree.SubElement(groups[idx], "Group", + name="pg1", include="true"), + lxml.etree.SubElement(groups[idx], "Client", + name="pc1", include="false")] + subchildren[idx] = [lxml.etree.SubElement(subgroups[idx][0], + "SubChild", name="sc1")] + + idx += 1 + groups[idx] = lxml.etree.SubElement( + xdata, "Path", name="path2", include="false") + children[idx] = [] + subgroups[idx] = [] + subchildren[idx] = [] + + path2 = lxml.etree.SubElement(groups[0], "Path", name="path2", + include="true") + subgroups[0].append(path2) + subchildren[0].append(lxml.etree.SubElement(path2, "SubChild", + name="sc2")) + return xdata, groups, subgroups, children, subchildren, standalone - mock_include.side_effect = \ - lambda x, _: (x.tag not in ['Client', 'Group'] or - x.get("include") == "true") + def test_include_element(self): + TestStructFile.test_include_element(self) - actual = copy.deepcopy(xdata) - for el in actual.getchildren(): - sf._xml_match(el, metadata) - expected = lxml.etree.Element(xdata.tag, **dict(xdata.attrib)) - expected.text = xdata.text - expected.extend(reduce(lambda x, y: x + y, - list(children.values()) + list(subchildren.values()))) - expected.extend(standalone) - self.assertXMLEqual(actual, expected) - - @patch("Bcfg2.Server.Plugin.helpers.%s._xml_match" % test_obj.__name__) - def test_XMLMatch(self, mock_xml_match): - sf = self.get_obj() + ix = self.get_obj() metadata = Mock() + entry = lxml.etree.Element("Path", name="/etc/foo.conf") + inc = lambda tag, **attrs: \ + ix._include_element(lxml.etree.Element(tag, **attrs), + metadata, entry) + + self.assertFalse(inc("Path", name="/etc/bar.conf")) + self.assertFalse(inc("Path", name="/etc/foo.conf", negate="true")) + self.assertFalse(inc("Path", name="/etc/foo.conf", negate="tRuE")) + self.assertTrue(inc("Path", name="/etc/foo.conf")) + self.assertTrue(inc("Path", name="/etc/foo.conf", negate="false")) + self.assertTrue(inc("Path", name="/etc/foo.conf", negate="faLSe")) + self.assertTrue(inc("Path", name="/etc/bar.conf", negate="true")) + self.assertTrue(inc("Path", name="/etc/bar.conf", negate="tRUe")) - (sf.xdata, groups, subgroups, children, subchildren, standalone) = \ - self._get_test_data() - - sf.XMLMatch(metadata) - actual = [] - for call in mock_xml_match.call_args_list: - actual.append(call[0][0]) - self.assertEqual(call[0][1], metadata) - expected = list(groups.values()) + standalone - # easiest way to compare the values is actually to make - # them into an XML document and let assertXMLEqual compare - # them - xactual = lxml.etree.Element("Container") - xactual.extend(actual) - xexpected = lxml.etree.Element("Container") - xexpected.extend(expected) - self.assertXMLEqual(xactual, xexpected) - - -class TestINode(Bcfg2TestCase): - test_obj = INode - - # INode.__init__ and INode._load_children() call each other - # recursively, which makes this class kind of a nightmare to test. - # we have to first patch INode._load_children so that we can - # create an INode object with no children loaded, then we unpatch - # INode._load_children and patch INode.__init__ so that child - # objects aren't actually created. but in order to test things - # atomically, we do this umpteen times in order to test with - # different data. this convenience method makes this a little - # easier. fun fun fun. - @patch("Bcfg2.Server.Plugin.helpers.%s._load_children" % - test_obj.__name__, Mock()) - def _get_inode(self, data, idict): - return self.test_obj(data, idict) - - def test_raw_predicates(self): + def test_BindEntry(self): + ix = self.get_obj() + entry = lxml.etree.Element("Path", name=self.path) metadata = Mock() - metadata.groups = ["group1", "group2"] - metadata.hostname = "foo.example.com" - entry = None - - parent_predicate = lambda m, e: True - pred = eval(self.test_obj.raw['Client'] % dict(name="foo.example.com"), - dict(predicate=parent_predicate)) - self.assertTrue(pred(metadata, entry)) - pred = eval(self.test_obj.raw['Client'] % dict(name="bar.example.com"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - - pred = eval(self.test_obj.raw['Group'] % dict(name="group1"), - dict(predicate=parent_predicate)) - self.assertTrue(pred(metadata, entry)) - pred = eval(self.test_obj.raw['Group'] % dict(name="group3"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - - pred = eval(self.test_obj.nraw['Client'] % dict(name="foo.example.com"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - pred = eval(self.test_obj.nraw['Client'] % dict(name="bar.example.com"), - dict(predicate=parent_predicate)) - self.assertTrue(pred(metadata, entry)) - - pred = eval(self.test_obj.nraw['Group'] % dict(name="group1"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - pred = eval(self.test_obj.nraw['Group'] % dict(name="group3"), - dict(predicate=parent_predicate)) - self.assertTrue(pred(metadata, entry)) - - parent_predicate = lambda m, e: False - pred = eval(self.test_obj.raw['Client'] % dict(name="foo.example.com"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - pred = eval(self.test_obj.raw['Group'] % dict(name="group1"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - pred = eval(self.test_obj.nraw['Client'] % dict(name="bar.example.com"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - pred = eval(self.test_obj.nraw['Group'] % dict(name="group3"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - - self.assertItemsEqual(self.test_obj.containers, - self.test_obj.raw.keys()) - self.assertItemsEqual(self.test_obj.containers, - self.test_obj.nraw.keys()) - - @patch("Bcfg2.Server.Plugin.helpers.INode._load_children") - def test__init(self, mock_load_children): - data = lxml.etree.Element("Bogus") - # called with no parent, should not raise an exception; it's a - # top-level tag in an XML file and so is not expected to be a - # proper predicate - INode(data, dict()) - self.assertRaises(PluginExecutionError, - INode, data, dict(), Mock()) - data = lxml.etree.Element("Client", name="foo.example.com") - idict = dict() - inode = INode(data, idict) - mock_load_children.assert_called_with(data, idict) - self.assertTrue(inode.predicate(Mock(), Mock())) + # test with bogus infoxml + ix.Match = Mock() + ix.Match.return_value = [] + self.assertRaises(PluginExecutionError, + ix.BindEntry, entry, metadata) + ix.Match.assert_called_with(metadata, entry) - parent = Mock() - parent.predicate = lambda m, e: True - metadata = Mock() - metadata.groups = ["group1", "group2"] - metadata.hostname = "foo.example.com" - entry = None - - # test setting predicate with parent object - mock_load_children.reset_mock() - inode = INode(data, idict, parent=parent) - mock_load_children.assert_called_with(data, idict) - self.assertTrue(inode.predicate(metadata, entry)) - - # test negation - data = lxml.etree.Element("Client", name="foo.example.com", - negate="true") - mock_load_children.reset_mock() - inode = INode(data, idict, parent=parent) - mock_load_children.assert_called_with(data, idict) - self.assertFalse(inode.predicate(metadata, entry)) - - # test failure of a matching predicate (client names do not match) - data = lxml.etree.Element("Client", name="foo.example.com") - metadata.hostname = "bar.example.com" - mock_load_children.reset_mock() - inode = INode(data, idict, parent=parent) - mock_load_children.assert_called_with(data, idict) - self.assertFalse(inode.predicate(metadata, entry)) - - # test that parent predicate is AND'ed in correctly - parent.predicate = lambda m, e: False - metadata.hostname = "foo.example.com" - mock_load_children.reset_mock() - inode = INode(data, idict, parent=parent) - mock_load_children.assert_called_with(data, idict) - self.assertFalse(inode.predicate(metadata, entry)) - - def test_load_children(self): - data = lxml.etree.Element("Parent") - child1 = lxml.etree.SubElement(data, "Client", name="foo.example.com") - child2 = lxml.etree.SubElement(data, "Group", name="bar", negate="true") - idict = dict() - - inode = self._get_inode(data, idict) - - @patch("Bcfg2.Server.Plugin.helpers.%s.__init__" % - inode.__class__.__name__) - def inner(mock_init): - mock_init.return_value = None - inode._load_children(data, idict) - self.assertItemsEqual(mock_init.call_args_list, - [call(child1, idict, inode), - call(child2, idict, inode)]) - self.assertEqual(idict, dict()) - self.assertItemsEqual(inode.contents, dict()) + # test with valid infoxml + ix.Match.reset_mock() + ix.Match.return_value = [lxml.etree.Element("Info", + mode="0600", owner="root")] + ix.BindEntry(entry, metadata) + ix.Match.assert_called_with(metadata, entry) + self.assertItemsEqual(entry.attrib, + dict(name=self.path, mode="0600", owner="root")) - inner() + def _get_test_data(self): + (xdata, groups, subgroups, children, subchildren, standalone) = \ + TestStructFile._get_test_data(self) + idx = max(groups.keys()) + 1 + groups[idx] = lxml.etree.SubElement( + xdata, "Path", name="path1", include="true") + children[idx] = [lxml.etree.SubElement(groups[idx], "Child", + name="pc1")] + subgroups[idx] = [lxml.etree.SubElement(groups[idx], "Group", + name="pg1", include="true"), + lxml.etree.SubElement(groups[idx], "Client", + name="pc1", include="false")] + subchildren[idx] = [lxml.etree.SubElement(subgroups[idx][0], + "SubChild", name="sc1")] + + idx += 1 + groups[idx] = lxml.etree.SubElement( + xdata, "Path", name="path2", include="false") + children[idx] = [] + subgroups[idx] = [] + subchildren[idx] = [] + + path2 = lxml.etree.SubElement(groups[0], "Path", name="path2", + include="true") + subgroups[0].append(path2) + subchildren[0].append(lxml.etree.SubElement(path2, "SubChild", + name="sc2")) + return xdata, groups, subgroups, children, subchildren, standalone - data = lxml.etree.Element("Parent") - child1 = lxml.etree.SubElement(data, "Data", name="child1", - attr="some attr") - child1.text = "text" - subchild1 = lxml.etree.SubElement(child1, "SubChild", name="subchild") - child2 = lxml.etree.SubElement(data, "Group", name="bar", negate="true") - idict = dict() - - inode = self._get_inode(data, idict) - inode.ignore = [] - - @patch("Bcfg2.Server.Plugin.helpers.%s.__init__" % - inode.__class__.__name__) - def inner2(mock_init): - mock_init.return_value = None - inode._load_children(data, idict) - mock_init.assert_called_with(child2, idict, inode) - tag = child1.tag - name = child1.get("name") - self.assertEqual(idict, dict(Data=[name])) - self.assertIn(tag, inode.contents) - self.assertIn(name, inode.contents[tag]) - self.assertItemsEqual(inode.contents[tag][name], - dict(name=name, - attr=child1.get('attr'), - __text__=child1.text, - __children__=[subchild1])) - - inner2() - - # test ignore. no ignore is set on INode by default, so we - # have to set one - old_ignore = copy.copy(self.test_obj.ignore) - self.test_obj.ignore.append("Data") - idict = dict() - - inode = self._get_inode(data, idict) - - @patch("Bcfg2.Server.Plugin.helpers.%s.__init__" % - inode.__class__.__name__) - def inner3(mock_init): - mock_init.return_value = None - inode._load_children(data, idict) - mock_init.assert_called_with(child2, idict, inode) - self.assertEqual(idict, dict()) - self.assertItemsEqual(inode.contents, dict()) - - inner3() - self.test_obj.ignore = old_ignore - - def test_Match(self): - idata = lxml.etree.Element("Parent") - contents = lxml.etree.SubElement(idata, "Data", name="contents", - attr="some attr") - child = lxml.etree.SubElement(idata, "Group", name="bar", negate="true") - - inode = INode(idata, dict()) - inode.predicate = Mock() - inode.predicate.return_value = False + def test_include_element(self): + TestStructFile.test_include_element(self) + ix = self.get_obj() metadata = Mock() - metadata.groups = ['foo'] - data = dict() - entry = child - - inode.Match(metadata, data, entry=child) - self.assertEqual(data, dict()) - inode.predicate.assert_called_with(metadata, child) - - inode.predicate.reset_mock() - inode.Match(metadata, data) - self.assertEqual(data, dict()) - # can't easily compare XML args without the original - # object, and we're testing that Match() works without an - # XML object passed in, so... - self.assertEqual(inode.predicate.call_args[0][0], - metadata) - self.assertXMLEqual(inode.predicate.call_args[0][1], - lxml.etree.Element("None")) - - inode.predicate.reset_mock() - inode.predicate.return_value = True - inode.Match(metadata, data, entry=child) - self.assertEqual(data, inode.contents) - inode.predicate.assert_called_with(metadata, child) - - -class TestInfoNode(TestINode): - __test__ = True - test_obj = InfoNode - - def test_raw_predicates(self): - TestINode.test_raw_predicates(self) - metadata = Mock() - entry = lxml.etree.Element("Path", name="/tmp/foo", - realname="/tmp/bar") - - parent_predicate = lambda m, d: True - pred = eval(self.test_obj.raw['Path'] % dict(name="/tmp/foo"), - dict(predicate=parent_predicate)) - self.assertTrue(pred(metadata, entry)) - pred = eval(InfoNode.raw['Path'] % dict(name="/tmp/bar"), - dict(predicate=parent_predicate)) - self.assertTrue(pred(metadata, entry)) - pred = eval(InfoNode.raw['Path'] % dict(name="/tmp/bogus"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - - pred = eval(self.test_obj.nraw['Path'] % dict(name="/tmp/foo"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - pred = eval(InfoNode.nraw['Path'] % dict(name="/tmp/bar"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - pred = eval(InfoNode.nraw['Path'] % dict(name="/tmp/bogus"), - dict(predicate=parent_predicate)) - self.assertTrue(pred(metadata, entry)) - - parent_predicate = lambda m, d: False - pred = eval(self.test_obj.raw['Path'] % dict(name="/tmp/foo"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - pred = eval(InfoNode.raw['Path'] % dict(name="/tmp/bar"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - pred = eval(InfoNode.nraw['Path'] % dict(name="/tmp/bogus"), - dict(predicate=parent_predicate)) - self.assertFalse(pred(metadata, entry)) - - -class TestXMLSrc(TestXMLFileBacked): - test_obj = XMLSrc - - def test_node_interface(self): - # ensure that the node object has the necessary interface - self.assertTrue(hasattr(self.test_obj.__node__, "Match")) - - @patch("lxml.etree.parse") - def test_HandleEvent(self, mock_parse): - xdata = lxml.etree.Element("Test") - lxml.etree.SubElement(xdata, "Path", name="path", attr="whatever") - - xsrc = self.get_obj("/test/foo.xml") - xsrc.__node__ = Mock() - mock_parse.return_value = xdata.getroottree() - - if xsrc.__priority_required__: - # test with no priority at all - self.assertRaises(PluginExecutionError, - xsrc.HandleEvent, Mock()) - - # test with bogus priority - xdata.set("priority", "cow") - mock_parse.return_value = xdata.getroottree() - self.assertRaises(PluginExecutionError, - xsrc.HandleEvent, Mock()) - - # assign a priority to use in future tests - xdata.set("priority", "10") - mock_parse.return_value = xdata.getroottree() - - mock_parse.reset_mock() - xsrc = self.get_obj("/test/foo.xml") - xsrc.__node__ = Mock() - xsrc.HandleEvent(Mock()) - mock_parse.assert_called_with("/test/foo.xml", - parser=Bcfg2.Server.XMLParser) - self.assertXMLEqual(xsrc.__node__.call_args[0][0], xdata) - self.assertEqual(xsrc.__node__.call_args[0][1], dict()) - self.assertEqual(xsrc.pnode, xsrc.__node__.return_value) - self.assertEqual(xsrc.cache, None) - - @patch("Bcfg2.Server.Plugin.helpers.XMLSrc.HandleEvent") - def test_Cache(self, mock_HandleEvent): - xsrc = self.get_obj("/test/foo.xml") + entry = lxml.etree.Element("Path", name="/etc/foo.conf") + inc = lambda tag, **attrs: \ + ix._include_element(lxml.etree.Element(tag, **attrs), + metadata, entry) + + self.assertFalse(inc("Path", name="/etc/bar.conf")) + self.assertFalse(inc("Path", name="/etc/foo.conf", negate="true")) + self.assertFalse(inc("Path", name="/etc/foo.conf", negate="tRuE")) + self.assertTrue(inc("Path", name="/etc/foo.conf")) + self.assertTrue(inc("Path", name="/etc/foo.conf", negate="false")) + self.assertTrue(inc("Path", name="/etc/foo.conf", negate="faLSe")) + self.assertTrue(inc("Path", name="/etc/bar.conf", negate="true")) + self.assertTrue(inc("Path", name="/etc/bar.conf", negate="tRUe")) + + def test_include_element_altsrc(self): + ix = self.get_obj() metadata = Mock() - xsrc.Cache(metadata) - mock_HandleEvent.assert_any_call() + entry = lxml.etree.Element("Path", name="/etc/bar.conf", + realname="/etc/foo.conf") + inc = lambda tag, **attrs: \ + ix._include_element(lxml.etree.Element(tag, **attrs), + metadata, entry) - xsrc.pnode = Mock() - xsrc.Cache(metadata) - xsrc.pnode.Match.assert_called_with(metadata, xsrc.__cacheobj__()) - self.assertEqual(xsrc.cache[0], metadata) + self.assertFalse(inc("Path", name="/etc/bar.conf")) + self.assertFalse(inc("Path", name="/etc/foo.conf", negate="true")) + self.assertFalse(inc("Path", name="/etc/foo.conf", negate="tRuE")) + self.assertTrue(inc("Path", name="/etc/foo.conf")) + self.assertTrue(inc("Path", name="/etc/foo.conf", negate="false")) + self.assertTrue(inc("Path", name="/etc/foo.conf", negate="faLSe")) + self.assertTrue(inc("Path", name="/etc/bar.conf", negate="true")) + self.assertTrue(inc("Path", name="/etc/bar.conf", negate="tRUe")) - xsrc.pnode.reset_mock() - xsrc.Cache(metadata) - self.assertFalse(xsrc.pnode.Mock.called) - self.assertEqual(xsrc.cache[0], metadata) - xsrc.cache = ("bogus") - xsrc.Cache(metadata) - xsrc.pnode.Match.assert_called_with(metadata, xsrc.__cacheobj__()) - self.assertEqual(xsrc.cache[0], metadata) + def test_BindEntry(self): + ix = self.get_obj() + entry = lxml.etree.Element("Path", name=self.path) + metadata = Mock() + # test with bogus infoxml + ix.Match = Mock() + ix.Match.return_value = [] + self.assertRaises(PluginExecutionError, + ix.BindEntry, entry, metadata) + ix.Match.assert_called_with(metadata, entry) -class TestInfoXML(TestXMLSrc): - test_obj = InfoXML + # test with valid infoxml + ix.Match.reset_mock() + ix.Match.return_value = [lxml.etree.Element("Info", + mode="0600", owner="root")] + ix.BindEntry(entry, metadata) + ix.Match.assert_called_with(metadata, entry) + self.assertItemsEqual(entry.attrib, + dict(name=self.path, mode="0600", owner="root")) class TestXMLDirectoryBacked(TestDirectoryBacked): @@ -1203,6 +1171,11 @@ class TestXMLDirectoryBacked(TestDirectoryBacked): class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked): test_obj = PrioDir + def setUp(self): + TestPlugin.setUp(self) + TestGenerator.setUp(self) + TestXMLDirectoryBacked.setUp(self) + def get_obj(self, core=None): if core is None: core = Mock() @@ -1212,7 +1185,7 @@ class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked): Mock()) @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock()) def inner(): - return self.test_obj(core, datastore) + return self.test_obj(core) return inner() @@ -1223,13 +1196,20 @@ class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked): Mock()) def inner(): pd = self.get_obj() - test1 = Mock() - test1.items = dict(Path=["/etc/foo.conf", "/etc/bar.conf"]) - test2 = Mock() - test2.items = dict(Path=["/etc/baz.conf"], - Package=["quux", "xyzzy"]) - pd.entries = {"/test1.xml": test1, - "/test2.xml": test2} + test1 = lxml.etree.Element("Test") + lxml.etree.SubElement(test1, "Path", name="/etc/foo.conf") + lxml.etree.SubElement(lxml.etree.SubElement(test1, + "Group", name="foo"), + "Path", name="/etc/bar.conf") + + test2 = lxml.etree.Element("Test") + lxml.etree.SubElement(test2, "Path", name="/etc/baz.conf") + lxml.etree.SubElement(test2, "Package", name="quux") + lxml.etree.SubElement(lxml.etree.SubElement(test2, + "Group", name="bar"), + "Package", name="xyzzy") + pd.entries = {"/test1.xml": Mock(xdata=test1), + "/test2.xml": Mock(xdata=test2)} pd.HandleEvent(Mock()) self.assertItemsEqual(pd.Entries, dict(Path={"/etc/foo.conf": pd.BindEntry, @@ -1242,32 +1222,17 @@ class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked): def test__matches(self): pd = self.get_obj() - self.assertTrue(pd._matches(lxml.etree.Element("Test", - name="/etc/foo.conf"), - Mock(), - {"/etc/foo.conf": pd.BindEntry, - "/etc/bar.conf": pd.BindEntry})) - self.assertFalse(pd._matches(lxml.etree.Element("Test", - name="/etc/baz.conf"), - Mock(), - {"/etc/foo.conf": pd.BindEntry, - "/etc/bar.conf": pd.BindEntry})) + entry = lxml.etree.Element("Test", name="/etc/foo.conf") + self.assertTrue(pd._matches(entry, Mock(), + lxml.etree.Element("Test", + name="/etc/foo.conf"))) + self.assertFalse(pd._matches(entry, Mock(), + lxml.etree.Element("Test", + name="/etc/baz.conf"))) def test_BindEntry(self): pd = self.get_obj() - pd.get_attrs = Mock(return_value=dict(test1="test1", test2="test2")) - entry = lxml.etree.Element("Path", name="/etc/foo.conf", test1="bogus") - metadata = Mock() - pd.BindEntry(entry, metadata) - pd.get_attrs.assert_called_with(entry, metadata) - self.assertItemsEqual(entry.attrib, - dict(name="/etc/foo.conf", - test1="test1", test2="test2")) - - def test_get_attrs(self): - pd = self.get_obj() - entry = lxml.etree.Element("Path", name="/etc/foo.conf") - children = [lxml.etree.Element("Child")] + children = [lxml.etree.Element("Child", name="child")] metadata = Mock() pd.entries = dict() @@ -1275,58 +1240,59 @@ class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked): metadata.reset_mock() for src in pd.entries.values(): src.reset_mock() - src.cache = None # test with no matches - self.assertRaises(PluginExecutionError, - pd.get_attrs, entry, metadata) + self.assertRaises(PluginExecutionError, pd.BindEntry, Mock(), metadata) - def add_entry(name, data, prio=10): + def add_entry(name, data): path = os.path.join(pd.data, name) pd.entries[path] = Mock() - pd.entries[path].priority = prio - def do_Cache(metadata): - pd.entries[path].cache = (metadata, data) - pd.entries[path].Cache.side_effect = do_Cache - - add_entry('test1.xml', - dict(Path={'/etc/foo.conf': dict(attr="attr1", - __children__=children), - '/etc/bar.conf': dict()})) - add_entry('test2.xml', - dict(Path={'/etc/bar.conf': dict(__text__="text", - attr="attr1")}, - Package={'quux': dict(), - 'xyzzy': dict()}), - prio=20) - add_entry('test3.xml', - dict(Path={'/etc/baz.conf': dict()}, - Package={'xyzzy': dict()}), - prio=20) - - # test with exactly one match, __children__ + pd.entries[path].priority = data.get("priority") + pd.entries[path].XMLMatch.return_value = data + + test1 = lxml.etree.Element("Rules", priority="10") + path1 = lxml.etree.SubElement(test1, "Path", name="/etc/foo.conf", + attr="attr1") + path1.extend(children) + lxml.etree.SubElement(test1, "Path", name="/etc/bar.conf") + add_entry('test1.xml', test1) + + test2 = lxml.etree.Element("Rules", priority="20") + path2 = lxml.etree.SubElement(test2, "Path", name="/etc/bar.conf", + attr="attr1") + path2.text = "text" + lxml.etree.SubElement(test2, "Package", name="quux") + lxml.etree.SubElement(test2, "Package", name="xyzzy") + add_entry('test2.xml', test2) + + test3 = lxml.etree.Element("Rules", priority="20") + lxml.etree.SubElement(test3, "Path", name="/etc/baz.conf") + lxml.etree.SubElement(test3, "Package", name="xyzzy") + add_entry('test3.xml', test3) + + # test with exactly one match, children reset() - self.assertItemsEqual(pd.get_attrs(entry, metadata), - dict(attr="attr1")) + entry = lxml.etree.Element("Path", name="/etc/foo.conf") + pd.BindEntry(entry, metadata) + self.assertXMLEqual(entry, path1) + self.assertIsNot(entry, path1) for src in pd.entries.values(): - src.Cache.assert_called_with(metadata) - self.assertEqual(len(entry.getchildren()), 1) - self.assertXMLEqual(entry.getchildren()[0], children[0]) + src.XMLMatch.assert_called_with(metadata) - # test with multiple matches with different priorities, __text__ + # test with multiple matches with different priorities, text reset() entry = lxml.etree.Element("Path", name="/etc/bar.conf") - self.assertItemsEqual(pd.get_attrs(entry, metadata), - dict(attr="attr1")) + pd.BindEntry(entry, metadata) + self.assertXMLEqual(entry, path2) + self.assertIsNot(entry, path2) for src in pd.entries.values(): - src.Cache.assert_called_with(metadata) - self.assertEqual(entry.text, "text") + src.XMLMatch.assert_called_with(metadata) # test with multiple matches with identical priorities reset() entry = lxml.etree.Element("Package", name="xyzzy") self.assertRaises(PluginExecutionError, - pd.get_attrs, entry, metadata) + pd.BindEntry, entry, metadata) class TestSpecificity(Bcfg2TestCase): @@ -1390,16 +1356,20 @@ class TestSpecificity(Bcfg2TestCase): self.assertGreaterEqual(specs[j], specs[i]) -class TestSpecificData(Bcfg2TestCase): +class TestSpecificData(TestDebuggable): test_obj = SpecificData path = os.path.join(datastore, "test.txt") - def get_obj(self, name=None, specific=None, encoding=None): + def setUp(self): + TestDebuggable.setUp(self) + set_setup_default("encoding", "utf-8") + + def get_obj(self, name=None, specific=None): if name is None: name = self.path if specific is None: specific = Mock() - return self.test_obj(name, specific, encoding) + return self.test_obj(name, specific) def test__init(self): pass @@ -1411,10 +1381,10 @@ class TestSpecificData(Bcfg2TestCase): sd = self.get_obj() sd.handle_event(event) self.assertFalse(mock_open.called) - if hasattr(sd, 'data'): - self.assertIsNone(sd.data) - else: + try: self.assertFalse(hasattr(sd, 'data')) + except AssertionError: + self.assertIsNone(sd.data) event = Mock() mock_open.return_value.read.return_value = "test" @@ -1441,9 +1411,18 @@ class TestEntrySet(TestDebuggable): ignore = ["foo~", ".#foo", ".foo.swp", ".foo.swx", "test.txt.genshi_include", "test.G_foo.genshi_include"] - def get_obj(self, basename="test", path=datastore, entry_type=MagicMock(), - encoding=None): - return self.test_obj(basename, path, entry_type, encoding) + def setUp(self): + TestDebuggable.setUp(self) + set_setup_default("default_owner") + set_setup_default("default_group") + set_setup_default("default_mode") + set_setup_default("default_secontext") + set_setup_default("default_important", False) + set_setup_default("default_paranoid", False) + set_setup_default("default_sensitive", False) + + def get_obj(self, basename="test", entry_type=MagicMock()): + return self.test_obj(basename, path, entry_type) def test__init(self): for basename in self.basenames: @@ -1573,25 +1552,25 @@ class TestEntrySet(TestDebuggable): eset.reset_metadata.reset_mock() eset.entry_init.reset_mock() - for fname in ["info", "info.xml", ":info"]: - for evt in ["exists", "created", "changed"]: - reset() - event = Mock() - event.code2str.return_value = evt - event.filename = fname - eset.handle_event(event) - eset.update_metadata.assert_called_with(event) - self.assertFalse(eset.entry_init.called) - self.assertFalse(eset.reset_metadata.called) - + fname = "info.xml" + for evt in ["exists", "created", "changed"]: reset() event = Mock() - event.code2str.return_value = "deleted" + event.code2str.return_value = evt event.filename = fname eset.handle_event(event) - eset.reset_metadata.assert_called_with(event) + eset.update_metadata.assert_called_with(event) self.assertFalse(eset.entry_init.called) - self.assertFalse(eset.update_metadata.called) + self.assertFalse(eset.reset_metadata.called) + + reset() + event = Mock() + event.code2str.return_value = "deleted" + event.filename = fname + eset.handle_event(event) + eset.reset_metadata.assert_called_with(event) + self.assertFalse(eset.entry_init.called) + self.assertFalse(eset.update_metadata.called) for evt in ["exists", "created", "changed"]: reset() @@ -1638,8 +1617,9 @@ class TestEntrySet(TestDebuggable): eset.entry_init(event) eset.specificity_from_filename.assert_called_with("test.txt", specific=None) - eset.entry_type.assert_called_with(os.path.join(eset.path, "test.txt"), - eset.specificity_from_filename.return_value, None) + eset.entry_type.assert_called_with( + os.path.join(eset.path, "test.txt"), + eset.specificity_from_filename.return_value) eset.entry_type.return_value.handle_event.assert_called_with(event) self.assertIn("test.txt", eset.entries) @@ -1660,8 +1640,7 @@ class TestEntrySet(TestDebuggable): eset.specificity_from_filename.assert_called_with("test2.txt", specific=specific) etype.assert_called_with(os.path.join(eset.path, "test2.txt"), - eset.specificity_from_filename.return_value, - None) + eset.specificity_from_filename.return_value) etype.return_value.handle_event.assert_called_with(event) self.assertIn("test2.txt", eset.entries) @@ -1750,26 +1729,8 @@ class TestEntrySet(TestDebuggable): self.assertFalse(mock_InfoXML.called) eset.infoxml.HandleEvent.assert_called_with(event) - for fname in [':info', 'info']: - event = Mock() - event.filename = fname - - idata = ["owner:owner", - "group: GROUP", - "mode: 775", - "important: true", - "bogus: line"] - mock_open.return_value.readlines.return_value = idata - eset.update_metadata(event) - expected = DEFAULT_FILE_METADATA.copy() - expected['owner'] = 'owner' - expected['group'] = 'GROUP' - expected['mode'] = '0775' - expected['important'] = 'true' - self.assertItemsEqual(eset.metadata, - expected) - - def test_reset_metadata(self): + @patch("Bcfg2.Server.Plugin.helpers.default_path_metadata") + def test_reset_metadata(self, mock_default_path_metadata): eset = self.get_obj() # test info.xml @@ -1779,29 +1740,22 @@ class TestEntrySet(TestDebuggable): eset.reset_metadata(event) self.assertIsNone(eset.infoxml) - for fname in [':info', 'info']: - event = Mock() - event.filename = fname - eset.metadata = Mock() - eset.reset_metadata(event) - self.assertItemsEqual(eset.metadata, DEFAULT_FILE_METADATA) - - @patch("Bcfg2.Server.Plugin.helpers.bind_info") - def test_bind_info_to_entry(self, mock_bind_info): - # There's a strange scoping issue in py3k that prevents this - # test from working as expected on sub-classes of EntrySet. - # No idea what's going on, but until I can figure it out we - # skip this test on subclasses - if inPy3k and self.test_obj != EntrySet: - return skip("Skipping this test for py3k scoping issues") - + def test_bind_info_to_entry(self): eset = self.get_obj() - entry = Mock() + eset.metadata = dict(owner="root", group="root") + entry = lxml.etree.Element("Path", name="/test") metadata = Mock() + eset.infoxml = None + eset.bind_info_to_entry(entry, metadata) + self.assertItemsEqual(entry.attrib, + dict(name="/test", owner="root", group="root")) + + entry = lxml.etree.Element("Path", name="/test") + eset.infoxml = Mock() eset.bind_info_to_entry(entry, metadata) - mock_bind_info.assert_called_with(entry, metadata, - infoxml=eset.infoxml, - default=eset.metadata) + self.assertItemsEqual(entry.attrib, + dict(name="/test", owner="root", group="root")) + eset.infoxml.BindEntry.assert_called_with(entry, metadata) def test_bind_entry(self): eset = self.get_obj() @@ -1820,15 +1774,14 @@ class TestEntrySet(TestDebuggable): class TestGroupSpool(TestPlugin, TestGenerator): test_obj = GroupSpool + def setUp(self): + TestPlugin.setUp(self) + TestGenerator.setUp(self) + set_setup_default("encoding", "utf-8") + def get_obj(self, core=None): if core is None: core = MagicMock() - core.setup = MagicMock() - else: - try: - core.setup['encoding'] - except TypeError: - core.setup.__getitem__ = MagicMock() @patch("%s.%s.AddDirectoryMonitor" % (self.test_obj.__module__, self.test_obj.__name__), @@ -1843,7 +1796,7 @@ class TestGroupSpool(TestPlugin, TestGenerator): @patch("%s.%s.AddDirectoryMonitor" % (self.test_obj.__module__, self.test_obj.__name__)) def inner(mock_Add): - gs = self.test_obj(MagicMock(), datastore) + gs = self.test_obj(MagicMock()) mock_Add.assert_called_with('') self.assertItemsEqual(gs.Entries, {gs.entry_type: {}}) @@ -1899,8 +1852,7 @@ class TestGroupSpool(TestPlugin, TestGenerator): self.assertFalse(gs.AddDirectoryMonitor.called) gs.es_cls.assert_called_with(gs.filename_pattern, gs.data + ident, - gs.es_child_cls, - gs.encoding) + gs.es_child_cls) self.assertIn(ident, gs.entries) self.assertEqual(gs.entries[ident], gs.es_cls.return_value) self.assertIn(ident, gs.Entries[gs.entry_type]) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py index 1f5c4790b..bbfb495c4 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py @@ -357,3 +357,20 @@ class TestVersion(TestPlugin): class TestClientRunHooks(Bcfg2TestCase): """ placeholder for future tests """ pass + + +class TestClientACLs(Bcfg2TestCase): + test_obj = ClientACLs + + def get_obj(self): + return self.test_obj() + + def test_check_acl_ip(self): + ca = self.get_obj() + self.assertIn(ca.check_acl_ip(Mock(), Mock()), + [True, False, None]) + + def test_check_acl_metadata(self): + ca = self.get_obj() + self.assertIn(ca.check_acl_metadata(Mock(), Mock()), + [True, False]) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestACL.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestACL.py new file mode 100644 index 000000000..86a960701 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestACL.py @@ -0,0 +1,223 @@ +import os +import sys +import lxml.etree +import Bcfg2.Server.Plugin +from mock import Mock, MagicMock, patch +from Bcfg2.Server.Plugins.ACL import * + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * +from TestPlugin import TestXMLFileBacked, TestStructFile, TestPlugin, \ + TestClientACLs + + +class TestFunctions(Bcfg2TestCase): + def test_rmi_names_equal(self): + good_cases = [('*', 'foo'), + ('foo', 'foo'), + ('foo.*', 'foo.bar'), + ('*.*', 'foo.bar'), + ('foo.bar', 'foo.bar'), + ('*.bar', 'foo.bar'), + ('foo.*.bar', 'foo.baz.bar')] + bad_cases = [('foo', 'bar'), + ('*', 'foo.bar'), + ('*.*', 'foo'), + ('*.*', 'foo.bar.baz'), + ('foo.*', 'bar.foo'), + ('*.bar', 'bar.foo'), + ('foo.*', 'foobar')] + for first, second in good_cases: + self.assertTrue(rmi_names_equal(first, second), + "rmi_names_equal(%s, %s) unexpectedly False" % + (first, second)) + self.assertTrue(rmi_names_equal(second, first), + "rmi_names_equal(%s, %s) unexpectedly False" % + (second, first)) + for first, second in bad_cases: + self.assertFalse(rmi_names_equal(first, second), + "rmi_names_equal(%s, %s) unexpectedly True" % + (first, second)) + self.assertFalse(rmi_names_equal(second, first), + "rmi_names_equal(%s, %s) unexpectedly True" % + (second, first)) + + def test_ip_matches(self): + good_cases = [ + ("192.168.1.1", lxml.etree.Element("test", address="192.168.1.1")), + ("192.168.1.17", lxml.etree.Element("test", address="192.168.1.0", + netmask="24")), + ("192.168.1.17", lxml.etree.Element("test", address="192.168.1.0", + netmask="255.255.255.0")), + ("192.168.1.31", lxml.etree.Element("test", address="192.168.1.0", + netmask="255.255.255.224")), + ("192.168.1.31", lxml.etree.Element("test", address="192.168.1.0", + netmask="27")), + ("10.55.67.191", lxml.etree.Element("test", address="10.55.0.0", + netmask="16"))] + bad_cases = [ + ("192.168.1.1", lxml.etree.Element("test", address="192.168.1.2")), + ("192.168.2.17", lxml.etree.Element("test", address="192.168.1.0", + netmask="24")), + ("192.168.2.17", lxml.etree.Element("test", address="192.168.1.0", + netmask="255.255.255.0")), + ("192.168.1.35", lxml.etree.Element("test", address="192.168.1.0", + netmask="255.255.255.224")), + ("192.168.1.35", lxml.etree.Element("test", address="192.168.1.0", + netmask="27")), + ("10.56.67.191", lxml.etree.Element("test", address="10.55.0.0", + netmask="16"))] + for ip, entry in good_cases: + self.assertTrue(ip_matches(ip, entry), + "ip_matches(%s, %s) unexpectedly False" % + (ip, lxml.etree.tostring(entry))) + for ip, entry in bad_cases: + self.assertFalse(ip_matches(ip, entry), + "ip_matches(%s, %s) unexpectedly True" % + (ip, lxml.etree.tostring(entry))) + + +class TestIPACLFile(TestXMLFileBacked): + test_obj = IPACLFile + + @patch("Bcfg2.Server.Plugins.ACL.ip_matches") + @patch("Bcfg2.Server.Plugins.ACL.rmi_names_equal") + def test_check_acl(self, mock_rmi_names_equal, mock_ip_matches): + af = self.get_obj() + ip = "10.0.0.8" + rmi = "ACL.test" + + def reset(): + mock_rmi_names_equal.reset_mock() + mock_ip_matches.reset_mock() + + # test default defer with no entries + af.entries = [] + self.assertIsNone(af.check_acl(ip, rmi)) + + # test explicit allow, deny, and defer + entries = dict(Allow=lxml.etree.Element("Allow", method=rmi), + Deny=lxml.etree.Element("Deny", method=rmi), + Defer=lxml.etree.Element("Defer", method=rmi)) + af.entries = list(entries.values()) + + def get_ip_matches(tag): + def ip_matches(ip, entry): + return entry.tag == tag + + return ip_matches + + mock_rmi_names_equal.return_value = True + + reset() + mock_ip_matches.side_effect = get_ip_matches("Allow") + self.assertTrue(af.check_acl(ip, rmi)) + mock_ip_matches.assert_called_with(ip, entries['Allow']) + mock_rmi_names_equal.assert_called_with(rmi, rmi) + + reset() + mock_ip_matches.side_effect = get_ip_matches("Deny") + self.assertFalse(af.check_acl(ip, rmi)) + mock_ip_matches.assert_called_with(ip, entries['Deny']) + mock_rmi_names_equal.assert_called_with(rmi, rmi) + + reset() + mock_ip_matches.side_effect = get_ip_matches("Defer") + self.assertIsNone(af.check_acl(ip, rmi)) + mock_ip_matches.assert_called_with(ip, entries['Defer']) + mock_rmi_names_equal.assert_called_with(rmi, rmi) + + # test matching RMI names + reset() + mock_ip_matches.side_effect = lambda i, e: True + mock_rmi_names_equal.side_effect = lambda a, b: a == b + rmi = "ACL.test2" + matching = lxml.etree.Element("Allow", method=rmi) + af.entries.append(matching) + self.assertTrue(af.check_acl(ip, rmi)) + mock_ip_matches.assert_called_with(ip, matching) + self.assertTrue( + call('ACL.test', rmi) in mock_rmi_names_equal.call_args_list or + call(rmi, 'ACL.test') in mock_rmi_names_equal.call_args_list) + + # test implicit allow for localhost, defer for others + reset() + mock_ip_matches.side_effect = lambda i, e: False + self.assertIsNone(af.check_acl(ip, rmi)) + + reset() + self.assertTrue(af.check_acl("127.0.0.1", rmi)) + + +class TestMetadataACLFile(TestStructFile): + test_obj = MetadataACLFile + + @patch("Bcfg2.Server.Plugins.ACL.rmi_names_equal") + def test_check_acl(self, mock_rmi_names_equal): + af = self.get_obj() + af.Match = Mock() + metadata = Mock() + mock_rmi_names_equal.side_effect = lambda a, b: a == b + + def reset(): + af.Match.reset_mock() + mock_rmi_names_equal.reset_mock() + + # test default allow + af.entries = [] + self.assertTrue(af.check_acl(metadata, 'ACL.test')) + + # test explicit allow and deny + reset() + af.entries = [lxml.etree.Element("Allow", method='ACL.test'), + lxml.etree.Element("Deny", method='ACL.test2')] + af.Match.return_value = af.entries + self.assertTrue(af.check_acl(metadata, 'ACL.test')) + af.Match.assert_called_with(metadata) + self.assertIn(call('ACL.test', 'ACL.test'), + mock_rmi_names_equal.call_args_list) + + reset() + self.assertFalse(af.check_acl(metadata, 'ACL.test2')) + af.Match.assert_called_with(metadata) + self.assertIn(call('ACL.test2', 'ACL.test2'), + mock_rmi_names_equal.call_args_list) + + # test default deny for non-localhost + reset() + self.assertFalse(af.check_acl(metadata, 'ACL.test3')) + af.Match.assert_called_with(metadata) + + # test default allow for localhost + reset() + metadata.hostname = 'localhost' + self.assertTrue(af.check_acl(metadata, 'ACL.test3')) + af.Match.assert_called_with(metadata) + + +class TestACL(TestPlugin, TestClientACLs): + test_obj = ACL + + def test_check_acl_ip(self): + acl = self.get_obj() + acl.ip_acls = Mock() + self.assertEqual(acl.check_acl_ip(("192.168.1.10", "12345"), + "ACL.test"), + acl.ip_acls.check_acl.return_value) + acl.ip_acls.check_acl.assert_called_with("192.168.1.10", "ACL.test") + + def test_check_acl_metadata(self): + acl = self.get_obj() + acl.metadata_acls = Mock() + metadata = Mock() + self.assertEqual(acl.check_acl_metadata(metadata, "ACL.test"), + acl.metadata_acls.check_acl.return_value) + acl.metadata_acls.check_acl.assert_called_with(metadata, "ACL.test") diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py new file mode 100644 index 000000000..cfb379c40 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py @@ -0,0 +1,111 @@ +import os +import sys +import lxml.etree +from mock import Mock, MagicMock, patch +from Bcfg2.Server.Plugins.Bundler import * + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * +from TestPlugin import TestStructFile, TestPlugin, TestStructure, \ + TestXMLDirectoryBacked + + +class TestBundleFile(TestStructFile): + test_obj = BundleFile + path = os.path.join(datastore, "test", "test1.xml") + + def test_bundle_name(self): + cases = [("foo.xml", "foo"), + ("foo.bar.xml", "foo.bar"), + ("foo-bar-baz.xml", "foo-bar-baz"), + ("foo....xml", "foo..."), + ("foo.genshi", "foo")] + bf = self.get_obj() + for fname, bname in cases: + bf.name = fname + self.assertEqual(bf.bundle_name, bname) + + +class TestBundler(TestPlugin, TestStructure, TestXMLDirectoryBacked): + test_obj = Bundler + + def get_obj(self, core=None): + @patch("%s.%s.add_directory_monitor" % (self.test_obj.__module__, + self.test_obj.__name__), + Mock()) + def inner(): + return TestPlugin.get_obj(self, core=core) + return inner() + + @patch("Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent") + def test_HandleEvent(self, mock_HandleEvent): + b = self.get_obj() + b.bundles = dict(foo=Mock(), bar=Mock()) + b.entries = {"foo.xml": BundleFile("foo.xml"), + "baz.xml": BundleFile("baz.xml")} + event = Mock() + b.HandleEvent(event) + mock_HandleEvent.assert_called_with(b, event) + self.assertItemsEqual(b.bundles, + dict(foo=b.entries['foo.xml'], + baz=b.entries['baz.xml'])) + + def test_BuildStructures(self): + b = self.get_obj() + b.bundles = dict(error=Mock(), skip=Mock(), xinclude=Mock(), + has_dep=Mock(), is_dep=Mock(), indep=Mock()) + expected = dict() + + b.bundles['error'].XMLMatch.side_effect = TemplateError(None) + + xinclude = lxml.etree.Element("Bundle") + lxml.etree.SubElement(lxml.etree.SubElement(xinclude, "Bundle"), + "Path", name="/test") + b.bundles['xinclude'].XMLMatch.return_value = xinclude + expected['xinclude'] = lxml.etree.Element("Bundle", name="xinclude") + lxml.etree.SubElement(expected['xinclude'], "Path", name="/test") + + has_dep = lxml.etree.Element("Bundle") + lxml.etree.SubElement(has_dep, "Bundle", name="is_dep") + lxml.etree.SubElement(has_dep, "Package", name="foo") + b.bundles['has_dep'].XMLMatch.return_value = has_dep + expected['has_dep'] = lxml.etree.Element("Bundle", name="has_dep") + lxml.etree.SubElement(expected['has_dep'], "Package", name="foo") + + is_dep = lxml.etree.Element("Bundle") + lxml.etree.SubElement(is_dep, "Package", name="bar") + b.bundles['is_dep'].XMLMatch.return_value = is_dep + expected['is_dep'] = lxml.etree.Element("Bundle", name="is_dep") + lxml.etree.SubElement(expected['is_dep'], "Package", name="bar") + + indep = lxml.etree.Element("Bundle", independent="true") + lxml.etree.SubElement(indep, "Service", name="baz") + b.bundles['indep'].XMLMatch.return_value = indep + expected['indep'] = lxml.etree.Element("Independent", name="indep") + lxml.etree.SubElement(expected['indep'], "Service", name="baz") + + metadata = Mock() + metadata.bundles = ["error", "xinclude", "has_dep", "indep"] + + rv = b.BuildStructures(metadata) + self.assertEqual(len(rv), 4) + for bundle in rv: + name = bundle.get("name") + self.assertIsNotNone(name, + "Bundle %s was not built" % name) + self.assertIn(name, expected, + "Unexpected bundle %s was built" % name) + self.assertXMLEqual(bundle, expected[name], + "Bundle %s was not built correctly" % name) + b.bundles[name].XMLMatch.assert_called_with(metadata) + + b.bundles['error'].XMLMatch.assert_called_with(metadata) + self.assertFalse(b.bundles['skip'].XMLMatch.called) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgAuthorizedKeysGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgAuthorizedKeysGenerator.py index d655a20cd..f41ae8a46 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgAuthorizedKeysGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgAuthorizedKeysGenerator.py @@ -23,12 +23,16 @@ class TestCfgAuthorizedKeysGenerator(TestCfgGenerator, TestStructFile): test_obj = CfgAuthorizedKeysGenerator should_monitor = False - def get_obj(self, name=None, core=None, fam=None): + def setUp(self): + TestCfgGenerator.setUp(self) + TestStructFile.setUp(self) + + @patch("Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.get_cfg") + def get_obj(self, mock_get_cfg, name=None, core=None, fam=None): if name is None: name = self.path - Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.CFG = Mock() if core is not None: - Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.CFG.core = core + mock_get_cfg.return_value.core = core return self.test_obj(name) @patch("Bcfg2.Server.Plugins.Cfg.CfgGenerator.handle_event") @@ -40,33 +44,9 @@ class TestCfgAuthorizedKeysGenerator(TestCfgGenerator, TestStructFile): mock_HandleEvent.assert_called_with(akg, evt) mock_handle_event.assert_called_with(akg, evt) - def test_category(self): - akg = self.get_obj() - cfp = Mock() - cfp.has_section.return_value = False - cfp.has_option.return_value = False - Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.SETUP = Mock() - Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.SETUP.cfp = cfp - - self.assertIsNone(akg.category) - cfp.has_section.assert_called_with("sshkeys") - - cfp.reset_mock() - cfp.has_section.return_value = True - self.assertIsNone(akg.category) - cfp.has_section.assert_called_with("sshkeys") - cfp.has_option.assert_called_with("sshkeys", "category") - - cfp.reset_mock() - cfp.has_option.return_value = True - self.assertEqual(akg.category, cfp.get.return_value) - cfp.has_section.assert_called_with("sshkeys") - cfp.has_option.assert_called_with("sshkeys", "category") - cfp.get.assert_called_with("sshkeys", "category") - @patch("Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.ClientMetadata") - @patch("Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.CfgAuthorizedKeysGenerator.category", "category") def test_get_data(self, mock_ClientMetadata): + Bcfg2.Options.setup.sshkeys_category = "category" akg = self.get_obj() akg.XMLMatch = Mock() @@ -131,17 +111,18 @@ class TestCfgAuthorizedKeysGenerator(TestCfgGenerator, TestStructFile): reset() host = "baz.example.com" spec = lxml.etree.Element("AuthorizedKeys") - lxml.etree.SubElement( - lxml.etree.SubElement(spec, - "Allow", - attrib={"from": pubkey, "host": host}), - "Params", foo="foo", bar="bar=bar") + allow = lxml.etree.SubElement(spec, "Allow", + attrib={"from": pubkey, "host": host}) + lxml.etree.SubElement(allow, "Option", name="foo", value="foo") + lxml.etree.SubElement(allow, "Option", name="bar") + lxml.etree.SubElement(allow, "Option", name="baz", value="baz=baz") akg.XMLMatch.return_value = spec params, actual_host, actual_pubkey = akg.get_data(entry, metadata).split() self.assertEqual(actual_host, host) self.assertEqual(actual_pubkey, pubkey) - self.assertItemsEqual(params.split(","), ["foo=foo", "bar=bar=bar"]) + self.assertItemsEqual(params.split(","), ["foo=foo", "bar", + "baz=baz=baz"]) akg.XMLMatch.assert_called_with(metadata) akg.core.build_metadata.assert_called_with(host) self.assertEqual(akg.core.Bind.call_args[0][0].get("name"), pubkey) @@ -151,10 +132,10 @@ class TestCfgAuthorizedKeysGenerator(TestCfgGenerator, TestStructFile): spec = lxml.etree.Element("AuthorizedKeys") text = lxml.etree.SubElement(spec, "Allow") text.text = "ssh-rsa publickey /foo/bar\n" - lxml.etree.SubElement(text, "Params", foo="foo") + lxml.etree.SubElement(text, "Option", name="foo") akg.XMLMatch.return_value = spec self.assertEqual(akg.get_data(entry, metadata), - "foo=foo %s" % text.text.strip()) + "foo %s" % text.text.strip()) akg.XMLMatch.assert_called_with(metadata) self.assertFalse(akg.core.build_metadata.called) self.assertFalse(akg.core.Bind.called) @@ -163,7 +144,7 @@ class TestCfgAuthorizedKeysGenerator(TestCfgGenerator, TestStructFile): lxml.etree.SubElement(spec, "Allow", attrib={"from": pubkey}) akg.XMLMatch.return_value = spec self.assertItemsEqual(akg.get_data(entry, metadata).splitlines(), - ["foo=foo %s" % text.text.strip(), + ["foo %s" % text.text.strip(), "profile %s" % pubkey]) akg.XMLMatch.assert_called_with(metadata) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py index fc5d5e53d..93331304a 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py @@ -17,32 +17,39 @@ from common import * from TestServer.TestPlugins.TestCfg.Test_init import TestCfgGenerator -if HAS_CHEETAH or can_skip: - class TestCfgCheetahGenerator(TestCfgGenerator): - test_obj = CfgCheetahGenerator +class TestCfgCheetahGenerator(TestCfgGenerator): + test_obj = CfgCheetahGenerator - @skipUnless(HAS_CHEETAH, "Cheetah libraries not found, skipping") - def setUp(self): - pass + @skipUnless(HAS_CHEETAH, "Cheetah libraries not found, skipping") + def setUp(self): + TestCfgGenerator.setUp(self) + set_setup_default("repository", datastore) - @patch("Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.Template") - def test_get_data(self, mock_Template): - ccg = self.get_obj(encoding='UTF-8') - ccg.data = "data" - entry = lxml.etree.Element("Path", name="/test.txt") - metadata = Mock() - Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.SETUP = MagicMock() + @patch("Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.Template") + @patch("Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.get_template_data") + def test_get_data(self, mock_get_template_data, mock_Template): + ccg = self.get_obj() + ccg.data = "data" + entry = lxml.etree.Element("Path", name="/test.txt") + metadata = Mock() - self.assertEqual(ccg.get_data(entry, metadata), - mock_Template.return_value.respond.return_value) - Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.SETUP.__getitem__.assert_called_with("repo") - mock_Template.assert_called_with("data".decode(ccg.encoding), - compilerSettings=ccg.settings) - tmpl = mock_Template.return_value - tmpl.respond.assert_called_with() - self.assertEqual(tmpl.metadata, metadata) - self.assertEqual(tmpl.name, entry.get("name")) - self.assertEqual(tmpl.path, entry.get("name")) - self.assertEqual(tmpl.source_path, ccg.name) - self.assertEqual(tmpl.repo, - Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.SETUP.__getitem__.return_value) + template_vars = dict(name=entry.get("name"), + metadata=metadata, + path=ccg.name, + source_path=ccg.name, + repo=datastore) + mock_get_template_data.return_value = template_vars + + self.assertEqual(ccg.get_data(entry, metadata), + mock_Template.return_value.respond.return_value) + mock_Template.assert_called_with( + "data".decode(Bcfg2.Options.setup.encoding), + compilerSettings=ccg.settings) + tmpl = mock_Template.return_value + tmpl.respond.assert_called_with() + for key, val in template_vars.items(): + self.assertEqual(getattr(tmpl, key), val) + self.assertItemsEqual(mock_get_template_data.call_args[0], + [entry, metadata, ccg.name]) + self.assertIsInstance(mock_get_template_data.call_args[1]['default'], + DefaultCheetahDataProvider) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedCheetahGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedCheetahGenerator.py index 46062569d..4c987551b 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedCheetahGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedCheetahGenerator.py @@ -30,18 +30,17 @@ except ImportError: HAS_CRYPTO = False -if can_skip or (HAS_CRYPTO and HAS_CHEETAH): - class TestCfgEncryptedCheetahGenerator(TestCfgCheetahGenerator, - TestCfgEncryptedGenerator): - test_obj = CfgEncryptedCheetahGenerator +class TestCfgEncryptedCheetahGenerator(TestCfgCheetahGenerator, + TestCfgEncryptedGenerator): + test_obj = CfgEncryptedCheetahGenerator - @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") - @skipUnless(HAS_CHEETAH, "Cheetah libraries not found, skipping") - def setUp(self): - pass + @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") + @skipUnless(HAS_CHEETAH, "Cheetah libraries not found, skipping") + def setUp(self): + pass - def test_handle_event(self): - TestCfgEncryptedGenerator.test_handle_event(self) + def test_handle_event(self): + TestCfgEncryptedGenerator.test_handle_event(self) - def test_get_data(self): - TestCfgCheetahGenerator.test_get_data(self) + def test_get_data(self): + TestCfgCheetahGenerator.test_get_data(self) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py index f07994d34..63403208b 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py @@ -19,62 +19,54 @@ from common import * from TestServer.TestPlugins.TestCfg.Test_init import TestCfgGenerator -if can_skip or HAS_CRYPTO: - class TestCfgEncryptedGenerator(TestCfgGenerator): - test_obj = CfgEncryptedGenerator +class TestCfgEncryptedGenerator(TestCfgGenerator): + test_obj = CfgEncryptedGenerator - @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") - def setUp(self): - pass + @skipUnless(HAS_CRYPTO, "M2Crypto is not available") + def setUp(self): + TestCfgGenerator.setUp(self) - @patchIf(HAS_CRYPTO, - "Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator.get_algorithm") - @patchIf(HAS_CRYPTO, - "Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator.bruteforce_decrypt") - @patch("Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator.SETUP") - def test_handle_event(self, mock_SETUP, mock_decrypt, - mock_get_algorithm): - @patch("Bcfg2.Server.Plugins.Cfg.CfgGenerator.handle_event") - def inner(mock_handle_event): - def reset(): - mock_decrypt.reset_mock() - mock_get_algorithm.reset_mock() - mock_handle_event.reset_mock() + @patchIf(HAS_CRYPTO, + "Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator.bruteforce_decrypt") + def test_handle_event(self, mock_decrypt): + @patch("Bcfg2.Server.Plugins.Cfg.CfgGenerator.handle_event") + @patch("Bcfg2.Options.setup.lax_decryption", False) + def inner(mock_handle_event): + def reset(): + mock_decrypt.reset_mock() + mock_handle_event.reset_mock() - def get_event_data(obj, event): - obj.data = "encrypted" + def get_event_data(obj, event): + obj.data = "encrypted" - mock_handle_event.side_effect = get_event_data - mock_decrypt.side_effect = lambda d, **kw: "plaintext" - event = Mock() - ceg = self.get_obj() - ceg.handle_event(event) - mock_handle_event.assert_called_with(ceg, event) - mock_decrypt.assert_called_with( - "encrypted", - setup=mock_SETUP, - algorithm=mock_get_algorithm.return_value) - self.assertEqual(ceg.data, "plaintext") + mock_handle_event.side_effect = get_event_data + mock_decrypt.side_effect = lambda d, **kw: "plaintext" + event = Mock() + ceg = self.get_obj() + ceg.handle_event(event) + mock_handle_event.assert_called_with(ceg, event) + mock_decrypt.assert_called_with("encrypted") + self.assertEqual(ceg.data, "plaintext") - reset() - mock_decrypt.side_effect = EVPError - self.assertRaises(PluginExecutionError, - ceg.handle_event, event) - inner() + reset() + mock_decrypt.side_effect = EVPError + self.assertRaises(PluginExecutionError, + ceg.handle_event, event) + inner() - # to perform the tests from the parent test object, we - # make bruteforce_decrypt just return whatever data was - # given to it - mock_decrypt.side_effect = lambda d, **kw: d - TestCfgGenerator.test_handle_event(self) + # to perform the tests from the parent test object, we + # make bruteforce_decrypt just return whatever data was + # given to it + mock_decrypt.side_effect = lambda d, **kw: d + TestCfgGenerator.test_handle_event(self) - def test_get_data(self): - ceg = self.get_obj() - ceg.data = None - entry = lxml.etree.Element("Path", name="/test.txt") - metadata = Mock() + def test_get_data(self): + ceg = self.get_obj() + ceg.data = None + entry = lxml.etree.Element("Path", name="/test.txt") + metadata = Mock() - self.assertRaises(PluginExecutionError, - ceg.get_data, entry, metadata) + self.assertRaises(PluginExecutionError, + ceg.get_data, entry, metadata) - TestCfgGenerator.test_get_data(self) + TestCfgGenerator.test_get_data(self) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenshiGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenshiGenerator.py index b447a9bb8..0b74e4a60 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenshiGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenshiGenerator.py @@ -14,20 +14,13 @@ while path != "/": path = os.path.dirname(path) from common import * -try: - from TestServer.TestPlugins.TestCfg.TestCfgGenshiGenerator import \ - TestCfgGenshiGenerator - HAS_GENSHI = True -except ImportError: - TestCfgGenshiGenerator = object - HAS_GENSHI = False +from TestServer.TestPlugins.TestCfg.TestCfgGenshiGenerator import \ + TestCfgGenshiGenerator -if can_skip or (HAS_CRYPTO and HAS_GENSHI): - class TestCfgEncryptedGenshiGenerator(TestCfgGenshiGenerator): - test_obj = CfgEncryptedGenshiGenerator +class TestCfgEncryptedGenshiGenerator(TestCfgGenshiGenerator): + test_obj = CfgEncryptedGenshiGenerator - @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") - @skipUnless(HAS_GENSHI, "Genshi libraries not found, skipping") - def setUp(self): - pass + @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") + def setUp(self): + TestCfgGenshiGenerator.setUp(self) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedJinja2Generator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedJinja2Generator.py new file mode 100644 index 000000000..6857f933b --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedJinja2Generator.py @@ -0,0 +1,46 @@ +import os +import sys +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator import * + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * + +try: + from TestServer.TestPlugins.TestCfg.TestCfgJinja2Generator import \ + TestCfgJinja2Generator + from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import HAS_JINJA2 +except ImportError: + TestCfgJinja2Generator = object + HAS_JINJA2 = False + +try: + from TestServer.TestPlugins.TestCfg.TestCfgEncryptedGenerator import \ + TestCfgEncryptedGenerator + from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator import HAS_CRYPTO +except ImportError: + TestCfgEncryptedGenerator = object + HAS_CRYPTO = False + + +class TestCfgEncryptedJinja2Generator(TestCfgJinja2Generator, + TestCfgEncryptedGenerator): + test_obj = CfgEncryptedJinja2Generator + + @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") + @skipUnless(HAS_JINJA2, "Jinja2 libraries not found, skipping") + def setUp(self): + pass + + def test_handle_event(self): + TestCfgEncryptedGenerator.test_handle_event(self) + + def test_get_data(self): + TestCfgJinja2Generator.test_get_data(self) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgExternalCommandVerifier.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgExternalCommandVerifier.py index 0f369113b..7ceedb7c2 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgExternalCommandVerifier.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgExternalCommandVerifier.py @@ -21,35 +21,32 @@ from TestServer.TestPlugins.TestCfg.Test_init import TestCfgVerifier class TestCfgExternalCommandVerifier(TestCfgVerifier): test_obj = CfgExternalCommandVerifier - @patch("Bcfg2.Server.Plugins.Cfg.CfgExternalCommandVerifier.Popen") - def test_verify_entry(self, mock_Popen): - proc = Mock() - mock_Popen.return_value = proc - proc.wait.return_value = 0 - proc.communicate.return_value = ("stdout", "stderr") + def test_verify_entry(self): entry = lxml.etree.Element("Path", name="/test.txt") metadata = Mock() ecv = self.get_obj() ecv.cmd = ["/bin/bash", "-x", "foo"] + ecv.exc = Mock() + ecv.exc.run.return_value = Mock() + ecv.exc.run.return_value.success = True + ecv.verify_entry(entry, metadata, "data") - self.assertEqual(mock_Popen.call_args[0], (ecv.cmd,)) - proc.communicate.assert_called_with(input="data") - proc.wait.assert_called_with() + ecv.exc.run.assert_called_with(ecv.cmd, inputdata="data") - mock_Popen.reset_mock() - proc.wait.return_value = 13 + ecv.exc.reset_mock() + ecv.exc.run.return_value.success = False self.assertRaises(CfgVerificationError, ecv.verify_entry, entry, metadata, "data") - self.assertEqual(mock_Popen.call_args[0], (ecv.cmd,)) - proc.communicate.assert_called_with(input="data") - proc.wait.assert_called_with() + ecv.exc.run.assert_called_with(ecv.cmd, inputdata="data") + + ecv.exc.reset_mock() - mock_Popen.reset_mock() - mock_Popen.side_effect = OSError + ecv.exc.reset_mock() + ecv.exc.run.side_effect = OSError self.assertRaises(CfgVerificationError, ecv.verify_entry, entry, metadata, "data") - self.assertEqual(mock_Popen.call_args[0], (ecv.cmd,)) + ecv.exc.run.assert_called_with(ecv.cmd, inputdata="data") @patch("os.access") def test_handle_event(self, mock_access): diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py index 2e8b7bfa5..b667d417a 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py @@ -19,111 +19,120 @@ from common import * from TestServer.TestPlugins.TestCfg.Test_init import TestCfgGenerator -if can_skip or HAS_GENSHI: - class TestCfgGenshiGenerator(TestCfgGenerator): - test_obj = CfgGenshiGenerator - - @skipUnless(HAS_GENSHI, "Genshi libraries not found, skipping") - def setUp(self): - pass - - def test_removecomment(self): - data = [(None, "test", 1), - (None, "test2", 2)] - stream = [(genshi.core.COMMENT, "test", 0), - data[0], - (genshi.core.COMMENT, "test3", 0), - data[1]] - self.assertItemsEqual(list(removecomment(stream)), data) - - def test__init(self): - TestCfgGenerator.test__init(self) - cgg = self.get_obj() - self.assertIsInstance(cgg.loader, cgg.__loader_cls__) - - def test_get_data(self): - cgg = self.get_obj() - cgg._handle_genshi_exception = Mock() - cgg.template = Mock() - fltr = Mock() - cgg.template.generate.return_value = fltr - stream = Mock() - fltr.filter.return_value = stream - entry = lxml.etree.Element("Path", name="/test.txt") - metadata = Mock() - - Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP = MagicMock() - - def reset(): - cgg.template.reset_mock() - cgg._handle_genshi_exception.reset_mock() - Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.reset_mock() - - template_vars = dict( - name=entry.get("name"), - metadata=metadata, - path=cgg.name, - source_path=cgg.name, - repo=Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.return_value) - - self.assertEqual(cgg.get_data(entry, metadata), - stream.render.return_value) - cgg.template.generate.assert_called_with(**template_vars) - Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo") - fltr.filter.assert_called_with(removecomment) - stream.render.assert_called_with("text", encoding=cgg.encoding, - strip_whitespace=False) - - reset() - def render(fmt, **kwargs): - stream.render.side_effect = None - raise TypeError - stream.render.side_effect = render - self.assertEqual(cgg.get_data(entry, metadata), - stream.render.return_value) - cgg.template.generate.assert_called_with(**template_vars) - Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo") - fltr.filter.assert_called_with(removecomment) - self.assertEqual(stream.render.call_args_list, - [call("text", encoding=cgg.encoding, - strip_whitespace=False), - call("text", encoding=cgg.encoding)]) - - reset() - stream.render.side_effect = UndefinedError("test") - self.assertRaises(UndefinedError, - cgg.get_data, entry, metadata) - cgg.template.generate.assert_called_with(**template_vars) - Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo") - fltr.filter.assert_called_with(removecomment) - stream.render.assert_called_with("text", encoding=cgg.encoding, - strip_whitespace=False) - - reset() - stream.render.side_effect = ValueError - cgg._handle_genshi_exception.side_effect = ValueError - self.assertRaises(ValueError, - cgg.get_data, entry, metadata) - cgg.template.generate.assert_called_with(**template_vars) - Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo") - fltr.filter.assert_called_with(removecomment) - stream.render.assert_called_with("text", encoding=cgg.encoding, - strip_whitespace=False) - self.assertTrue(cgg._handle_genshi_exception.called) - - def test_handle_event(self): - cgg = self.get_obj() - cgg.loader = Mock() - event = Mock() - cgg.handle_event(event) - cgg.loader.load.assert_called_with(cgg.name, - cls=NewTextTemplate, - encoding=cgg.encoding) - - cgg.loader.reset_mock() - cgg.loader.load.side_effect = OSError - self.assertRaises(PluginExecutionError, - cgg.handle_event, event) - cgg.loader.load.assert_called_with(cgg.name, - cls=NewTextTemplate, - encoding=cgg.encoding) +class TestCfgGenshiGenerator(TestCfgGenerator): + test_obj = CfgGenshiGenerator + + def setUp(self): + TestCfgGenerator.setUp(self) + set_setup_default("repository", datastore) + + def test__init(self): + TestCfgGenerator.test__init(self) + cgg = self.get_obj() + self.assertIsInstance(cgg.loader, cgg.__loader_cls__) + + @patch("Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.get_template_data") + def test_get_data(self, mock_get_template_data): + cgg = self.get_obj() + cgg._handle_genshi_exception = Mock() + cgg.template = Mock() + fltr = Mock() + cgg.template.generate.return_value = fltr + stream = Mock() + fltr.filter.return_value = stream + entry = lxml.etree.Element("Path", name="/test.txt") + metadata = Mock() + + def reset(): + cgg.template.reset_mock() + cgg._handle_genshi_exception.reset_mock() + mock_get_template_data.reset_mock() + + template_vars = dict(name=entry.get("name"), + metadata=metadata, + path=cgg.name, + source_path=cgg.name, + repo=datastore) + mock_get_template_data.return_value = template_vars + + self.assertEqual(cgg.get_data(entry, metadata), + stream.render.return_value) + cgg.template.generate.assert_called_with(**template_vars) + self.assertItemsEqual(mock_get_template_data.call_args[0], + [entry, metadata, cgg.name]) + self.assertIsInstance(mock_get_template_data.call_args[1]['default'], + DefaultGenshiDataProvider) + fltr.filter.assert_called_with(removecomment) + stream.render.assert_called_with( + "text", + encoding=Bcfg2.Options.setup.encoding, + strip_whitespace=False) + + reset() + def render(fmt, **kwargs): + stream.render.side_effect = None + raise TypeError + stream.render.side_effect = render + self.assertEqual(cgg.get_data(entry, metadata), + stream.render.return_value) + cgg.template.generate.assert_called_with(**template_vars) + self.assertItemsEqual(mock_get_template_data.call_args[0], + [entry, metadata, cgg.name]) + self.assertIsInstance(mock_get_template_data.call_args[1]['default'], + DefaultGenshiDataProvider) + fltr.filter.assert_called_with(removecomment) + self.assertEqual(stream.render.call_args_list, + [call("text", + encoding=Bcfg2.Options.setup.encoding, + strip_whitespace=False), + call("text", + encoding=Bcfg2.Options.setup.encoding)]) + + reset() + stream.render.side_effect = UndefinedError("test") + self.assertRaises(UndefinedError, + cgg.get_data, entry, metadata) + cgg.template.generate.assert_called_with(**template_vars) + self.assertItemsEqual(mock_get_template_data.call_args[0], + [entry, metadata, cgg.name]) + self.assertIsInstance(mock_get_template_data.call_args[1]['default'], + DefaultGenshiDataProvider) + fltr.filter.assert_called_with(removecomment) + stream.render.assert_called_with("text", + encoding=Bcfg2.Options.setup.encoding, + strip_whitespace=False) + + reset() + stream.render.side_effect = ValueError + cgg._handle_genshi_exception.side_effect = ValueError + self.assertRaises(ValueError, + cgg.get_data, entry, metadata) + cgg.template.generate.assert_called_with(**template_vars) + self.assertItemsEqual(mock_get_template_data.call_args[0], + [entry, metadata, cgg.name]) + self.assertIsInstance(mock_get_template_data.call_args[1]['default'], + DefaultGenshiDataProvider) + fltr.filter.assert_called_with(removecomment) + stream.render.assert_called_with("text", + encoding=Bcfg2.Options.setup.encoding, + strip_whitespace=False) + self.assertTrue(cgg._handle_genshi_exception.called) + + def test_handle_event(self): + cgg = self.get_obj() + cgg.loader = Mock() + event = Mock() + cgg.handle_event(event) + cgg.loader.load.assert_called_with( + cgg.name, + cls=NewTextTemplate, + encoding=Bcfg2.Options.setup.encoding) + + cgg.loader.reset_mock() + cgg.loader.load.side_effect = OSError + self.assertRaises(PluginExecutionError, + cgg.handle_event, event) + cgg.loader.load.assert_called_with( + cgg.name, + cls=NewTextTemplate, + encoding=Bcfg2.Options.setup.encoding) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgInfoXML.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgInfoXML.py index 839e9c3b8..349da2213 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgInfoXML.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgInfoXML.py @@ -21,53 +21,26 @@ from TestServer.TestPlugins.TestCfg.Test_init import TestCfgInfo class TestCfgInfoXML(TestCfgInfo): test_obj = CfgInfoXML + def setUp(self): + TestCfgInfo.setUp(self) + set_setup_default("filemonitor", MagicMock()) + def test__init(self): TestCfgInfo.test__init(self) ci = self.get_obj() self.assertIsInstance(ci.infoxml, InfoXML) def test_bind_info_to_entry(self): - entry = lxml.etree.Element("Path", name="/test.txt") - metadata = Mock() ci = self.get_obj() ci.infoxml = Mock() - ci._set_info = Mock() - - self.assertRaises(PluginExecutionError, - ci.bind_info_to_entry, entry, metadata) - ci.infoxml.pnode.Match.assert_called_with(metadata, dict(), - entry=entry) - self.assertFalse(ci._set_info.called) - - ci.infoxml.reset_mock() - ci._set_info.reset_mock() - mdata_value = Mock() - def set_mdata(metadata, mdata, entry=None): - mdata['Info'] = {None: mdata_value} + entry = Mock() + metadata = Mock() - ci.infoxml.pnode.Match.side_effect = set_mdata ci.bind_info_to_entry(entry, metadata) - ci.infoxml.pnode.Match.assert_called_with(metadata, - dict(Info={None: mdata_value}), - entry=entry) - ci._set_info.assert_called_with(entry, mdata_value) + ci.infoxml.BindEntry.assert_called_with(entry, metadata) def test_handle_event(self): ci = self.get_obj() ci.infoxml = Mock() ci.handle_event(Mock) ci.infoxml.HandleEvent.assert_called_with() - - def test__set_info(self): - @patch("Bcfg2.Server.Plugins.Cfg.CfgInfo._set_info") - def inner(mock_set_info): - ci = self.get_obj() - entry = Mock() - info = {"foo": "foo", - "__children__": ["one", "two"]} - ci._set_info(entry, info) - self.assertItemsEqual(entry.append.call_args_list, - [call(c) for c in info['__children__']]) - - inner() - TestCfgInfo.test__set_info(self) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgJinja2Generator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgJinja2Generator.py new file mode 100644 index 000000000..036380d56 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgJinja2Generator.py @@ -0,0 +1,47 @@ +import os +import sys +import lxml.etree +from mock import Mock, MagicMock, patch +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import * + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * +from TestServer.TestPlugins.TestCfg.Test_init import TestCfgGenerator + + +class TestCfgJinja2Generator(TestCfgGenerator): + test_obj = CfgJinja2Generator + + @skipUnless(HAS_JINJA2, "Jinja2 libraries not found, skipping") + def setUp(self): + TestCfgGenerator.setUp(self) + set_setup_default("repository", datastore) + + @patch("Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.Template") + @patch("Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.get_template_data") + def test_get_data(self, mock_get_template_data, mock_Template): + ccg = self.get_obj() + ccg.data = "data" + entry = lxml.etree.Element("Path", name="/test.txt") + metadata = Mock() + + template_vars = dict(name=entry.get("name"), + metadata=metadata, + path=ccg.name, + source_path=ccg.name, + repo=datastore) + mock_get_template_data.return_value = template_vars + + self.assertEqual(ccg.get_data(entry, metadata), + mock_Template.return_value.render.return_value) + mock_Template.assert_called_with("data".decode(Bcfg2.Options.setup.encoding)) + tmpl = mock_Template.return_value + tmpl.render.assert_called_with(template_vars) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py index e139a592b..d64bbaabf 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py @@ -7,7 +7,7 @@ from Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator import * from Bcfg2.Server.Plugin import PluginExecutionError import Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator try: - from Bcfg2.Encryption import EVPError + from Bcfg2.Server.Encryption import EVPError HAS_CRYPTO = True except: HAS_CRYPTO = False @@ -22,100 +22,36 @@ while path != "/": break path = os.path.dirname(path) from common import * -from TestServer.TestPlugins.TestCfg.Test_init import TestCfgCreator -from TestServer.TestPlugin.Testhelpers import TestStructFile +from TestServer.TestPlugins.TestCfg.Test_init import TestXMLCfgCreator -class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): +class TestCfgPrivateKeyCreator(TestXMLCfgCreator): test_obj = CfgPrivateKeyCreator should_monitor = False + @patch("Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator.get_cfg", Mock()) def get_obj(self, name=None, fam=None): - Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator.CFG = Mock() - return TestCfgCreator.get_obj(self, name=name) - - @patch("Bcfg2.Server.Plugins.Cfg.CfgCreator.handle_event") - @patch("Bcfg2.Server.Plugin.helpers.StructFile.HandleEvent") - def test_handle_event(self, mock_HandleEvent, mock_handle_event): - pkc = self.get_obj() - evt = Mock() - pkc.handle_event(evt) - mock_HandleEvent.assert_called_with(pkc, evt) - mock_handle_event.assert_called_with(pkc, evt) - - def test_category(self): - pkc = self.get_obj() - cfp = Mock() - cfp.has_section.return_value = False - cfp.has_option.return_value = False - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP = Mock() - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP.cfp = cfp - - self.assertIsNone(pkc.category) - cfp.has_section.assert_called_with("sshkeys") - - cfp.reset_mock() - cfp.has_section.return_value = True - self.assertIsNone(pkc.category) - cfp.has_section.assert_called_with("sshkeys") - cfp.has_option.assert_called_with("sshkeys", "category") - - cfp.reset_mock() - cfp.has_option.return_value = True - self.assertEqual(pkc.category, cfp.get.return_value) - cfp.has_section.assert_called_with("sshkeys") - cfp.has_option.assert_called_with("sshkeys", "category") - cfp.get.assert_called_with("sshkeys", "category") - - @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping") - @patchIf(HAS_CRYPTO, "Bcfg2.Encryption.get_passphrases") - def test_passphrase(self, mock_get_passphrases): - pkc = self.get_obj() - cfp = Mock() - cfp.has_section.return_value = False - cfp.has_option.return_value = False - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP = Mock() - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP.cfp = cfp - - self.assertIsNone(pkc.passphrase) - cfp.has_section.assert_called_with("sshkeys") - - cfp.reset_mock() - cfp.has_section.return_value = True - self.assertIsNone(pkc.passphrase) - cfp.has_section.assert_called_with("sshkeys") - cfp.has_option.assert_called_with("sshkeys", "passphrase") - - cfp.reset_mock() - cfp.get.return_value = "test" - mock_get_passphrases.return_value = dict(test="foo", test2="bar") - cfp.has_option.return_value = True - self.assertEqual(pkc.passphrase, "foo") - cfp.has_section.assert_called_with("sshkeys") - cfp.has_option.assert_called_with("sshkeys", "passphrase") - cfp.get.assert_called_with("sshkeys", "passphrase") - mock_get_passphrases.assert_called_with(Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP) + return TestXMLCfgCreator.get_obj(self, name=name) @patch("shutil.rmtree") @patch("tempfile.mkdtemp") - @patch("subprocess.Popen") - def test__gen_keypair(self, mock_Popen, mock_mkdtemp, mock_rmtree): + def test__gen_keypair(self, mock_mkdtemp, mock_rmtree): pkc = self.get_obj() + pkc.cmd = Mock() pkc.XMLMatch = Mock() mock_mkdtemp.return_value = datastore metadata = Mock() - proc = Mock() - proc.wait.return_value = 0 - proc.communicate.return_value = MagicMock() - mock_Popen.return_value = proc + exc = Mock() + exc.success = True + pkc.cmd.run.return_value = exc spec = lxml.etree.Element("PrivateKey") pkc.XMLMatch.return_value = spec def reset(): pkc.XMLMatch.reset_mock() - mock_Popen.reset_mock() + pkc.cmd.reset_mock() mock_mkdtemp.reset_mock() mock_rmtree.reset_mock() @@ -123,10 +59,9 @@ class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): os.path.join(datastore, "privkey")) pkc.XMLMatch.assert_called_with(metadata) mock_mkdtemp.assert_called_with() - self.assertItemsEqual(mock_Popen.call_args[0][0], - ["ssh-keygen", "-f", - os.path.join(datastore, "privkey"), - "-t", "rsa", "-N", ""]) + pkc.cmd.run.assert_called_with(["ssh-keygen", "-f", + os.path.join(datastore, "privkey"), + "-t", "rsa", "-N", ""]) reset() lxml.etree.SubElement(spec, "Params", bits="768", type="dsa") @@ -137,73 +72,15 @@ class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): os.path.join(datastore, "privkey")) pkc.XMLMatch.assert_called_with(metadata) mock_mkdtemp.assert_called_with() - self.assertItemsEqual(mock_Popen.call_args[0][0], - ["ssh-keygen", "-f", - os.path.join(datastore, "privkey"), - "-t", "dsa", "-b", "768", "-N", "foo"]) + pkc.cmd.run.assert_called_with(["ssh-keygen", "-f", + os.path.join(datastore, "privkey"), + "-t", "dsa", "-b", "768", "-N", "foo"]) reset() - proc.wait.return_value = 1 + pkc.cmd.run.return_value.success = False self.assertRaises(CfgCreationError, pkc._gen_keypair, metadata) mock_rmtree.assert_called_with(datastore) - def test_get_specificity(self): - pkc = self.get_obj() - pkc.XMLMatch = Mock() - - metadata = Mock() - - def reset(): - pkc.XMLMatch.reset_mock() - metadata.group_in_category.reset_mock() - - category = "Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.CfgPrivateKeyCreator.category" - @patch(category, None) - def inner(): - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey") - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(host=metadata.hostname)) - inner() - - @patch(category, "foo") - def inner2(): - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey") - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(group=metadata.group_in_category.return_value, - prio=50)) - metadata.group_in_category.assert_called_with("foo") - - reset() - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey", - perhost="true") - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(host=metadata.hostname)) - - reset() - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey", - category="bar") - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(group=metadata.group_in_category.return_value, - prio=50)) - metadata.group_in_category.assert_called_with("bar") - - reset() - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey", - prio="10") - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(group=metadata.group_in_category.return_value, - prio=10)) - metadata.group_in_category.assert_called_with("foo") - - reset() - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey") - metadata.group_in_category.return_value = '' - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(host=metadata.hostname)) - metadata.group_in_category.assert_called_with("foo") - - inner2() - @patch("shutil.rmtree") @patch("%s.open" % builtins) def test_create_data(self, mock_open, mock_rmtree): @@ -216,7 +93,7 @@ class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): # the get_specificity() return value is being used # appropriately, we put some dummy data in it and test for # that data - pkc.get_specificity.side_effect = lambda m, s: dict(group="foo") + pkc.get_specificity.side_effect = lambda m: dict(group="foo") pkc._gen_keypair = Mock() privkey = os.path.join(datastore, "privkey") pkc._gen_keypair.return_value = privkey @@ -242,179 +119,15 @@ class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): mock_open.return_value.read.side_effect = open_read_rv reset() - passphrase = "Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.CfgPrivateKeyCreator.passphrase" - - @patch(passphrase, None) - def inner(): - self.assertEqual(pkc.create_data(entry, metadata), "privatekey") - pkc.XMLMatch.assert_called_with(metadata) - pkc.get_specificity.assert_called_with(metadata, - pkc.XMLMatch.return_value) - pkc._gen_keypair.assert_called_with(metadata, - pkc.XMLMatch.return_value) - self.assertItemsEqual(mock_open.call_args_list, - [call(privkey + ".pub"), call(privkey)]) - pkc.pubkey_creator.get_filename.assert_called_with(group="foo") - pkc.pubkey_creator.write_data.assert_called_with( - "ssh-rsa publickey pubkey.filename\n", group="foo") - pkc.write_data.assert_called_with("privatekey", group="foo") - mock_rmtree.assert_called_with(datastore) - - inner() - - if HAS_CRYPTO: - @patch(passphrase, "foo") - @patch("Bcfg2.Encryption.ssl_encrypt") - @patch("Bcfg2.Encryption.get_algorithm") - def inner2(mock_get_algorithm, mock_ssl_encrypt): - reset() - mock_ssl_encrypt.return_value = "encryptedprivatekey" - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.HAS_CRYPTO = True - self.assertEqual(pkc.create_data(entry, metadata), - "encryptedprivatekey") - pkc.XMLMatch.assert_called_with(metadata) - pkc.get_specificity.assert_called_with( - metadata, - pkc.XMLMatch.return_value) - pkc._gen_keypair.assert_called_with(metadata, - pkc.XMLMatch.return_value) - self.assertItemsEqual(mock_open.call_args_list, - [call(privkey + ".pub"), call(privkey)]) - pkc.pubkey_creator.get_filename.assert_called_with(group="foo") - pkc.pubkey_creator.write_data.assert_called_with( - "ssh-rsa publickey pubkey.filename\n", group="foo") - pkc.write_data.assert_called_with("encryptedprivatekey", - group="foo", ext=".crypt") - mock_ssl_encrypt.assert_called_with( - "privatekey", "foo", - algorithm=mock_get_algorithm.return_value) - mock_rmtree.assert_called_with(datastore) - - inner2() - - def test_Index(self): - has_crypto = Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.HAS_CRYPTO - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.HAS_CRYPTO = False - TestStructFile.test_Index(self) - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.HAS_CRYPTO = has_crypto - - @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping") - def test_Index_crypto(self): - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP = Mock() - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP.cfp.get.return_value = "strict" - - pkc = self.get_obj() - pkc._decrypt = Mock() - pkc._decrypt.return_value = 'plaintext' - pkc.data = ''' -<PrivateKey> - <Group name="test"> - <Passphrase encrypted="foo">crypted</Passphrase> - </Group> - <Group name="test" negate="true"> - <Passphrase>plain</Passphrase> - </Group> -</PrivateKey>''' - - # test successful decryption - pkc.Index() - self.assertItemsEqual( - pkc._decrypt.call_args_list, - [call(el) - for el in pkc.xdata.xpath("//Passphrase[@encrypted]")]) - for el in pkc.xdata.xpath("//Crypted"): - self.assertEqual(el.text, pkc._decrypt.return_value) - - # test failed decryption, strict - pkc._decrypt.reset_mock() - pkc._decrypt.side_effect = EVPError - self.assertRaises(PluginExecutionError, pkc.Index) - - # test failed decryption, lax - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP.cfp.get.return_value = "lax" - pkc._decrypt.reset_mock() - pkc.Index() - self.assertItemsEqual( - pkc._decrypt.call_args_list, - [call(el) - for el in pkc.xdata.xpath("//Passphrase[@encrypted]")]) - - @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping") - @patchIf(HAS_CRYPTO, "Bcfg2.Encryption.ssl_decrypt") - @patchIf(HAS_CRYPTO, "Bcfg2.Encryption.get_algorithm") - @patchIf(HAS_CRYPTO, "Bcfg2.Encryption.get_passphrases") - @patchIf(HAS_CRYPTO, "Bcfg2.Encryption.bruteforce_decrypt") - def test_decrypt(self, mock_bruteforce, mock_get_passphrases, - mock_get_algorithm, mock_ssl): - pkc = self.get_obj() - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP = MagicMock() - - def reset(): - mock_bruteforce.reset_mock() - mock_get_algorithm.reset_mock() - mock_get_passphrases.reset_mock() - mock_ssl.reset_mock() - - # test element without text contents - self.assertIsNone(pkc._decrypt(lxml.etree.Element("Test"))) - self.assertFalse(mock_bruteforce.called) - self.assertFalse(mock_get_passphrases.called) - self.assertFalse(mock_ssl.called) - - # test element with a passphrase in the config file - reset() - el = lxml.etree.Element("Test", encrypted="foo") - el.text = "crypted" - mock_get_passphrases.return_value = dict(foo="foopass", - bar="barpass") - mock_get_algorithm.return_value = "bf_cbc" - mock_ssl.return_value = "decrypted with ssl" - self.assertEqual(pkc._decrypt(el), mock_ssl.return_value) - mock_get_passphrases.assert_called_with( - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP) - mock_get_algorithm.assert_called_with( - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP) - mock_ssl.assert_called_with(el.text, "foopass", - algorithm="bf_cbc") - self.assertFalse(mock_bruteforce.called) - - # test failure to decrypt element with a passphrase in the config - reset() - mock_ssl.side_effect = EVPError - self.assertRaises(EVPError, pkc._decrypt, el) - mock_get_passphrases.assert_called_with( - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP) - mock_get_algorithm.assert_called_with( - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP) - mock_ssl.assert_called_with(el.text, "foopass", - algorithm="bf_cbc") - self.assertFalse(mock_bruteforce.called) - - # test element without valid passphrase - reset() - el.set("encrypted", "true") - mock_bruteforce.return_value = "decrypted with bruteforce" - self.assertEqual(pkc._decrypt(el), mock_bruteforce.return_value) - mock_get_passphrases.assert_called_with( - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP) - mock_get_algorithm.assert_called_with( - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP) - mock_bruteforce.assert_called_with(el.text, - passphrases=["foopass", - "barpass"], - algorithm="bf_cbc") - self.assertFalse(mock_ssl.called) - - # test failure to decrypt element without valid passphrase - reset() - mock_bruteforce.side_effect = EVPError - self.assertRaises(EVPError, pkc._decrypt, el) - mock_get_passphrases.assert_called_with( - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP) - mock_get_algorithm.assert_called_with( - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.SETUP) - mock_bruteforce.assert_called_with(el.text, - passphrases=["foopass", - "barpass"], - algorithm="bf_cbc") - self.assertFalse(mock_ssl.called) + self.assertEqual(pkc.create_data(entry, metadata), "privatekey") + pkc.XMLMatch.assert_called_with(metadata) + pkc.get_specificity.assert_called_with(metadata) + pkc._gen_keypair.assert_called_with(metadata, + pkc.XMLMatch.return_value) + self.assertItemsEqual(mock_open.call_args_list, + [call(privkey + ".pub"), call(privkey)]) + pkc.pubkey_creator.get_filename.assert_called_with(group="foo") + pkc.pubkey_creator.write_data.assert_called_with( + "ssh-rsa publickey pubkey.filename\n", group="foo") + pkc.write_data.assert_called_with("privatekey", group="foo") + mock_rmtree.assert_called_with(datastore) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPublicKeyCreator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPublicKeyCreator.py index ef4610fae..f512a6803 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPublicKeyCreator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPublicKeyCreator.py @@ -25,8 +25,8 @@ class TestCfgPublicKeyCreator(TestCfgCreator, TestStructFile): test_obj = CfgPublicKeyCreator should_monitor = False + @patch("Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator.get_cfg", Mock()) def get_obj(self, name=None, fam=None): - Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator.CFG = Mock() return TestCfgCreator.get_obj(self, name=name) @patch("Bcfg2.Server.Plugins.Cfg.CfgCreator.handle_event") diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py index fdfb3a9f7..1b55beded 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py @@ -3,7 +3,7 @@ import sys import errno import lxml.etree import Bcfg2.Options -from Bcfg2.Compat import walk_packages +from Bcfg2.Compat import walk_packages, ConfigParser from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugins.Cfg import * from Bcfg2.Server.Plugin import PluginExecutionError, Specificity @@ -19,7 +19,7 @@ while path != "/": path = os.path.dirname(path) from common import * from TestPlugin import TestSpecificData, TestEntrySet, TestGroupSpool, \ - TestPullTarget + TestPullTarget, TestStructFile class TestCfgBaseFileMatcher(TestSpecificData): @@ -152,28 +152,13 @@ class TestCfgInfo(TestCfgBaseFileMatcher): @patch("Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__init__") def test__init(self, mock__init): ci = self.get_obj("test.txt") - mock__init.assert_called_with(ci, "test.txt", None, None) + mock__init.assert_called_with(ci, "test.txt", None) def test_bind_info_to_entry(self): ci = self.get_obj() self.assertRaises(NotImplementedError, ci.bind_info_to_entry, Mock(), Mock()) - def test__set_info(self): - ci = self.get_obj() - entry = Mock() - entry.attrib = dict() - - info = {"foo": "foo", - "_bar": "bar", - "bar:baz=quux": "quux", - "baz__": "baz", - "__quux": "quux"} - ci._set_info(entry, info) - self.assertItemsEqual(entry.attrib, - dict([(k, v) for k, v in info.items() - if not k.startswith("__")])) - class TestCfgVerifier(TestCfgBaseFileMatcher): test_obj = CfgVerifier @@ -187,6 +172,12 @@ class TestCfgVerifier(TestCfgBaseFileMatcher): class TestCfgCreator(TestCfgBaseFileMatcher): test_obj = CfgCreator path = "/foo/bar/test.txt" + should_monitor = False + + def setUp(self): + TestCfgBaseFileMatcher.setUp(self) + set_setup_default("filemonitor", MagicMock()) + set_setup_default("cfg_passphrase", None) def get_obj(self, name=None): if name is None: @@ -256,62 +247,122 @@ class TestCfgCreator(TestCfgBaseFileMatcher): self.assertRaises(CfgCreationError, cc.write_data, data) +class TestXMLCfgCreator(TestCfgCreator, TestStructFile): + test_obj = XMLCfgCreator + + def setUp(self): + TestCfgCreator.setUp(self) + TestStructFile.setUp(self) + + @patch("Bcfg2.Server.Plugins.Cfg.CfgCreator.handle_event") + @patch("Bcfg2.Server.Plugin.helpers.StructFile.HandleEvent") + def test_handle_event(self, mock_HandleEvent, mock_handle_event): + cc = self.get_obj() + evt = Mock() + cc.handle_event(evt) + mock_HandleEvent.assert_called_with(cc, evt) + mock_handle_event.assert_called_with(cc, evt) + + def test_get_specificity(self): + cc = self.get_obj() + metadata = Mock() + + def reset(): + metadata.group_in_category.reset_mock() + + category = "%s.%s.category" % (self.test_obj.__module__, + self.test_obj.__name__) + @patch(category, None) + def inner(): + cc.xdata = lxml.etree.Element("PrivateKey") + self.assertItemsEqual(cc.get_specificity(metadata), + dict(host=metadata.hostname)) + inner() + + @patch(category, "foo") + def inner2(): + cc.xdata = lxml.etree.Element("PrivateKey") + self.assertItemsEqual(cc.get_specificity(metadata), + dict(group=metadata.group_in_category.return_value, + prio=50)) + metadata.group_in_category.assert_called_with("foo") + + reset() + cc.xdata = lxml.etree.Element("PrivateKey", perhost="true") + self.assertItemsEqual(cc.get_specificity(metadata), + dict(host=metadata.hostname)) + + reset() + cc.xdata = lxml.etree.Element("PrivateKey", category="bar") + self.assertItemsEqual(cc.get_specificity(metadata), + dict(group=metadata.group_in_category.return_value, + prio=50)) + metadata.group_in_category.assert_called_with("bar") + + reset() + cc.xdata = lxml.etree.Element("PrivateKey", prio="10") + self.assertItemsEqual(cc.get_specificity(metadata), + dict(group=metadata.group_in_category.return_value, + prio=10)) + metadata.group_in_category.assert_called_with("foo") + + reset() + cc.xdata = lxml.etree.Element("PrivateKey") + metadata.group_in_category.return_value = '' + self.assertItemsEqual(cc.get_specificity(metadata), + dict(host=metadata.hostname)) + metadata.group_in_category.assert_called_with("foo") + + inner2() + + class TestCfgDefaultInfo(TestCfgInfo): test_obj = CfgDefaultInfo - def get_obj(self, defaults=None): - if defaults is None: - defaults = dict() - return self.test_obj(defaults) + def get_obj(self, *_): + return self.test_obj() - @patch("Bcfg2.Server.Plugins.Cfg.CfgInfo.__init__") - def test__init(self, mock__init): - defaults = Mock() - cdi = self.get_obj(defaults=defaults) - mock__init.assert_called_with(cdi, '') - self.assertEqual(defaults, cdi.defaults) + def test__init(self): + pass def test_handle_event(self): # this CfgInfo handler doesn't handle any events -- it's not # file-driven, but based on the built-in defaults pass - def test_bind_info_to_entry(self): + @patch("Bcfg2.Server.Plugin.default_path_metadata") + def test_bind_info_to_entry(self, mock_default_path_metadata): cdi = self.get_obj() - cdi._set_info = Mock() - entry = Mock() + entry = lxml.etree.Element("Test", name="test") + mock_default_path_metadata.return_value = \ + dict(owner="root", mode="0600") cdi.bind_info_to_entry(entry, Mock()) - cdi._set_info.assert_called_with(entry, cdi.defaults) + self.assertItemsEqual(entry.attrib, + dict(owner="root", mode="0600", name="test")) class TestCfgEntrySet(TestEntrySet): test_obj = CfgEntrySet + def setUp(self): + TestEntrySet.setUp(self) + set_setup_default("cfg_validation", False) + set_setup_default("cfg_handlers", []) + def test__init(self): pass - def test_handlers(self): - # this is really really difficult to mock out, so we just get - # a list of handlers and make sure that it roughly matches - # what's on the filesystem - expected = [] - for submodule in walk_packages(path=Bcfg2.Server.Plugins.Cfg.__path__, - prefix="Bcfg2.Server.Plugins.Cfg."): - expected.append(submodule[1].rsplit('.', 1)[-1]) - self.assertItemsEqual(expected, [h.__name__ for h in handlers()]) - - @patch("Bcfg2.Server.Plugins.Cfg.handlers") - def test_handle_event(self, mock_handlers): + def test_handle_event(self): eset = self.get_obj() eset.entry_init = Mock() - mock_handlers.return_value = [Mock(), Mock(), Mock()] - for hdlr in mock_handlers.return_value: + Bcfg2.Options.setup.cfg_handlers = [Mock(), Mock(), Mock()] + for hdlr in Bcfg2.Options.setup.cfg_handlers: hdlr.__name__ = "handler" eset.entries = dict() def reset(): eset.entry_init.reset_mock() - for hdlr in mock_handlers.return_value: + for hdlr in Bcfg2.Options.setup.cfg_handlers: hdlr.reset_mock() # test that a bogus deleted event is discarded @@ -321,18 +372,19 @@ class TestCfgEntrySet(TestEntrySet): eset.handle_event(evt) self.assertFalse(eset.entry_init.called) self.assertItemsEqual(eset.entries, dict()) - for hdlr in mock_handlers.return_value: + for hdlr in Bcfg2.Options.setup.cfg_handlers: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) # test creation of a new file for action in ["exists", "created", "changed"]: + print("Testing handling of %s events" % action) evt = Mock() evt.code2str.return_value = action evt.filename = os.path.join(datastore, "test.txt") # test with no handler that handles - for hdlr in mock_handlers.return_value: + for hdlr in Bcfg2.Options.setup.cfg_handlers: hdlr.handles.return_value = False hdlr.ignore.return_value = False @@ -340,16 +392,16 @@ class TestCfgEntrySet(TestEntrySet): eset.handle_event(evt) self.assertFalse(eset.entry_init.called) self.assertItemsEqual(eset.entries, dict()) - for hdlr in mock_handlers.return_value: + for hdlr in Bcfg2.Options.setup.cfg_handlers: hdlr.handles.assert_called_with(evt, basename=eset.path) hdlr.ignore.assert_called_with(evt, basename=eset.path) # test with a handler that handles the entry reset() - mock_handlers.return_value[-1].handles.return_value = True + Bcfg2.Options.setup.cfg_handlers[-1].handles.return_value = True eset.handle_event(evt) - eset.entry_init.assert_called_with(evt, mock_handlers.return_value[-1]) - for hdlr in mock_handlers.return_value: + eset.entry_init.assert_called_with(evt, Bcfg2.Options.setup.cfg_handlers[-1]) + for hdlr in Bcfg2.Options.setup.cfg_handlers: hdlr.handles.assert_called_with(evt, basename=eset.path) if not hdlr.return_value: hdlr.ignore.assert_called_with(evt, basename=eset.path) @@ -357,14 +409,14 @@ class TestCfgEntrySet(TestEntrySet): # test with a handler that ignores the entry before one # that handles it reset() - mock_handlers.return_value[0].ignore.return_value = True + Bcfg2.Options.setup.cfg_handlers[0].ignore.return_value = True eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - mock_handlers.return_value[0].handles.assert_called_with(evt, - basename=eset.path) - mock_handlers.return_value[0].ignore.assert_called_with(evt, - basename=eset.path) - for hdlr in mock_handlers.return_value[1:]: + Bcfg2.Options.setup.cfg_handlers[0].handles.assert_called_with( + evt, basename=eset.path) + Bcfg2.Options.setup.cfg_handlers[0].ignore.assert_called_with( + evt, basename=eset.path) + for hdlr in Bcfg2.Options.setup.cfg_handlers[1:]: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) @@ -376,7 +428,7 @@ class TestCfgEntrySet(TestEntrySet): eset.entries[evt.filename] = Mock() eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - for hdlr in mock_handlers.return_value: + for hdlr in Bcfg2.Options.setup.cfg_handlers: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) eset.entries[evt.filename].handle_event.assert_called_with(evt) @@ -386,7 +438,7 @@ class TestCfgEntrySet(TestEntrySet): evt.code2str.return_value = "deleted" eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - for hdlr in mock_handlers.return_value: + for hdlr in Bcfg2.Options.setup.cfg_handlers: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) self.assertItemsEqual(eset.entries, dict()) @@ -438,15 +490,15 @@ class TestCfgEntrySet(TestEntrySet): @patch("Bcfg2.Server.Plugins.Cfg.u_str") @patch("Bcfg2.Server.Plugins.Cfg.b64encode") def test_bind_entry(self, mock_b64encode, mock_u_str): - Bcfg2.Server.Plugins.Cfg.SETUP = dict(validate=False) - mock_u_str.side_effect = lambda x: x + Bcfg2.Options.setup.cfg_validation = False eset = self.get_obj() eset.bind_info_to_entry = Mock() eset._generate_data = Mock() eset.get_handlers = Mock() eset._validate_data = Mock() + eset.setup = dict(validate=False) def reset(): mock_b64encode.reset_mock() @@ -524,7 +576,7 @@ class TestCfgEntrySet(TestEntrySet): # test successful validation entry = reset() - Bcfg2.Server.Plugins.Cfg.SETUP['validate'] = True + Bcfg2.Options.setup.cfg_validation = True bound = eset.bind_entry(entry, metadata) eset.bind_info_to_entry.assert_called_with(entry, metadata) eset._generate_data.assert_called_with(entry, metadata) @@ -546,16 +598,16 @@ class TestCfgEntrySet(TestEntrySet): def test_get_handlers(self): eset = self.get_obj() eset.entries['test1.txt'] = CfgInfo("test1.txt") - eset.entries['test2.txt'] = CfgGenerator("test2.txt", Mock(), None) + eset.entries['test2.txt'] = CfgGenerator("test2.txt", Mock()) eset.entries['test2.txt'].specific.matches.return_value = True eset.entries['test3.txt'] = CfgInfo("test3.txt") - eset.entries['test4.txt'] = CfgGenerator("test4.txt", Mock(), None) + eset.entries['test4.txt'] = CfgGenerator("test4.txt", Mock()) eset.entries['test4.txt'].specific.matches.return_value = False - eset.entries['test5.txt'] = CfgGenerator("test5.txt", Mock(), None) + eset.entries['test5.txt'] = CfgGenerator("test5.txt", Mock()) eset.entries['test5.txt'].specific.matches.return_value = True - eset.entries['test6.txt'] = CfgVerifier("test6.txt", Mock(), None) + eset.entries['test6.txt'] = CfgVerifier("test6.txt", Mock()) eset.entries['test6.txt'].specific.matches.return_value = True - eset.entries['test7.txt'] = CfgFilter("test7.txt", Mock(), None) + eset.entries['test7.txt'] = CfgFilter("test7.txt", Mock()) eset.entries['test7.txt'].specific.matches.return_value = False def reset(): @@ -603,24 +655,24 @@ class TestCfgEntrySet(TestEntrySet): if hasattr(entry.specific.matches, "called"): self.assertFalse(entry.specific.matches.called) - def test_bind_info_to_entry(self): - default_info = Bcfg2.Server.Plugins.Cfg.DEFAULT_INFO + @patch("Bcfg2.Server.Plugins.Cfg.CfgDefaultInfo") + def test_bind_info_to_entry(self, mock_DefaultInfo): eset = self.get_obj() eset.get_handlers = Mock() eset.get_handlers.return_value = [] - Bcfg2.Server.Plugins.Cfg.DEFAULT_INFO = Mock() metadata = Mock() def reset(): eset.get_handlers.reset_mock() - Bcfg2.Server.Plugins.Cfg.DEFAULT_INFO.reset_mock() + mock_DefaultInfo.reset_mock() return lxml.etree.Element("Path", name="/test.txt") # test with no info handlers entry = reset() eset.bind_info_to_entry(entry, metadata) eset.get_handlers.assert_called_with(metadata, CfgInfo) - Bcfg2.Server.Plugins.Cfg.DEFAULT_INFO.bind_info_to_entry.assert_called_with(entry, metadata) + mock_DefaultInfo.return_value.bind_info_to_entry.assert_called_with( + entry, metadata) self.assertEqual(entry.get("type"), "file") # test with one info handler @@ -629,7 +681,8 @@ class TestCfgEntrySet(TestEntrySet): eset.get_handlers.return_value = [handler] eset.bind_info_to_entry(entry, metadata) eset.get_handlers.assert_called_with(metadata, CfgInfo) - Bcfg2.Server.Plugins.Cfg.DEFAULT_INFO.bind_info_to_entry.assert_called_with(entry, metadata) + mock_DefaultInfo.return_value.bind_info_to_entry.assert_called_with( + entry, metadata) handler.bind_info_to_entry.assert_called_with(entry, metadata) self.assertEqual(entry.get("type"), "file") @@ -639,7 +692,8 @@ class TestCfgEntrySet(TestEntrySet): eset.get_handlers.return_value = handlers eset.bind_info_to_entry(entry, metadata) eset.get_handlers.assert_called_with(metadata, CfgInfo) - Bcfg2.Server.Plugins.Cfg.DEFAULT_INFO.bind_info_to_entry.assert_called_with(entry, metadata) + mock_DefaultInfo.return_value.bind_info_to_entry.assert_called_with( + entry, metadata) # we don't care which handler gets called as long as exactly # one of them does called = 0 @@ -650,8 +704,6 @@ class TestCfgEntrySet(TestEntrySet): self.assertEqual(called, 1) self.assertEqual(entry.get("type"), "file") - Bcfg2.Server.Plugins.Cfg.DEFAULT_INFO = default_info - def test_create_data(self): eset = self.get_obj() eset.best_matching = Mock() @@ -753,35 +805,16 @@ class TestCfgEntrySet(TestEntrySet): class TestCfg(TestGroupSpool, TestPullTarget): test_obj = Cfg + def setUp(self): + TestGroupSpool.setUp(self) + TestPullTarget.setUp(self) + set_setup_default("cfg_handlers", []) + def get_obj(self, core=None): if core is None: core = Mock() - core.setup = MagicMock() return TestGroupSpool.get_obj(self, core=core) - @patch("Bcfg2.Server.Plugin.GroupSpool.__init__") - @patch("Bcfg2.Server.Plugin.PullTarget.__init__") - def test__init(self, mock_pulltarget_init, mock_groupspool_init): - core = Mock() - core.setup = MagicMock() - cfg = self.test_obj(core, datastore) - mock_pulltarget_init.assert_called_with(cfg) - mock_groupspool_init.assert_called_with(cfg, core, datastore) - core.setup.add_option.assert_called_with("validate", - Bcfg2.Options.CFG_VALIDATION) - core.setup.reparse.assert_called_with() - - core.reset_mock() - core.setup.reset_mock() - mock_pulltarget_init.reset_mock() - mock_groupspool_init.reset_mock() - core.setup.__contains__.return_value = True - cfg = self.test_obj(core, datastore) - mock_pulltarget_init.assert_called_with(cfg) - mock_groupspool_init.assert_called_with(cfg, core, datastore) - self.assertFalse(core.setup.add_option.called) - self.assertFalse(core.setup.reparse.called) - def test_has_generator(self): cfg = self.get_obj() cfg.entries = dict() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDecisions.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDecisions.py new file mode 100644 index 000000000..8b4df8abb --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDecisions.py @@ -0,0 +1,60 @@ +import os +import sys +import lxml.etree +from mock import Mock, MagicMock, patch +from Bcfg2.Server.Plugins.Decisions import * + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * +from TestPlugin import TestStructFile, TestPlugin, TestDecision + + +class TestDecisionFile(TestStructFile): + test_obj = DecisionFile + + def test_get_decisions(self): + df = self.get_obj() + metadata = Mock() + + df.xdata = None + self.assertItemsEqual(df.get_decisions(metadata), []) + + df.xdata = lxml.etree.Element("Decisions") + df.XMLMatch = Mock() + df.XMLMatch.return_value = lxml.etree.Element("Decisions") + lxml.etree.SubElement(df.XMLMatch.return_value, + "Decision", type="Service", name='*') + lxml.etree.SubElement(df.XMLMatch.return_value, + "Decision", type="Path", + name='/etc/apt/apt.conf') + + self.assertItemsEqual(df.get_decisions(metadata), + [("Service", '*'), + ("Path", '/etc/apt/apt.conf')]) + df.XMLMatch.assert_called_with(metadata) + + +class TestDecisions(TestPlugin, TestDecision): + test_obj = Decisions + + def test_GetDecisions(self): + d = self.get_obj() + d.whitelist = Mock() + d.blacklist = Mock() + metadata = Mock() + + self.assertEqual(d.GetDecisions(metadata, "whitelist"), + d.whitelist.get_decisions.return_value) + d.whitelist.get_decisions.assert_called_with(metadata) + + self.assertEqual(d.GetDecisions(metadata, "blacklist"), + d.blacklist.get_decisions.return_value) + d.blacklist.get_decisions.assert_called_with(metadata) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDefaults.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDefaults.py index 7be3d8e84..9b4a6af88 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDefaults.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDefaults.py @@ -1,5 +1,6 @@ import os import sys +import copy import lxml.etree from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugins.Defaults import * @@ -62,3 +63,31 @@ class TestDefaults(TestRules, TestGoalValidator): def test__regex_enabled(self): r = self.get_obj() self.assertTrue(r._regex_enabled) + + def _do_test(self, name, groups=None): + if groups is None: + groups = [] + d = self.get_obj() + metadata = Mock(groups=groups) + config = lxml.etree.Element("Configuration") + struct = lxml.etree.SubElement(config, "Bundle", name=name) + entry = copy.deepcopy(self.abstract[name]) + struct.append(entry) + d.validate_goals(metadata, config) + self.assertXMLEqual(entry, self.concrete[name]) + + def _do_test_failure(self, name, groups=None, handles=None): + if groups is None: + groups = [] + d = self.get_obj() + metadata = Mock(groups=groups) + config = lxml.etree.Element("Configuration") + struct = lxml.etree.SubElement(config, "Bundle", name=name) + orig = copy.deepcopy(self.abstract[name]) + entry = copy.deepcopy(self.abstract[name]) + struct.append(entry) + d.validate_goals(metadata, config) + self.assertXMLEqual(entry, orig) + + def test_regex(self): + self._do_test('regex') diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py index a07fffba1..d3fa15236 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -6,7 +6,6 @@ import socket import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugins.Metadata import * from mock import Mock, MagicMock, patch # add all parent testsuite directories to sys.path to allow (most) @@ -19,9 +18,13 @@ while path != "/": break path = os.path.dirname(path) from common import * +from Bcfg2.Server.Plugins.Metadata import load_django_models from TestPlugin import TestXMLFileBacked, TestMetadata as _TestMetadata, \ TestClientRunHooks, TestDatabaseBacked +load_django_models() +from Bcfg2.Server.Plugins.Metadata import * + def get_clients_test_tree(): return lxml.etree.XML(''' @@ -88,18 +91,18 @@ def get_groups_test_tree(): </Groups>''').getroottree() -def get_metadata_object(core=None, watch_clients=False, use_db=False): +def get_metadata_object(core=None): if core is None: core = Mock() - core.setup = MagicMock() core.metadata_cache = MagicMock() - core.setup.cfp.getboolean = Mock(return_value=use_db) + set_setup_default("password") @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock()) @patchIf(not isinstance(lxml.etree.Element, Mock), "lxml.etree.Element", Mock()) + def inner(): - return Metadata(core, datastore, watch_clients=watch_clients) + return Metadata(core) return inner() @@ -108,117 +111,117 @@ class TestMetadataDB(DBModelTestCase): models = [MetadataClientModel] -if HAS_DJANGO or can_skip: - class TestClientVersions(TestDatabaseBacked): - test_clients = dict(client1="1.2.0", - client2="1.2.2", - client3="1.3.0pre1", - client4="1.1.0", - client5=None, - client6=None) - - @skipUnless(HAS_DJANGO, "Django not found") - def setUp(self): - self.test_obj = ClientVersions - syncdb(TestMetadataDB) - for client, version in self.test_clients.items(): - MetadataClientModel(hostname=client, version=version).save() - - def test__contains(self): - v = self.get_obj() - self.assertIn("client1", v) - self.assertIn("client5", v) - self.assertNotIn("client__contains", v) - - def test_keys(self): - v = self.get_obj() - self.assertItemsEqual(self.test_clients.keys(), v.keys()) - - def test__setitem(self): - v = self.get_obj() - - # test setting version of existing client - v["client1"] = "1.2.3" - self.assertIn("client1", v) - self.assertEqual(v['client1'], "1.2.3") - client = MetadataClientModel.objects.get(hostname="client1") - self.assertEqual(client.version, "1.2.3") - - # test adding new client - new = "client__setitem" - v[new] = "1.3.0" - self.assertIn(new, v) - self.assertEqual(v[new], "1.3.0") - client = MetadataClientModel.objects.get(hostname=new) - self.assertEqual(client.version, "1.3.0") - - # test adding new client with no version - new2 = "client__setitem_2" - v[new2] = None - self.assertIn(new2, v) - self.assertEqual(v[new2], None) - client = MetadataClientModel.objects.get(hostname=new2) - self.assertEqual(client.version, None) - - def test__getitem(self): - v = self.get_obj() - - # test getting existing client - self.assertEqual(v['client2'], "1.2.2") - self.assertIsNone(v['client5']) - - # test exception on nonexistent client - expected = KeyError - try: - v['clients__getitem'] - except expected: - pass - except: - err = sys.exc_info()[1] - self.assertFalse(True, "%s raised instead of %s" % - (err.__class__.__name__, - expected.__class__.__name__)) - else: - self.assertFalse(True, - "%s not raised" % expected.__class__.__name__) +class TestClientVersions(TestDatabaseBacked): + test_clients = dict(client1="1.2.0", + client2="1.2.2", + client3="1.3.0pre1", + client4="1.1.0", + client5=None, + client6=None) + + @skipUnless(HAS_DJANGO, "Django not found") + def setUp(self): + TestDatabaseBacked.setUp(self) + self.test_obj = ClientVersions + syncdb(TestMetadataDB) + for client, version in self.test_clients.items(): + MetadataClientModel(hostname=client, version=version).save() + + def test__contains(self): + v = self.get_obj() + self.assertIn("client1", v) + self.assertIn("client5", v) + self.assertNotIn("client__contains", v) + + def test_keys(self): + v = self.get_obj() + self.assertItemsEqual(self.test_clients.keys(), v.keys()) + + def test__setitem(self): + v = self.get_obj() + + # test setting version of existing client + v["client1"] = "1.2.3" + self.assertIn("client1", v) + self.assertEqual(v['client1'], "1.2.3") + client = MetadataClientModel.objects.get(hostname="client1") + self.assertEqual(client.version, "1.2.3") + + # test adding new client + new = "client__setitem" + v[new] = "1.3.0" + self.assertIn(new, v) + self.assertEqual(v[new], "1.3.0") + client = MetadataClientModel.objects.get(hostname=new) + self.assertEqual(client.version, "1.3.0") + + # test adding new client with no version + new2 = "client__setitem_2" + v[new2] = None + self.assertIn(new2, v) + self.assertEqual(v[new2], None) + client = MetadataClientModel.objects.get(hostname=new2) + self.assertEqual(client.version, None) + + def test__getitem(self): + v = self.get_obj() + + # test getting existing client + self.assertEqual(v['client2'], "1.2.2") + self.assertIsNone(v['client5']) + + # test exception on nonexistent client + expected = KeyError + try: + v['clients__getitem'] + except expected: + pass + except: + err = sys.exc_info()[1] + self.assertFalse(True, "%s raised instead of %s" % + (err.__class__.__name__, + expected.__class__.__name__)) + else: + self.assertFalse(True, + "%s not raised" % expected.__class__.__name__) - def test__len(self): - v = self.get_obj() - self.assertEqual(len(v), MetadataClientModel.objects.count()) + def test__len(self): + v = self.get_obj() + self.assertEqual(len(v), MetadataClientModel.objects.count()) - def test__iter(self): - v = self.get_obj() - self.assertItemsEqual([h for h in iter(v)], v.keys()) + def test__iter(self): + v = self.get_obj() + self.assertItemsEqual([h for h in iter(v)], v.keys()) - def test__delitem(self): - v = self.get_obj() + def test__delitem(self): + v = self.get_obj() - # test adding new client - new = "client__delitem" - v[new] = "1.3.0" + # test adding new client + new = "client__delitem" + v[new] = "1.3.0" - del v[new] - self.assertIn(new, v) - self.assertIsNone(v[new]) + del v[new] + self.assertIn(new, v) + self.assertIsNone(v[new]) class TestXMLMetadataConfig(TestXMLFileBacked): test_obj = XMLMetadataConfig path = os.path.join(datastore, 'Metadata', 'clients.xml') - def get_obj(self, basefile="clients.xml", core=None, watch_clients=False): - self.metadata = get_metadata_object(core=core, - watch_clients=watch_clients) + def get_obj(self, basefile="clients.xml", core=None): + self.metadata = get_metadata_object(core=core) @patchIf(not isinstance(lxml.etree.Element, Mock), "lxml.etree.Element", Mock()) def inner(): - return XMLMetadataConfig(self.metadata, watch_clients, basefile) + return XMLMetadataConfig(self.metadata, basefile) return inner() + @patch("Bcfg2.Server.FileMonitor.get_fam", Mock()) def test__init(self): xmc = self.get_obj() - self.assertEqual(self.metadata.core.fam, xmc.fam) - self.assertFalse(xmc.fam.AddMonitor.called) + self.assertNotIn(call(xmc.basefile), + xmc.fam.AddMonitor.call_args_list) def test_xdata(self): config = self.get_obj() @@ -262,20 +265,15 @@ class TestXMLMetadataConfig(TestXMLFileBacked): self.assertEqual(config.base_xdata, "<test/>") def test_add_monitor(self): - core = MagicMock() - config = self.get_obj(core=core) + config = self.get_obj() + config.fam = Mock() fname = "test.xml" fpath = os.path.join(self.metadata.data, fname) config.extras = [] config.add_monitor(fpath) - self.assertFalse(core.fam.AddMonitor.called) - self.assertEqual(config.extras, [fpath]) - - config = self.get_obj(core=core, watch_clients=True) - config.add_monitor(fpath) - core.fam.AddMonitor.assert_called_with(fpath, config.metadata) + config.fam.AddMonitor.assert_called_with(fpath, config.metadata) self.assertItemsEqual(config.extras, [fpath]) def test_Index(self): @@ -480,11 +478,16 @@ class TestClientMetadata(Bcfg2TestCase): class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): test_obj = Metadata - use_db = False - def get_obj(self, core=None, watch_clients=False): - return get_metadata_object(core=core, watch_clients=watch_clients, - use_db=self.use_db) + def setUp(self): + _TestMetadata.setUp(self) + TestClientRunHooks.setUp(self) + TestDatabaseBacked.setUp(self) + Bcfg2.Options.setup.metadata_db = False + Bcfg2.Options.setup.authentication = "cert+password" + + def get_obj(self, core=None): + return get_metadata_object(core=core) @skipUnless(HAS_DJANGO, "Django not found") def test__use_db(self): @@ -504,33 +507,24 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): client_name = "%s%s" % (prefix, i) return client_name - def test__init(self): - # test with watch_clients=False + @patch("Bcfg2.Server.FileMonitor.get_fam") + def test__init(self, mock_get_fam): core = MagicMock() metadata = self.get_obj(core=core) - self.assertIsInstance(metadata, Bcfg2.Server.Plugin.Plugin) - self.assertIsInstance(metadata, Bcfg2.Server.Plugin.Metadata) - self.assertIsInstance(metadata, Bcfg2.Server.Plugin.ClientRunHooks) - self.assertIsInstance(metadata.clients_xml, XMLMetadataConfig) - self.assertIsInstance(metadata.groups_xml, XMLMetadataConfig) - self.assertIsInstance(metadata.query, MetadataQuery) - self.assertEqual(metadata.states, dict()) - - # test with watch_clients=True - core.fam = MagicMock() - metadata = self.get_obj(core=core, watch_clients=True) self.assertEqual(len(metadata.states), 2) - core.fam.AddMonitor.assert_any_call(os.path.join(metadata.data, - "groups.xml"), - metadata) - core.fam.AddMonitor.assert_any_call(os.path.join(metadata.data, - "clients.xml"), - metadata) - - core.fam.reset_mock() - core.fam.AddMonitor = Mock(side_effect=IOError) + mock_get_fam.return_value.AddMonitor.assert_any_call( + os.path.join(metadata.data, "groups.xml"), + metadata) + mock_get_fam.return_value.AddMonitor.assert_any_call( + os.path.join(metadata.data, "clients.xml"), + metadata) + + mock_get_fam.reset_mock() + fam = Mock() + fam.AddMonitor = Mock(side_effect=IOError) + mock_get_fam.return_value = fam self.assertRaises(Bcfg2.Server.Plugin.PluginInitError, - self.get_obj, core=core, watch_clients=True) + self.get_obj, core=core) @patch('os.makedirs', Mock()) @patch('%s.open' % builtins) @@ -591,6 +585,7 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): def test_add_group(self): metadata = self.get_obj() metadata.groups_xml.write = Mock() + metadata.groups_xml.load_xml = Mock() metadata.groups_xml.data = lxml.etree.XML('<Groups/>').getroottree() metadata.groups_xml.basedata = copy.copy(metadata.groups_xml.data) @@ -623,6 +618,7 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): def test_update_group(self): metadata = self.get_obj() metadata.groups_xml.write_xml = Mock() + metadata.groups_xml.load_xml = Mock() metadata.groups_xml.data = copy.deepcopy(get_groups_test_tree()) metadata.groups_xml.basedata = copy.copy(metadata.groups_xml.data) @@ -640,6 +636,7 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): def test_remove_group(self): metadata = self.get_obj() metadata.groups_xml.write_xml = Mock() + metadata.groups_xml.load_xml = Mock() metadata.groups_xml.data = copy.deepcopy(get_groups_test_tree()) metadata.groups_xml.basedata = copy.copy(metadata.groups_xml.data) @@ -655,6 +652,7 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): def test_add_bundle(self): metadata = self.get_obj() metadata.groups_xml.write = Mock() + metadata.groups_xml.load_xml = Mock() metadata.groups_xml.data = lxml.etree.XML('<Groups/>').getroottree() metadata.groups_xml.basedata = copy.copy(metadata.groups_xml.data) @@ -678,6 +676,7 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): def test_remove_bundle(self): metadata = self.get_obj() metadata.groups_xml.write_xml = Mock() + metadata.groups_xml.load_xml = Mock() metadata.groups_xml.data = copy.deepcopy(get_groups_test_tree()) metadata.groups_xml.basedata = copy.copy(metadata.groups_xml.data) @@ -693,6 +692,7 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): def test_add_client(self): metadata = self.get_obj() metadata.clients_xml.write = Mock() + metadata.clients_xml.load_xml = Mock() metadata.clients_xml.data = lxml.etree.XML('<Clients/>').getroottree() metadata.clients_xml.basedata = copy.copy(metadata.clients_xml.data) @@ -727,6 +727,7 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): def test_update_client(self): metadata = self.get_obj() metadata.clients_xml.write_xml = Mock() + metadata.clients_xml.load_xml = Mock() metadata.clients_xml.data = copy.deepcopy(get_clients_test_tree()) metadata.clients_xml.basedata = copy.copy(metadata.clients_xml.data) @@ -762,7 +763,7 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): metadata.clients_xml.xdata = copy.deepcopy(get_clients_test_tree()) metadata._handle_clients_xml_event(Mock()) - if not self.use_db: + if not Bcfg2.Options.setup.metadata_db: self.assertItemsEqual(metadata.clients, dict([(c.get("name"), c.get("profile")) for c in get_clients_test_tree().findall("//Client")])) @@ -1251,10 +1252,13 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): class TestMetadataBase(TestMetadata): """ base test object for testing Metadata with database enabled """ __test__ = False - use_db = True @skipUnless(HAS_DJANGO, "Django not found") def setUp(self): + _TestMetadata.setUp(self) + TestClientRunHooks.setUp(self) + TestDatabaseBacked.setUp(self) + Bcfg2.Options.setup.metadata_db = True syncdb(TestMetadataDB) def load_clients_data(self, metadata=None, xdata=None): @@ -1274,25 +1278,24 @@ class TestMetadataBase(TestMetadata): return client_name @patch('os.path.exists') - def test__init(self, mock_exists): - core = MagicMock() - core.fam = Mock() + @patch('Bcfg2.Server.FileMonitor.get_fam') + def test__init(self, mock_get_fam, mock_exists): mock_exists.return_value = False - metadata = self.get_obj(core=core, watch_clients=True) + metadata = self.get_obj() self.assertIsInstance(metadata, Bcfg2.Server.Plugin.DatabaseBacked) - core.fam.AddMonitor.assert_called_once_with(os.path.join(metadata.data, - "groups.xml"), - metadata) + mock_get_fam.return_value.AddMonitor.assert_called_with( + os.path.join(metadata.data, "groups.xml"), + metadata) mock_exists.return_value = True - core.fam.reset_mock() - metadata = self.get_obj(core=core, watch_clients=True) - core.fam.AddMonitor.assert_any_call(os.path.join(metadata.data, - "groups.xml"), - metadata) - core.fam.AddMonitor.assert_any_call(os.path.join(metadata.data, - "clients.xml"), - metadata) + mock_get_fam.reset_mock() + metadata = self.get_obj() + mock_get_fam.return_value.AddMonitor.assert_any_call( + os.path.join(metadata.data, "groups.xml"), + metadata) + mock_get_fam.return_value.AddMonitor.assert_any_call( + os.path.join(metadata.data, "clients.xml"), + metadata) def test_add_group(self): pass @@ -1356,12 +1359,7 @@ class TestMetadataBase(TestMetadata): class TestMetadata_NoClientsXML(TestMetadataBase): """ test Metadata without a clients.xml. we have to disable or override tests that rely on client options """ - # only run these tests if it's possible to skip tests or if we - # have django. otherwise they'll all get run because our fake - # skipping decorators for python < 2.7 won't work when they - # decorate setUp() - if can_skip or HAS_DJANGO: - __test__ = True + __test__ = True def load_groups_data(self, metadata=None, xdata=None): if metadata is None: @@ -1525,17 +1523,13 @@ class TestMetadata_NoClientsXML(TestMetadataBase): class TestMetadata_ClientsXML(TestMetadataBase): """ test Metadata with a clients.xml. """ - # only run these tests if it's possible to skip tests or if we - # have django. otherwise they'll all get run because our fake - # skipping decorators for python < 2.7 won't work when they - # decorate setUp() - if can_skip or HAS_DJANGO: - __test__ = True + __test__ = True def load_clients_data(self, metadata=None, xdata=None): if metadata is None: metadata = self.get_obj() - metadata.core.fam = Mock() + fam = Bcfg2.Server.FileMonitor._FAM + Bcfg2.Server.FileMonitor._FAM = MagicMock() @patchIf(not isinstance(lxml.etree.Element, Mock), "lxml.etree.Element", Mock()) def inner(): @@ -1543,5 +1537,7 @@ class TestMetadata_ClientsXML(TestMetadataBase): inner() metadata = TestMetadata.load_clients_data(self, metadata=metadata, xdata=xdata) - return TestMetadataBase.load_clients_data(self, metadata=metadata, - xdata=xdata) + rv = TestMetadataBase.load_clients_data(self, metadata=metadata, + xdata=xdata) + Bcfg2.Server.FileMonitor._FAM = fam + return rv diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index 2face023f..32766b5c1 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -1,8 +1,8 @@ import os import re import sys -import copy -import time +import shutil +import tempfile import lxml.etree import Bcfg2.version import Bcfg2.Server @@ -19,54 +19,22 @@ while path != "/": break path = os.path.dirname(path) from common import * -from Bcfg2.Server.Plugins.Probes import * -from TestPlugin import TestEntrySet, TestProbing, TestConnector, \ +from Bcfg2.Server.Plugins.Probes import load_django_models +from TestPlugin import TestEntrySet, TestPlugin, \ TestDatabaseBacked -# test data for JSON and YAML tests -test_data = dict(a=1, b=[1, 2, 3], c="test", - d=dict(a=1, b=dict(a=1), c=(1, "2", 3))) - - -class FakeElement(lxml.etree._Element): - getroottree = Mock() - - def __init__(self, el): - self._element = el - - def __getattribute__(self, attr): - el = lxml.etree._Element.__getattribute__(self, - '__dict__')['_element'] - if attr == "getroottree": - return FakeElement.getroottree - elif attr == "_element": - return el - else: - return getattr(el, attr) - - -class StoringElement(object): - OriginalElement = copy.copy(lxml.etree.Element) - - def __init__(self): - self.element = None - self.return_value = None - - def __call__(self, *args, **kwargs): - self.element = self.OriginalElement(*args, **kwargs) - self.return_value = FakeElement(self.element) - return self.return_value +load_django_models() +from Bcfg2.Server.Plugins.Probes import * +if HAS_JSON: + json = json -class StoringSubElement(object): - OriginalSubElement = copy.copy(lxml.etree.SubElement) +if HAS_YAML: + yaml = yaml - def __call__(self, parent, tag, **kwargs): - try: - return self.OriginalSubElement(parent._element, tag, - **kwargs) - except AttributeError: - return self.OriginalSubElement(parent, tag, **kwargs) +# test data for JSON and YAML tests +test_data = dict(a=1, b=[1, 2, 3], c="test", + d=dict(a=1, b=dict(a=1), c=(1, "2", 3))) class FakeList(list): @@ -75,19 +43,8 @@ class FakeList(list): class TestProbesDB(DBModelTestCase): if HAS_DJANGO: - models = [ProbesGroupsModel, ProbesDataModel] - - -class TestClientProbeDataSet(Bcfg2TestCase): - def test__init(self): - ds = ClientProbeDataSet() - self.assertLessEqual(ds.timestamp, time.time()) - self.assertIsInstance(ds, dict) - self.assertNotIn("timestamp", ds) - - ds = ClientProbeDataSet(timestamp=123) - self.assertEqual(ds.timestamp, 123) - self.assertNotIn("timestamp", ds) + models = [ProbesGroupsModel, + ProbesDataModel] class TestProbeData(Bcfg2TestCase): @@ -109,19 +66,22 @@ class TestProbeData(Bcfg2TestCase): def test_xdata(self): xdata = lxml.etree.Element("test") lxml.etree.SubElement(xdata, "test2") - data = ProbeData(lxml.etree.tostring(xdata, - xml_declaration=False).decode('UTF-8')) + data = ProbeData( + lxml.etree.tostring(xdata, + xml_declaration=False).decode('UTF-8')) self.assertIsNotNone(data.xdata) self.assertIsNotNone(data.xdata.find("test2")) - @skipUnless(HAS_JSON, "JSON libraries not found, skipping JSON tests") + @skipUnless(HAS_JSON, + "JSON libraries not found, skipping JSON tests") def test_json(self): jdata = json.dumps(test_data) data = ProbeData(jdata) self.assertIsNotNone(data.json) self.assertItemsEqual(test_data, data.json) - @skipUnless(HAS_YAML, "YAML libraries not found, skipping YAML tests") + @skipUnless(HAS_YAML, + "YAML libraries not found, skipping YAML tests") def test_yaml(self): jdata = yaml.dump(test_data) data = ProbeData(jdata) @@ -135,22 +95,20 @@ class TestProbeSet(TestEntrySet): ignore = ["foo~", ".#foo", ".foo.swp", ".foo.swx", "probed.xml"] bogus_names = ["test.py"] - def get_obj(self, path=datastore, fam=None, encoding=None, + def get_obj(self, path=datastore, encoding=None, plugin_name="Probes", basename=None): # get_obj() accepts the basename argument, accepted by the # parent get_obj() method, and just throws it away, since # ProbeSet uses a regex for the "basename" - if fam is None: - fam = Mock() - rv = self.test_obj(path, fam, encoding, plugin_name) + rv = self.test_obj(path, plugin_name) rv.entry_type = MagicMock() return rv - def test__init(self): - fam = Mock() - ps = self.get_obj(fam=fam) + @patch("Bcfg2.Server.FileMonitor.get_fam") + def test__init(self, mock_get_fam): + ps = self.get_obj() self.assertEqual(ps.plugin_name, "Probes") - fam.AddMonitor.assert_called_with(datastore, ps) + mock_get_fam.return_value.AddMonitor.assert_called_with(datastore, ps) TestEntrySet.test__init(self) def test_HandleEvent(self): @@ -243,418 +201,192 @@ group-specific""" assert False, "Strange probe found in get_probe_data() return" -class TestProbes(TestProbing, TestConnector, TestDatabaseBacked): +class TestProbes(TestPlugin): test_obj = Probes - def get_obj(self, core=None): - return TestDatabaseBacked.get_obj(self, core=core) - - def get_test_probedata(self): - test_xdata = lxml.etree.Element("test") - lxml.etree.SubElement(test_xdata, "test", foo="foo") - rv = dict() - rv["foo.example.com"] = ClientProbeDataSet(timestamp=time.time()) - rv["foo.example.com"]["xml"] = \ - ProbeData(lxml.etree.tostring(test_xdata, - xml_declaration=False).decode('UTF-8')) - rv["foo.example.com"]["text"] = ProbeData("freeform text") - rv["foo.example.com"]["multiline"] = ProbeData("""multiple + test_xdata = lxml.etree.Element("test") + lxml.etree.SubElement(test_xdata, "test", foo="foo") + test_xdoc = lxml.etree.tostring(test_xdata, + xml_declaration=False).decode('UTF-8') + + data = dict() + data['xml'] = "group:group\n" + test_xdoc + data['text'] = "freeform text" + data['multiline'] = """multiple lines of freeform text -""") - rv["bar.example.com"] = ClientProbeDataSet(timestamp=time.time()) - rv["bar.example.com"]["empty"] = ProbeData("") - if HAS_JSON: - rv["bar.example.com"]["json"] = ProbeData(json.dumps(test_data)) - if HAS_YAML: - rv["bar.example.com"]["yaml"] = ProbeData(yaml.dump(test_data)) - return rv - - def get_test_cgroups(self): - return {"foo.example.com": ["group", "group with spaces", - "group-with-dashes"], - "bar.example.com": []} - - def get_probes_object(self, use_db=False, load_data=None): - core = MagicMock() - core.setup.cfp.getboolean = Mock() - core.setup.cfp.getboolean.return_value = use_db - if load_data is None: - load_data = MagicMock() - # we have to patch load_data() in a funny way because - # different versions of Mock have different scopes for - # patching. in some versions, a patch applied to - # get_probes_object() would only apply to that function, while - # in others it would also apply to the calling function (e.g., - # test__init(), which relies on being able to check the calls - # of load_data(), and thus on load_data() being consistently - # mocked) - @patch("%s.%s.load_data" % (self.test_obj.__module__, - self.test_obj.__name__), new=load_data) - def inner(): - return self.get_obj(core) - - rv = inner() - rv.allowed_cgroups = [re.compile("^.*$")] - return rv +group:group-with-dashes +group: group:with:colons +""" + data['empty'] = '' + data['almost_empty'] = 'group: other_group' + if HAS_JSON: + data['json'] = json.dumps(test_data) + if HAS_YAML: + data['yaml'] = yaml.dump(test_data) + + def setUp(self): + Bcfg2TestCase.setUp(self) + set_setup_default("probes_db") + set_setup_default("probes_allowed_groups", [re.compile(".*")]) + self.datastore = None + Bcfg2.Server.Cache.expire("Probes") + + def tearDown(self): + Bcfg2.Server.Cache.expire("Probes") + if self.datastore is not None: + shutil.rmtree(self.datastore) + self.datastore = None + Bcfg2.Options.setup.repository = datastore + + def get_obj(self): + if not Bcfg2.Options.setup.probes_db: + # actually use a real datastore so we can read and write + # probed.xml + if self.datastore is None: + self.datastore = tempfile.mkdtemp() + Bcfg2.Options.setup.repository = self.datastore + datadir = os.path.join(self.datastore, self.test_obj.name) + if not os.path.exists(datadir): + os.makedirs(datadir) + return TestPlugin.get_obj(self) def test__init(self): - mock_load_data = Mock() - probes = self.get_probes_object(load_data=mock_load_data) - probes.core.fam.AddMonitor.assert_called_with(os.path.join(datastore, - probes.name), - probes.probes) - mock_load_data.assert_any_call() - self.assertEqual(probes.probedata, ClientProbeDataSet()) - self.assertEqual(probes.cgroups, dict()) - - @patch("Bcfg2.Server.Plugins.Probes.Probes.load_data", Mock()) - def test__use_db(self): - probes = self.get_probes_object() - self.assertFalse(probes._use_db) - probes.core.setup.cfp.getboolean.assert_called_with("probes", - "use_database", - default=False) - - @skipUnless(HAS_DJANGO, "Django not found, skipping") - @patch("Bcfg2.Server.Plugins.Probes.Probes._write_data_db", Mock()) - @patch("Bcfg2.Server.Plugins.Probes.Probes._write_data_xml", Mock()) - def test_write_data_xml(self): - probes = self.get_probes_object(use_db=False) - probes.write_data("test") - probes._write_data_xml.assert_called_with("test") - self.assertFalse(probes._write_data_db.called) - - @skipUnless(HAS_DJANGO, "Django not found, skipping") - @patch("Bcfg2.Server.Plugins.Probes.Probes._write_data_db", Mock()) - @patch("Bcfg2.Server.Plugins.Probes.Probes._write_data_xml", Mock()) - def test_write_data_db(self): - probes = self.get_probes_object(use_db=True) - probes.write_data("test") - probes._write_data_db.assert_called_with("test") - self.assertFalse(probes._write_data_xml.called) - - def test__write_data_xml(self): - probes = self.get_probes_object(use_db=False) - probes.probedata = self.get_test_probedata() - probes.cgroups = self.get_test_cgroups() - - @patch("lxml.etree.Element") - @patch("lxml.etree.SubElement", StoringSubElement()) - def inner(mock_Element): - mock_Element.side_effect = StoringElement() - probes._write_data_xml(None) - - top = mock_Element.side_effect.return_value - write = top.getroottree.return_value.write - self.assertEqual(write.call_args[0][0], - os.path.join(datastore, probes.name, - "probed.xml")) - - data = top._element - foodata = data.find("Client[@name='foo.example.com']") - self.assertIsNotNone(foodata) - self.assertIsNotNone(foodata.get("timestamp")) - self.assertEqual(len(foodata.findall("Probe")), - len(probes.probedata['foo.example.com'])) - self.assertEqual(len(foodata.findall("Group")), - len(probes.cgroups['foo.example.com'])) - xml = foodata.find("Probe[@name='xml']") - self.assertIsNotNone(xml) - self.assertIsNotNone(xml.get("value")) - xdata = lxml.etree.XML(xml.get("value")) - self.assertIsNotNone(xdata) - self.assertIsNotNone(xdata.find("test")) - self.assertEqual(xdata.find("test").get("foo"), "foo") - text = foodata.find("Probe[@name='text']") - self.assertIsNotNone(text) - self.assertIsNotNone(text.get("value")) - multiline = foodata.find("Probe[@name='multiline']") - self.assertIsNotNone(multiline) - self.assertIsNotNone(multiline.get("value")) - self.assertGreater(len(multiline.get("value").splitlines()), 1) - - bardata = data.find("Client[@name='bar.example.com']") - self.assertIsNotNone(bardata) - self.assertIsNotNone(bardata.get("timestamp")) - self.assertEqual(len(bardata.findall("Probe")), - len(probes.probedata['bar.example.com'])) - self.assertEqual(len(bardata.findall("Group")), - len(probes.cgroups['bar.example.com'])) - empty = bardata.find("Probe[@name='empty']") - self.assertIsNotNone(empty) - self.assertIsNotNone(empty.get("value")) - self.assertEqual(empty.get("value"), "") - if HAS_JSON: - jdata = bardata.find("Probe[@name='json']") - self.assertIsNotNone(jdata) - self.assertIsNotNone(jdata.get("value")) - self.assertItemsEqual(test_data, - json.loads(jdata.get("value"))) - if HAS_YAML: - ydata = bardata.find("Probe[@name='yaml']") - self.assertIsNotNone(ydata) - self.assertIsNotNone(ydata.get("value")) - self.assertItemsEqual(test_data, - yaml.load(ydata.get("value"))) - - inner() - - @skipUnless(HAS_DJANGO, "Django not found, skipping") - def test__write_data_db(self): - syncdb(TestProbesDB) - probes = self.get_probes_object(use_db=True) - probes.probedata = self.get_test_probedata() - probes.cgroups = self.get_test_cgroups() - - for cname in ["foo.example.com", "bar.example.com"]: - client = Mock() - client.hostname = cname - probes._write_data_db(client) - - pdata = ProbesDataModel.objects.filter(hostname=cname).all() - self.assertEqual(len(pdata), len(probes.probedata[cname])) - - for probe in pdata: - self.assertEqual(probe.hostname, client.hostname) - self.assertIsNotNone(probe.data) - if probe.probe == "xml": - xdata = lxml.etree.XML(probe.data) - self.assertIsNotNone(xdata) - self.assertIsNotNone(xdata.find("test")) - self.assertEqual(xdata.find("test").get("foo"), "foo") - elif probe.probe == "text": - pass - elif probe.probe == "multiline": - self.assertGreater(len(probe.data.splitlines()), 1) - elif probe.probe == "empty": - self.assertEqual(probe.data, "") - elif probe.probe == "yaml": - self.assertItemsEqual(test_data, yaml.load(probe.data)) - elif probe.probe == "json": - self.assertItemsEqual(test_data, json.loads(probe.data)) - else: - assert False, "Strange probe found in _write_data_db data" - - pgroups = ProbesGroupsModel.objects.filter(hostname=cname).all() - self.assertEqual(len(pgroups), len(probes.cgroups[cname])) - - # test that old probe data is removed properly - cname = 'foo.example.com' - del probes.probedata[cname]['text'] - probes.cgroups[cname].pop() - client = Mock() - client.hostname = cname - probes._write_data_db(client) - - pdata = ProbesDataModel.objects.filter(hostname=cname).all() - self.assertEqual(len(pdata), len(probes.probedata[cname])) - pgroups = ProbesGroupsModel.objects.filter(hostname=cname).all() - self.assertEqual(len(pgroups), len(probes.cgroups[cname])) - - @skipUnless(HAS_DJANGO, "Django not found, skipping") - @patch("Bcfg2.Server.Plugins.Probes.Probes._load_data_db", Mock()) - @patch("Bcfg2.Server.Plugins.Probes.Probes._load_data_xml", Mock()) - def test_load_data_xml(self): - probes = self.get_probes_object(use_db=False) - probes.load_data() - probes._load_data_xml.assert_any_call() - self.assertFalse(probes._load_data_db.called) - - @skipUnless(HAS_DJANGO, "Django not found, skipping") - @patch("Bcfg2.Server.Plugins.Probes.Probes._load_data_db", Mock()) - @patch("Bcfg2.Server.Plugins.Probes.Probes._load_data_xml", Mock()) - def test_load_data_db(self): - probes = self.get_probes_object(use_db=True) - probes.load_data() - probes._load_data_db.assert_any_call(client=None) - self.assertFalse(probes._load_data_xml.called) - - @patch("lxml.etree.parse") - def test__load_data_xml(self, mock_parse): - probes = self.get_probes_object(use_db=False) - probes.probedata = self.get_test_probedata() - probes.cgroups = self.get_test_cgroups() - - # to get the value for lxml.etree.parse to parse, we call - # _write_data_xml, mock the lxml.etree._ElementTree.write() - # call, and grab the data that gets "written" to probed.xml - @patch("lxml.etree.Element") - @patch("lxml.etree.SubElement", StoringSubElement()) - def inner(mock_Element): - mock_Element.side_effect = StoringElement() - probes._write_data_xml(None) - top = mock_Element.side_effect.return_value - return top._element - - xdata = inner() - mock_parse.return_value = xdata.getroottree() - probes.probedata = dict() - probes.cgroups = dict() - - probes._load_data_xml() - mock_parse.assert_called_with(os.path.join(datastore, probes.name, - 'probed.xml'), - parser=Bcfg2.Server.XMLParser) - self.assertItemsEqual(probes.probedata, self.get_test_probedata()) - self.assertItemsEqual(probes.cgroups, self.get_test_cgroups()) - - @skipUnless(HAS_DJANGO, "Django not found, skipping") - def test__load_data_db(self): - syncdb(TestProbesDB) - probes = self.get_probes_object(use_db=True) - probes.probedata = self.get_test_probedata() - probes.cgroups = self.get_test_cgroups() - for cname in probes.probedata.keys(): - client = Mock() - client.hostname = cname - probes._write_data_db(client) - - probes.probedata = dict() - probes.cgroups = dict() - probes._load_data_db() - self.assertItemsEqual(probes.probedata, self.get_test_probedata()) - # the db backend does not store groups at all if a client has - # no groups set, so we can't just use assertItemsEqual here, - # because loading saved data may _not_ result in the original - # data if some clients had no groups set. - test_cgroups = self.get_test_cgroups() - for cname, groups in test_cgroups.items(): - if cname in probes.cgroups: - self.assertEqual(groups, probes.cgroups[cname]) - else: - self.assertEqual(groups, []) + if Bcfg2.Options.setup.probes_db: + TestPlugin.test__init(self) - @patch("Bcfg2.Server.Plugins.Probes.ProbeSet.get_probe_data") - def test_GetProbes(self, mock_get_probe_data): - probes = self.get_probes_object() + def test_GetProbes(self): + p = self.get_obj() + p.probes = Mock() metadata = Mock() - probes.GetProbes(metadata) - mock_get_probe_data.assert_called_with(metadata) - - @patch("Bcfg2.Server.Plugins.Probes.Probes.write_data") - @patch("Bcfg2.Server.Plugins.Probes.Probes.ReceiveDataItem") - def test_ReceiveData(self, mock_ReceiveDataItem, mock_write_data): - # we use a simple (read: bogus) datalist here to make this - # easy to test - datalist = ["a", "b", "c"] - - probes = self.get_probes_object() - probes.core.metadata_cache_mode = 'off' + p.GetProbes(metadata) + p.probes.get_probe_data.assert_called_with(metadata) + + def additionalDataEqual(self, actual, expected): + self.assertItemsEqual( + dict([(k, str(d)) for k, d in actual.items()]), + expected) + + def test_probes_xml(self): + """ Set and retrieve probe data with database disabled """ + Bcfg2.Options.setup.probes_db = False + self._perform_tests() + + @skipUnless(HAS_DJANGO, "Django not found") + def test_probes_db(self): + """ Set and retrieve probe data with database enabled """ + Bcfg2.Options.setup.probes_db = True + syncdb(TestProbesDB) + self._perform_tests() + + def test_allowed_cgroups(self): + """ Test option to only allow probes to set certain groups """ + probes = self.get_obj() + + test_text = """a couple lines +of freeform text +""" + test_groups = ["group", "group2", "group-with-dashes"] + test_probe_data = lxml.etree.Element("Probe", name="test") + test_probe_data.text = test_text + for group in test_groups: + test_probe_data.text += "group:%s\n" % group + client = Mock() - client.hostname = "foo.example.com" - probes.ReceiveData(client, datalist) - - cgroups = [] - cprobedata = ClientProbeDataSet() - self.assertItemsEqual(mock_ReceiveDataItem.call_args_list, - [call(client, "a", cgroups, cprobedata), - call(client, "b", cgroups, cprobedata), - call(client, "c", cgroups, cprobedata)]) - mock_write_data.assert_called_with(client) - self.assertFalse(probes.core.metadata_cache.expire.called) - - # change the datalist, ensure that the cache is cleared - probes.cgroups[client.hostname] = datalist - probes.core.metadata_cache_mode = 'aggressive' - probes.ReceiveData(client, ['a', 'b', 'd']) - - mock_write_data.assert_called_with(client) - probes.core.metadata_cache.expire.assert_called_with(client.hostname) - - def test_ReceiveDataItem(self): - probes = self.get_probes_object() - for cname, cdata in self.get_test_probedata().items(): - client = Mock() - client.hostname = cname - cgroups = [] - cprobedata = ClientProbeDataSet() - for pname, pdata in cdata.items(): - dataitem = lxml.etree.Element("Probe", name=pname) - if pname == "text": - # add some groups to the plaintext test to test - # group parsing - data = [pdata] - for group in self.get_test_cgroups()[cname]: - data.append("group:%s" % group) - dataitem.text = "\n".join(data) - else: - dataitem.text = str(pdata) - - probes.ReceiveDataItem(client, dataitem, cgroups, cprobedata) - - probes.cgroups[client.hostname] = cgroups - probes.probedata[client.hostname] = cprobedata - self.assertIn(client.hostname, probes.probedata) - self.assertIn(pname, probes.probedata[cname]) - self.assertEqual(pdata, probes.probedata[cname][pname]) - self.assertIn(client.hostname, probes.cgroups) - self.assertEqual(probes.cgroups[cname], - self.get_test_cgroups()[cname]) - - # test again, with an explicit list of allowed groups - probes.allowed_cgroups = [re.compile(r'^.*s$')] - for cname, cdata in self.get_test_probedata().items(): - client = Mock() - client.hostname = cname - cgroups = [] - cprobedata = ClientProbeDataSet() - for pname, pdata in cdata.items(): - dataitem = lxml.etree.Element("Probe", name=pname) - if pname == "text": - # add some groups to the plaintext test to test - # group parsing - data = [pdata] - for group in self.get_test_cgroups()[cname]: - data.append("group:%s" % group) - dataitem.text = "\n".join(data) - else: - dataitem.text = str(pdata) - - probes.ReceiveDataItem(client, dataitem, cgroups, cprobedata) - - probes.cgroups[client.hostname] = cgroups - probes.probedata[client.hostname] = cprobedata - self.assertIn(client.hostname, probes.probedata) - self.assertIn(pname, probes.probedata[cname]) - self.assertEqual(pdata, probes.probedata[cname][pname]) - self.assertIn(client.hostname, probes.cgroups) - self.assertEqual(probes.cgroups[cname], - [g for g in self.get_test_cgroups()[cname] - if g.endswith("s")]) - - def test_get_additional_groups(self): - TestConnector.test_get_additional_groups(self) - - probes = self.get_probes_object() - test_cgroups = self.get_test_cgroups() - probes.cgroups = self.get_test_cgroups() - for cname in test_cgroups.keys(): - metadata = Mock() - metadata.hostname = cname - self.assertEqual(test_cgroups[cname], - probes.get_additional_groups(metadata)) - # test a non-existent client - metadata = Mock() - metadata.hostname = "nonexistent" - self.assertEqual(probes.get_additional_groups(metadata), - list()) - - def test_get_additional_data(self): - TestConnector.test_get_additional_data(self) - - probes = self.get_probes_object() - test_probedata = self.get_test_probedata() - probes.probedata = self.get_test_probedata() - for cname in test_probedata.keys(): - metadata = Mock() - metadata.hostname = cname - self.assertEqual(test_probedata[cname], - probes.get_additional_data(metadata)) - # test a non-existent client - metadata = Mock() - metadata.hostname = "nonexistent" - self.assertEqual(probes.get_additional_data(metadata), - ClientProbeDataSet()) + groups, data = probes.ReceiveDataItem(client, test_probe_data) + self.assertItemsEqual(groups, test_groups) + self.assertEqual(data, test_text) + + old_allowed_groups = Bcfg2.Options.setup.probes_allowed_groups + Bcfg2.Options.setup.probes_allowed_groups = [re.compile(r'^group.?$')] + groups, data = probes.ReceiveDataItem(client, test_probe_data) + self.assertItemsEqual(groups, ['group', 'group2']) + self.assertEqual(data, test_text) + Bcfg2.Options.setup.probes_allowed_groups = old_allowed_groups + + def _perform_tests(self): + p = self.get_obj() + + # first, sanity checks + foo_md = Mock(hostname="foo.example.com") + bar_md = Mock(hostname="bar.example.com") + self.assertItemsEqual(p.get_additional_groups(foo_md), []) + self.assertItemsEqual(p.get_additional_data(foo_md), dict()) + self.assertItemsEqual(p.get_additional_groups(bar_md), []) + self.assertItemsEqual(p.get_additional_data(bar_md), dict()) + + # next, set some initial probe data + foo_datalist = [] + for key in ['xml', 'text', 'multiline']: + pdata = lxml.etree.Element("Probe", name=key) + pdata.text = self.data[key] + foo_datalist.append(pdata) + foo_addl_data = dict(xml=self.test_xdoc, + text="freeform text", + multiline="""multiple +lines +of +freeform +text""") + bar_datalist = [] + for key in ['empty', 'almost_empty', 'json', 'yaml']: + if key in self.data: + pdata = lxml.etree.Element("Probe", name=key) + pdata.text = self.data[key] + bar_datalist.append(pdata) + bar_addl_data = dict(empty="", almost_empty="") + if HAS_JSON: + bar_addl_data['json'] = self.data['json'] + if HAS_YAML: + bar_addl_data['yaml'] = self.data['yaml'] + + p.ReceiveData(foo_md, foo_datalist) + self.assertItemsEqual(p.get_additional_groups(foo_md), + ["group", "group-with-dashes", + "group:with:colons"]) + self.additionalDataEqual(p.get_additional_data(foo_md), foo_addl_data) + + p.ReceiveData(bar_md, bar_datalist) + self.assertItemsEqual(p.get_additional_groups(foo_md), + ["group", "group-with-dashes", + "group:with:colons"]) + self.additionalDataEqual(p.get_additional_data(foo_md), foo_addl_data) + self.assertItemsEqual(p.get_additional_groups(bar_md), ['other_group']) + self.additionalDataEqual(p.get_additional_data(bar_md), bar_addl_data) + + # instantiate a new Probes object and clear Probes caches to + # imitate a server restart + p = self.get_obj() + Bcfg2.Server.Cache.expire("Probes") + + self.assertItemsEqual(p.get_additional_groups(foo_md), + ["group", "group-with-dashes", + "group:with:colons"]) + self.additionalDataEqual(p.get_additional_data(foo_md), foo_addl_data) + self.assertItemsEqual(p.get_additional_groups(bar_md), ['other_group']) + self.additionalDataEqual(p.get_additional_data(bar_md), bar_addl_data) + + # set new data (and groups) for foo + foo_datalist = [] + pdata = lxml.etree.Element("Probe", name='xml') + pdata.text = self.data['xml'] + foo_datalist.append(pdata) + foo_addl_data = dict(xml=self.test_xdoc) + + p.ReceiveData(foo_md, foo_datalist) + self.assertItemsEqual(p.get_additional_groups(foo_md), ["group"]) + self.additionalDataEqual(p.get_additional_data(foo_md), foo_addl_data) + self.assertItemsEqual(p.get_additional_groups(bar_md), ['other_group']) + self.additionalDataEqual(p.get_additional_data(bar_md), bar_addl_data) + + # instantiate a new Probes object and clear Probes caches to + # imitate a server restart + p = self.get_obj() + Bcfg2.Server.Cache.expire("Probes") + + self.assertItemsEqual(p.get_additional_groups(foo_md), ["group"]) + self.additionalDataEqual(p.get_additional_data(foo_md), foo_addl_data) + self.assertItemsEqual(p.get_additional_groups(bar_md), ['other_group']) + self.additionalDataEqual(p.get_additional_data(bar_md), bar_addl_data) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py index 7bdf344b5..159dc6e66 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py @@ -19,12 +19,6 @@ from TestPlugin import TestStructFile, TestFileBacked, TestConnector, \ TestPlugin, TestDirectoryBacked try: - from Bcfg2.Encryption import EVPError - HAS_CRYPTO = True -except: - HAS_CRYPTO = False - -try: import json JSON = "json" except ImportError: @@ -36,12 +30,12 @@ class TestPropertyFile(Bcfg2TestCase): path = os.path.join(datastore, "test") def get_obj(self, path=None): + set_setup_default("writes_enabled", False) if path is None: path = self.path return self.test_obj(path) def test_write(self): - Bcfg2.Server.Plugins.Properties.SETUP = MagicMock() pf = self.get_obj() pf.validate_data = Mock() pf._write = Mock() @@ -52,20 +46,16 @@ class TestPropertyFile(Bcfg2TestCase): def reset(): pf.validate_data.reset_mock() pf._write.reset_mock() - Bcfg2.Server.Plugins.Properties.SETUP.reset_mock() # test writes disabled - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.return_value = False + Bcfg2.Options.setup.writes_enabled = False self.assertRaises(PluginExecutionError, pf.write) self.assertFalse(pf.validate_data.called) self.assertFalse(pf._write.called) - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.assert_called_with("properties", - "writes_enabled", - default=True) # test successful write reset() - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.return_value = True + Bcfg2.Options.setup.writes_enabled = True self.assertEqual(pf.write(), pf._write.return_value) pf.validate_data.assert_called_with() pf._write.assert_called_with() @@ -99,96 +89,95 @@ class TestPropertyFile(Bcfg2TestCase): mock_copy.assert_called_with(pf) -if can_skip or HAS_JSON: - class TestJSONPropertyFile(TestFileBacked, TestPropertyFile): - test_obj = JSONPropertyFile - - def get_obj(self, *args, **kwargs): - return TestFileBacked.get_obj(self, *args, **kwargs) - - @skipUnless(HAS_JSON, "JSON libraries not found, skipping") - def setUp(self): - pass - - @patch("%s.loads" % JSON) - def test_Index(self, mock_loads): - pf = self.get_obj() - pf.Index() - mock_loads.assert_called_with(pf.data) - self.assertEqual(pf.json, mock_loads.return_value) - - mock_loads.reset_mock() - mock_loads.side_effect = ValueError - self.assertRaises(PluginExecutionError, pf.Index) - mock_loads.assert_called_with(pf.data) - - @patch("%s.dump" % JSON) - @patch("%s.open" % builtins) - def test__write(self, mock_open, mock_dump): - pf = self.get_obj() - self.assertTrue(pf._write()) - mock_open.assert_called_with(pf.name, 'wb') - mock_dump.assert_called_with(pf.json, mock_open.return_value) - - @patch("%s.dumps" % JSON) - def test_validate_data(self, mock_dumps): - pf = self.get_obj() - pf.validate_data() - mock_dumps.assert_called_with(pf.json) - - mock_dumps.reset_mock() - mock_dumps.side_effect = ValueError - self.assertRaises(PluginExecutionError, pf.validate_data) - mock_dumps.assert_called_with(pf.json) - - -if can_skip or HAS_YAML: - class TestYAMLPropertyFile(TestFileBacked, TestPropertyFile): - test_obj = YAMLPropertyFile - - def get_obj(self, *args, **kwargs): - return TestFileBacked.get_obj(self, *args, **kwargs) - - @skipUnless(HAS_YAML, "YAML libraries not found, skipping") - def setUp(self): - pass - - @patch("yaml.load") - def test_Index(self, mock_load): - pf = self.get_obj() - pf.Index() - mock_load.assert_called_with(pf.data) - self.assertEqual(pf.yaml, mock_load.return_value) - - mock_load.reset_mock() - mock_load.side_effect = yaml.YAMLError - self.assertRaises(PluginExecutionError, pf.Index) - mock_load.assert_called_with(pf.data) - - @patch("yaml.dump") - @patch("%s.open" % builtins) - def test__write(self, mock_open, mock_dump): - pf = self.get_obj() - self.assertTrue(pf._write()) - mock_open.assert_called_with(pf.name, 'wb') - mock_dump.assert_called_with(pf.yaml, mock_open.return_value) - - @patch("yaml.dump") - def test_validate_data(self, mock_dump): - pf = self.get_obj() - pf.validate_data() - mock_dump.assert_called_with(pf.yaml) - - mock_dump.reset_mock() - mock_dump.side_effect = yaml.YAMLError - self.assertRaises(PluginExecutionError, pf.validate_data) - mock_dump.assert_called_with(pf.yaml) +class TestJSONPropertyFile(TestFileBacked, TestPropertyFile): + test_obj = JSONPropertyFile + + @skipUnless(HAS_JSON, "JSON libraries not found, skipping") + def setUp(self): + TestFileBacked.setUp(self) + TestPropertyFile.setUp(self) + + @patch("%s.loads" % JSON) + def test_Index(self, mock_loads): + pf = self.get_obj() + pf.Index() + mock_loads.assert_called_with(pf.data) + self.assertEqual(pf.json, mock_loads.return_value) + + mock_loads.reset_mock() + mock_loads.side_effect = ValueError + self.assertRaises(PluginExecutionError, pf.Index) + mock_loads.assert_called_with(pf.data) + + @patch("%s.dump" % JSON) + @patch("%s.open" % builtins) + def test__write(self, mock_open, mock_dump): + pf = self.get_obj() + self.assertTrue(pf._write()) + mock_open.assert_called_with(pf.name, 'wb') + mock_dump.assert_called_with(pf.json, mock_open.return_value) + + @patch("%s.dumps" % JSON) + def test_validate_data(self, mock_dumps): + pf = self.get_obj() + pf.validate_data() + mock_dumps.assert_called_with(pf.json) + + mock_dumps.reset_mock() + mock_dumps.side_effect = ValueError + self.assertRaises(PluginExecutionError, pf.validate_data) + mock_dumps.assert_called_with(pf.json) + + +class TestYAMLPropertyFile(TestFileBacked, TestPropertyFile): + test_obj = YAMLPropertyFile + + @skipUnless(HAS_YAML, "YAML libraries not found, skipping") + def setUp(self): + TestFileBacked.setUp(self) + TestPropertyFile.setUp(self) + + @patch("yaml.load") + def test_Index(self, mock_load): + pf = self.get_obj() + pf.Index() + mock_load.assert_called_with(pf.data) + self.assertEqual(pf.yaml, mock_load.return_value) + + mock_load.reset_mock() + mock_load.side_effect = yaml.YAMLError + self.assertRaises(PluginExecutionError, pf.Index) + mock_load.assert_called_with(pf.data) + + @patch("yaml.dump") + @patch("%s.open" % builtins) + def test__write(self, mock_open, mock_dump): + pf = self.get_obj() + self.assertTrue(pf._write()) + mock_open.assert_called_with(pf.name, 'wb') + mock_dump.assert_called_with(pf.yaml, mock_open.return_value) + + @patch("yaml.dump") + def test_validate_data(self, mock_dump): + pf = self.get_obj() + pf.validate_data() + mock_dump.assert_called_with(pf.yaml) + + mock_dump.reset_mock() + mock_dump.side_effect = yaml.YAMLError + self.assertRaises(PluginExecutionError, pf.validate_data) + mock_dump.assert_called_with(pf.yaml) class TestXMLPropertyFile(TestPropertyFile, TestStructFile): test_obj = XMLPropertyFile path = TestStructFile.path + def setUp(self): + TestPropertyFile.setUp(self) + TestStructFile.setUp(self) + set_setup_default("automatch", False) + def get_obj(self, *args, **kwargs): return TestStructFile.get_obj(self, *args, **kwargs) @@ -243,150 +232,47 @@ class TestXMLPropertyFile(TestPropertyFile, TestStructFile): mock_exists.assert_called_with(schemafile) mock_XMLSchema.assert_called_with(file=schemafile) - def test_Index(self): - TestStructFile.test_Index(self) - - pf = self.get_obj() - pf.xdata = lxml.etree.Element("Properties") - lxml.etree.SubElement(pf.xdata, "Crypted", encrypted="foo") - pf.data = lxml.etree.tostring(pf.xdata) - - @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping") - def test_Index_crypto(self): - pf = self.get_obj() - pf._decrypt = Mock() - pf._decrypt.return_value = 'plaintext' - pf.data = ''' -<Properties decrypt="strict"> - <Crypted encrypted="foo"> - crypted - <Plain foo="bar">plain</Plain> - </Crypted> - <Crypted encrypted="bar">crypted</Crypted> - <Plain bar="baz">plain</Plain> - <Plain> - <Crypted encrypted="foo">crypted</Crypted> - </Plain> -</Properties>''' - - # test successful decryption - pf.Index() - self.assertItemsEqual(pf._decrypt.call_args_list, - [call(el) for el in pf.xdata.xpath("//Crypted")]) - for el in pf.xdata.xpath("//Crypted"): - self.assertEqual(el.text, pf._decrypt.return_value) - - # test failed decryption, strict - pf._decrypt.reset_mock() - pf._decrypt.side_effect = EVPError - self.assertRaises(PluginExecutionError, pf.Index) - - # test failed decryption, lax - pf.data = pf.data.replace("strict", "lax") - pf._decrypt.reset_mock() - pf.Index() - self.assertItemsEqual(pf._decrypt.call_args_list, - [call(el) for el in pf.xdata.xpath("//Crypted")]) - - @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping") - @patchIf(HAS_CRYPTO, "Bcfg2.Encryption.ssl_decrypt") - @patchIf(HAS_CRYPTO, "Bcfg2.Encryption.get_algorithm") - @patchIf(HAS_CRYPTO, "Bcfg2.Encryption.get_passphrases") - def test_decrypt(self, mock_get_passphrases, - mock_get_algorithm, mock_ssl): - pf = self.get_obj() - Bcfg2.Server.Plugins.Properties.SETUP = MagicMock() - - def reset(): - mock_get_algorithm.reset_mock() - mock_get_passphrases.reset_mock() - mock_ssl.reset_mock() - - # test element without text contents - self.assertIsNone(pf._decrypt(lxml.etree.Element("Test"))) - self.assertFalse(mock_get_passphrases.called) - self.assertFalse(mock_ssl.called) - - # test element with a passphrase in the config file - reset() - el = lxml.etree.Element("Test", encrypted="foo") - el.text = "crypted" - mock_get_passphrases.return_value = dict(foo="foopass", - bar="barpass") - mock_get_algorithm.return_value = "bf_cbc" - mock_ssl.return_value = "decrypted with ssl" - self.assertEqual(pf._decrypt(el), mock_ssl.return_value) - mock_get_passphrases.assert_called_with( - Bcfg2.Server.Plugins.Properties.SETUP) - mock_get_algorithm.assert_called_with( - Bcfg2.Server.Plugins.Properties.SETUP) - mock_ssl.assert_called_with(el.text, "foopass", - algorithm="bf_cbc") - - # test failure to decrypt element with a passphrase in the config - reset() - mock_ssl.side_effect = EVPError - self.assertRaises(EVPError, pf._decrypt, el) - mock_get_passphrases.assert_called_with( - Bcfg2.Server.Plugins.Properties.SETUP) - mock_get_algorithm.assert_called_with( - Bcfg2.Server.Plugins.Properties.SETUP) - mock_ssl.assert_called_with(el.text, "foopass", - algorithm="bf_cbc") - - # test element without valid passphrase - reset() - el.set("encrypted", "true") - self.assertRaises(EVPError, pf._decrypt, el) - self.assertFalse(mock_ssl.called) - @patch("copy.copy") def test_get_additional_data(self, mock_copy): - Bcfg2.Server.Plugins.Properties.SETUP = Mock() pf = self.get_obj() + pf.setup = Mock() pf.XMLMatch = Mock() metadata = Mock() def reset(): mock_copy.reset_mock() pf.XMLMatch.reset_mock() - Bcfg2.Server.Plugins.Properties.SETUP.reset_mock() + pf.setup.reset_mock() pf.xdata = lxml.etree.Element("Properties", automatch="true") - for automatch in [True, False]: + for Bcfg2.Options.setup.automatch in [True, False]: reset() - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.return_value = automatch self.assertEqual(pf.get_additional_data(metadata), pf.XMLMatch.return_value) pf.XMLMatch.assert_called_with(metadata) - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.assert_called_with("properties", "automatch", default=False) self.assertFalse(mock_copy.called) pf.xdata = lxml.etree.Element("Properties", automatch="false") - for automatch in [True, False]: + for Bcfg2.Options.setup.automatch in [True, False]: reset() - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.return_value = automatch self.assertEqual(pf.get_additional_data(metadata), mock_copy.return_value) mock_copy.assert_called_with(pf) self.assertFalse(pf.XMLMatch.called) - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.assert_called_with("properties", "automatch", default=False) pf.xdata = lxml.etree.Element("Properties") reset() - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.return_value = False + Bcfg2.Options.setup.automatch = False self.assertEqual(pf.get_additional_data(metadata), mock_copy.return_value) mock_copy.assert_called_with(pf) self.assertFalse(pf.XMLMatch.called) - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.assert_called_with("properties", "automatch", default=False) reset() - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.return_value = True + Bcfg2.Options.setup.automatch = True self.assertEqual(pf.get_additional_data(metadata), pf.XMLMatch.return_value) pf.XMLMatch.assert_called_with(metadata) - Bcfg2.Server.Plugins.Properties.SETUP.cfp.getboolean.assert_called_with("properties", "automatch", default=False) self.assertFalse(mock_copy.called) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestRules.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestRules.py index f018b45dc..45f3671e8 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestRules.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestRules.py @@ -1,9 +1,11 @@ import os import sys +import copy import lxml.etree -import Bcfg2.Server.Plugin +import Bcfg2.Options from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugins.Rules import * +from Bcfg2.Server.Plugin import PluginExecutionError # add all parent testsuite directories to sys.path to allow (most) # relative imports in python 2.4 @@ -15,116 +17,159 @@ while path != "/": break path = os.path.dirname(path) from common import * -from TestPlugin import TestPrioDir +from TestPlugin.Testhelpers import TestPrioDir class TestRules(TestPrioDir): test_obj = Rules - def test_HandlesEntry(self): + abstract = dict( + basic=lxml.etree.Element("Path", name="/etc/basic"), + unhandled=lxml.etree.Element("Path", name="/etc/unhandled"), + priority=lxml.etree.Element("Path", name="/etc/priority"), + content=lxml.etree.Element("Path", name="/etc/text-content"), + duplicate=lxml.etree.Element("SEBoolean", name="duplicate"), + group=lxml.etree.Element("SEPort", name="6789/tcp"), + children=lxml.etree.Element("Path", name="/etc/child-entries"), + regex=lxml.etree.Element("Package", name="regex"), + slash=lxml.etree.Element("Path", name="/etc/trailing/slash"), + no_slash=lxml.etree.Element("Path", name="/etc/no/trailing/slash/")) + + concrete = dict( + basic=lxml.etree.Element("Path", name="/etc/basic", type="directory", + owner="root", group="root", mode="0600"), + priority=lxml.etree.Element("Path", name="/etc/priority", + type="directory", owner="root", + group="root", mode="0600"), + content=lxml.etree.Element("Path", name="/etc/text-content", + type="file", owner="bar", group="bar", + mode="0644"), + duplicate=lxml.etree.Element("SEBoolean", name="duplicate", + value="on"), + group=lxml.etree.Element("SEPort", name="6789/tcp", + selinuxtype="bcfg2_server_t"), + children=lxml.etree.Element("Path", name="/etc/child-entries", + type="directory", owner="root", + group="root", mode="0775"), + regex=lxml.etree.Element("Package", name="regex", type="yum", + version="any"), + slash=lxml.etree.Element("Path", name="/etc/trailing/slash", + type="directory", owner="root", group="root", + mode="0600"), + no_slash=lxml.etree.Element("Path", name="/etc/no/trailing/slash/", + type="directory", owner="root", + group="root", mode="0600")) + + concrete['content'].text = "Text content" + lxml.etree.SubElement(concrete['children'], + "ACL", type="default", scope="user", user="foouser", + perms="rw") + lxml.etree.SubElement(concrete['children'], + "ACL", type="default", scope="group", group="users", + perms="rx") + + in_file = copy.deepcopy(concrete) + in_file['regex'].set("name", ".*") + in_file['slash'].set("name", "/etc/trailing/slash/") + in_file['no_slash'].set("name", "/etc/no/trailing/slash") + + rules1 = lxml.etree.Element("Rules", priority="10") + rules1.append(in_file['basic']) + lxml.etree.SubElement(rules1, "Path", name="/etc/priority", + type="directory", owner="foo", group="foo", + mode="0644") + foogroup = lxml.etree.SubElement(rules1, "Group", name="foogroup") + foogroup.append(in_file['group']) + rules1.append(in_file['content']) + rules1.append(copy.copy(in_file['duplicate'])) + + rules2 = lxml.etree.Element("Rules", priority="20") + rules2.append(in_file['priority']) + rules2.append(in_file['children']) + rules2.append(in_file['no_slash']) + + rules3 = lxml.etree.Element("Rules", priority="10") + rules3.append(in_file['duplicate']) + rules3.append(in_file['regex']) + rules3.append(in_file['slash']) + + rules = {"rules1.xml": rules1, "rules2.xml": rules2, "rules3.xml": rules3} + + def setUp(self): + TestPrioDir.setUp(self) + set_setup_default("lax_decryption", True) + set_setup_default("rules_regex", False) + + def get_child(self, name): + """ Turn one of the XML documents in `rules` into a child + object """ + filename = os.path.join(datastore, self.test_obj.name, name) + rv = self.test_obj.__child__(filename) + rv.data = lxml.etree.tostring(self.rules[name]) + rv.Index() + return rv + + def get_obj(self, core=None): + r = TestPrioDir.get_obj(self, core=core) + r.entries = dict([(n, self.get_child(n)) for n in self.rules.keys()]) + return r + + def _do_test(self, name, groups=None): + if groups is None: + groups = [] r = self.get_obj() - r.Entries = dict(Path={"/etc/foo.conf": Mock(), - "/etc/bar.conf": Mock()}) - r._matches = Mock() - metadata = Mock() - - entry = lxml.etree.Element("Path", name="/etc/foo.conf") - self.assertEqual(r.HandlesEntry(entry, metadata), - r._matches.return_value) - r._matches.assert_called_with(entry, metadata, - r.Entries['Path'].keys()) - - r._matches.reset_mock() - entry = lxml.etree.Element("Path", name="/etc/baz.conf") - self.assertEqual(r.HandlesEntry(entry, metadata), - r._matches.return_value) - r._matches.assert_called_with(entry, metadata, - r.Entries['Path'].keys()) - - r._matches.reset_mock() - entry = lxml.etree.Element("Package", name="foo") - self.assertFalse(r.HandlesEntry(entry, metadata)) - - def test_BindEntry(self, method="BindEntry"): + metadata = Mock(groups=groups) + entry = copy.deepcopy(self.abstract[name]) + self.assertTrue(r.HandlesEntry(entry, metadata)) + r.HandleEntry(entry, metadata) + self.assertXMLEqual(entry, self.concrete[name]) + + def _do_test_failure(self, name, groups=None, handles=None): + if groups is None: + groups = [] r = self.get_obj() - r.get_attrs = Mock() - r.get_attrs.return_value = dict(overwrite="new", add="add", - text="text") - entry = lxml.etree.Element("Test", overwrite="old", keep="keep") - metadata = Mock() - - getattr(r, method)(entry, metadata) - r.get_attrs.assert_called_with(entry, metadata) - self.assertItemsEqual(entry.attrib, - dict(overwrite="old", add="add", keep="keep", - text="text")) - - def test_HandleEntry(self): - self.test_BindEntry(method="HandleEntry") - - @patch("Bcfg2.Server.Plugin.PrioDir._matches") - def test__matches(self, mock_matches): - """ test _matches() behavior regardless of state of _regex_enabled """ - r = self.get_obj() - metadata = Mock() - - entry = lxml.etree.Element("Path", name="/etc/foo.conf") - rules = [] - mock_matches.return_value = True - self.assertTrue(r._matches(entry, metadata, rules)) - mock_matches.assert_called_with(r, entry, metadata, rules) - - # test special Path cases -- adding and removing trailing slash - mock_matches.reset_mock() - mock_matches.return_value = False - rules = ["/etc/foo/", "/etc/bar"] - entry = lxml.etree.Element("Path", name="/etc/foo") - self.assertTrue(r._matches(entry, metadata, rules)) - mock_matches.assert_called_with(r, entry, metadata, rules) - - mock_matches.reset_mock() - entry = lxml.etree.Element("Path", name="/etc/bar/") - self.assertTrue(r._matches(entry, metadata, rules)) - mock_matches.assert_called_with(r, entry, metadata, rules) - - @patch("Bcfg2.Server.Plugin.PrioDir._matches") - def test__matches_regex_disabled(self, mock_matches): - """ test failure to match with regex disabled """ - r = self.get_obj() - self.set_regex_enabled(r, False) - metadata = Mock() - mock_matches.return_value = False - - entry = lxml.etree.Element("Path", name="/etc/foo.conf") - rules = [] - self.assertFalse(r._matches(entry, metadata, rules)) - mock_matches.assert_called_with(r, entry, metadata, rules) - - @patch("Bcfg2.Server.Plugin.PrioDir._matches") - def test__matches_regex_enabled(self, mock_matches): - """ test match with regex enabled """ - r = self.get_obj() - self.set_regex_enabled(r, True) - metadata = Mock() - mock_matches.return_value = False - - entry = lxml.etree.Element("Path", name="/etc/foo.conf") - rules = ["/etc/.*\.conf", "/etc/bar"] - self.assertTrue(r._matches(entry, metadata, rules)) - mock_matches.assert_called_with(r, entry, metadata, rules) - self.assertIn("/etc/.*\.conf", r._regex_cache.keys()) - - def set_regex_enabled(self, rules_obj, state): - """ set the state of regex_enabled for this implementation of - Rules """ - if not isinstance(rules_obj.core.setup, MagicMock): - rules_obj.core.setup = MagicMock() - rules_obj.core.setup.cfp.getboolean.return_value = state - - def test__regex_enabled(self): - r = self.get_obj() - r.core.setup = MagicMock() - self.assertEqual(r._regex_enabled, - r.core.setup.cfp.getboolean.return_value) - r.core.setup.cfp.getboolean.assert_called_with("rules", "regex", - default=False) + metadata = Mock(groups=groups) + entry = self.abstract[name] + if handles is not None: + self.assertEqual(handles, r.HandlesEntry(entry, metadata)) + self.assertRaises(PluginExecutionError, + r.HandleEntry, entry, metadata) + + def test_basic(self): + """ Test basic Rules usage """ + self._do_test('basic') + self._do_test_failure('unhandled', handles=False) + + def test_priority(self): + """ Test that Rules respects priority """ + self._do_test('priority') + + def test_duplicate(self): + """ Test that Rules raises exceptions for duplicate entries """ + self._do_test_failure('duplicate') + + def test_content(self): + """ Test that Rules copies text content from concrete entries """ + self._do_test('content') + + def test_group(self): + """ Test that Rules respects <Group/> tags """ + self._do_test('group', groups=['foogroup']) + self._do_test_failure('group', groups=['bargroup'], handles=False) + + def test_children(self): + """ Test that Rules copies child elements from concrete entries """ + self._do_test('children') + + def test_regex(self): + """ Test that Rules handles regular expressions properly """ + Bcfg2.Options.setup.rules_regex = False + self._do_test_failure('regex', handles=False) + Bcfg2.Options.setup.rules_regex = True + self._do_test('regex') + Bcfg2.Options.setup.rules_regex = False + + def test_slash(self): + """ Test that Rules handles trailing slashes on Path entries """ + self._do_test('slash') + self._do_test('no_slash') diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py index bf9a3ced3..128d6cae5 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py @@ -25,7 +25,7 @@ class TestHelperModule(Bcfg2TestCase): def get_obj(self, path=None): if path is None: path = self.path - return self.test_obj(path, fam=Mock()) + return self.test_obj(path) def test__init(self): hm = self.get_obj() diff --git a/testsuite/Testsrc/Testlib/TestStatistics.py b/testsuite/Testsrc/Testlib/TestServer/TestStatistics.py index 496cbac28..bf918ef76 100644 --- a/testsuite/Testsrc/Testlib/TestStatistics.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestStatistics.py @@ -13,7 +13,7 @@ while path != "/": path = os.path.dirname(path) from common import * -from Bcfg2.Statistics import * +from Bcfg2.Server.Statistics import * class TestStatistic(Bcfg2TestCase): diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/all-basic.xml b/testsuite/Testsrc/Testsbin/bcfg2-crypt/all-basic.xml new file mode 100644 index 000000000..ce68300b6 --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/all-basic.xml @@ -0,0 +1,12 @@ +<Properties> + <Foo name="1" encrypted="basic">U2FsdGVkX19C6Cy0nM0mlcGGBjqBMAC+GqyPfLpqgT0=</Foo> + <Bar name="1" encrypted="basic">U2FsdGVkX18KUHJTHdrgz3gWtNA5U3g3gq0i/AsdCVE=</Bar> + <Foo name="2" encrypted="basic">U2FsdGVkX1+9nUFxmbl8UJh1t5fWo4cQQa5nQm8hVtw=</Foo> + <Group name="test"> + <Bar name="2" encrypted="basic">U2FsdGVkX18tScJs1si9y45NxPkjYj66Ee+TsYDZAd0=</Bar> + <Foo name="3" encrypted="basic">U2FsdGVkX19q3USU1cnvgfV8roHeSNSQ2bCMD1CCR3jE0e53aT71ATtqHmfsJfDnTgQ28xbKGZhAwoML8ixXkdkyqnsSF69bnIwebaI4qqYXFA2FWF1Cop3bYEV67m6dSM9BkSluKIcs7VdPRANE71OQnd9P2cQbMig50IkBtuE3El8bnc+n4E0k31NT7ZZJ9s/9FJMHg/AfIjvB4KgMqylHcfT43gGICeq4JYPKIsxYjKq0bzFISPBgztD1++YTdCKbJDtjNJJOlqanB3LsBR3PQt6rliWqqPVT3aLP8BU/gIcGE3oyyce04ULxNGTPqFlWgw2r7RopygqgZbzTgU21thzef7bXRi/NATQpXkM70BdCLwRvRNaC3JrMY/z4k0//QliTiNYPNejpGvwezHf99PTN7VPWMhQyONSpLO905KmEJYRt1BXx8p+72b1Q/1S/QFfzU8JU3MO4yiLFf6kyB6nS+pCA40g/UfwKLQI2Fr6Oi5acOK7SRTXqSmxhI+96TQms6bWmm889BO8QOfiuAI1CvHWEBljPACXydcM9wACjhBvpra41UzVgkzadaUO8yBV0Z0bLVHuyIdI1I78vgrkd99tJvC/AmYazEwM=</Foo> + </Group> + <Group name="test2"> + <Bar name="3" encrypted="basic">U2FsdGVkX19aE/IqfkkhgkbhA0i9cb1PYp7tdTmfidg=<Foo name="4" encrypted="basic">U2FsdGVkX1+J4nhfxE5GjwDF1PzOjw2q5e5vrcFZyCQ=</Foo></Bar> + </Group> +</Properties> diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/basic-des-cbc.crypt b/testsuite/Testsrc/Testsbin/bcfg2-crypt/basic-des-cbc.crypt new file mode 100644 index 000000000..31aa80e26 --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/basic-des-cbc.crypt @@ -0,0 +1 @@ +U2FsdGVkX1/2LyHZkqC9Ri2CeDw45osMNNUwOMraHGD/Z3jsHekaU5w6YlTZ3DUPGJoNKsPiUp0jdheuvFGyv2jmXn4ocIPixcliR16n1lWTv5yE+lleELRRqzRCR+cYjmXJzAHg2L5YJ+/XwZR0tlN5cod81dsHiaVw86K7MU8SlkTuN+QSU24CFucuhi9cXTLt0E7ipIded4J5qlDS2ZZIR/m7LG++VS8jc47S65VHR5VHg6cTQ130b78eG8344LM7i4hVcH1Pm0YOm1Kq3Q+IrmyiPmJKFb1lABoEJHCKAlQ85H5xFdbnXP9FbcZVq9zLmIoB3RsGWLLZobY93FhwXN86VZ2APVn7rzLPZCk8KPqkxGoErUZl4of6eQMvi9Owvf7s6fhkKlG7c03BJ8FtbaKOGob9Y2vM2Vjs322QXnJnGXWZ2ZJLu6+OAdVNUueEhqe12SKIc9Olzv6Gn60tiB5XByqAuY8y6oPm3F+gCdbReh1Lu7fVVGP6brkgZWzoBv/T503nOU/Jw3yRSd9nHwBkiMA3HtjzgWZrXf+TRW5apSAoDAeX70nBR4GtqQO3GozFZgqvr4yzLHo1m3/+mB/svNjeMTbRKp5cKs+47C8rno3+asbVtykrPBnqgC16WRa+fB/juxjpRaEK7dbzhk701lNbJrXKT0S4S5Pdir+SDl1udgpTRiy1MaL9JnwQ5w5du6XfLUH43Q2UiMpNABQAyxHZDsdEcxzj16e7wKXVVguS2mqN8TrWhC1K7ZZ+q8BHZaV1c+R45CwT1nIuGoYzKe0Wsc3WtY8yYTbKJPeJQyL7qSyC/M8cV8RkkDGTgJ/FuRlsm5R89enBFRgb5tmT3S6pG6WxTsxcOyXNwc2VP47dJuetguiEYI8FX6pWbfwSfd1HG7uKkD3QMUYU1YtzCziNLNpag03jd/Ios2q/gR6agKGAzVTSIGeaNwpervlXiHXAdGThWXbqTy+1Zbp4OnEbLGEtksedpTD6Ij+JNFSgyIxDbBdrxvx0EZHf8GeIAiOifRZw3XTxla16oKylv2FnUJMJlwONOyswmwduWaY+0+LYHygzJYyF9YoJVYCASQpz5dll3Z1U/5vKqzrY8/88/SLINPP8nQ6mV1JbnXuRS8CD/LbzI3T0CTDKrsxQkkFsArtdUpYRHBqsPYyuikmG1GGeEtRznkNqcE2EXKoNdIU4Rm6GIK6thes3bYqnp5MK/HLQ2vKkOraSR3IyQJXW7EUVCqZk3M5wFGem+8aEw3sjYljrxNzSAB2EPitnxpBp21Mfu+6SmlGWOXANBNSBG630JzNgfqh31SKrWCw6jyFYrI43UXJEsAdEerfKK/UKBs2rb7Y/DGNF1lM9LDxH2nQ5ByELn/PxxpRdKYoXgsHmPwBHMuXhjPXxY1uw25GZgVVk58GR/l/Q78sRiqiwr2D0rriUTQfLW+bqpxEbdqxJjHa/rGjZQffHC08/xqY9P2FHqcE9VIGH3OdpoM6FKQ/NfBF8FMnkp756JC+nUVgFPTQ5Wn331DlUGyf4MVH6mKskUjOKGCnbt7GnG2K64eFcs0q8k0UK4f8lACtJEA5zSX2qNhPdkpG/yHyzvvLuX2p7j53QyIvX1Nz9k7ZoqD1rjhjDSsfV9C36KORyzqm/MKq8v8YVhEfnGI2UtWM= diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/basic.crypt b/testsuite/Testsrc/Testsbin/bcfg2-crypt/basic.crypt new file mode 100644 index 000000000..386544ca8 --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/basic.crypt @@ -0,0 +1 @@ +U2FsdGVkX1+fzWyB9BSYjGZTIEw6WGCFJvgWeS5BlGSVcLK6cDuoajcTj47falmJhrCg6uew41tps12UDaGe8Uaw8gzZnqGBCRYYSCazD3mqGsUQDbHdjRdsN5C1uTBlrUt52GuqP0EHEmJrqcmNYCkFw2KM8/RCflpuHkEj3L44PTsuq/TsxUmNLkn01z+U9Y1qK13bl60Gn0T3EYkI1hcO2JzyHW0HHwwe2E9xRB1u56Xj9xL8EozshlafTN2J9iIysG5j7P2dlmGHBQtVgrvbgZ8SycmiIOSenRgpln/Ax9WAwcXXbcvCcSUqxA5CrjfQAkfhMuHhYIoI/f5KJcLKnf1lnOXXlFaQVVLlvMxaw7v67jFOOIJekrphgnIF/VviR0ShP8KFvtTrG56lxHHP2ngZHPg6CxqUtyhCEMp2rES1s0PcAd4xvzUlrgZLvHpphZ0ZVtknDF5hmnFuO/Y1Ebo1PvN2vty1GH/XPcLyzk+Iks92hvYmeOy9fJ+9x+myaHCW/xbwesYrkcUnFvvovres20vM981Dv4GRgO1Yej98ac60wNEu8QigqCJCbVnst4g6+dEhd9xzHp9zsf8cCGcJPWp6BUOdk7Z96uA5UwOsbOgyShlLsp2eWnKAb/5HKx8C8JXdCPD7lFNa4QvrfrH6a1HUugZ7oxU8oQ1mPrg8t6Jf7HyMbBpKdx3BGwnb4yxaOPhpW66OXyFuU0l7yHx5tAzG4ee1HqA8wp5bpvwszCQcLuB5a8s5x+tDs/4pb8bb6f5kV4EnYXza9v8tVCwPegdz8IuluPGjbcO9XM8jjTJ6UDZPIOxNKGoxYUl0RKbAKJ1QC51/d2uNFGcd6j00zs4P5P9G5Pqyy1IMUHqCBpbIO1LI5SgHf9E51NYUjTZJMgxgDfXgo3isDVI39Bob5aNHdjJKWBalU4u5BOWosQ27Du7BmXfAF4CeB4nnilTC/MqvUp7i9RRBfiw9jiPxE1tBtjkqIRDf7k1nGtDWhDcP64ph/Iz5IJ/krafzNew36KAHwYISlDN8KYH5gEewixst7oqiRteCT8D55Es+7645MWHoLa6x/LHOT3sYcXQN+fd8SkjKbR8HBT5upAI+8/WWnU/umodJoyTW62xAhjFQVszj5S6451uTEEf6PnBEpbNaYgQhrUv8bX+WGxNL+Q4XmOUdM8holHyM2Lz7saMG5HlZMlYR6DClQqh5QLG1gmL17+ozARKfQWIXBtsWh5/aTlXVwTDAu8p50abgseNswRWm1QkqtDYxVgsj3qfdtOQo/K/gkHx1dJbkZVCVBRR2DNyKT1DZjU+6q8UJZmZ+JUZ1qcP16y+bc4WOXgBdxvDVWwsR1oe9GokQAHovxTms8VtW5+am37BXpJ+nRo6dVo71yaHXdjf6wQgxzFe9eo1BXypxtx+y5jEuN788d0jwLrZ8bF2KrnbzOinch2WkU+7eFt4pHCClX38hQ/d28q8UOyFEbZp1N9Guwtun3NzX4V0DbcZxIxJDGo45uPLDcT6r9dhAUwrfoM+atTEOM3K4BOBEyxehs7NoZrObWdy0FcrKokCT3iHOqImhVdJDnDYDtfqLFi/iycZ8mUnt6eF7oNLLh/TdB+HgC5bf684YPGYCGznAfjOVE8eIKsG5l6DqGDBxx4fh+Hb563aX/UA8tKrnMhcD/A== diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/basic2.crypt b/testsuite/Testsrc/Testsbin/bcfg2-crypt/basic2.crypt new file mode 100644 index 000000000..b26bd91a3 --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/basic2.crypt @@ -0,0 +1 @@ +U2FsdGVkX189QQFkAJ2EZErSsJ7h16U71KMeKEHIC803ktBOL0sBOEdWPKN/0G7Jt4mchdGSeBBCFcxGIJ7yGvhThWxgAsCNCZCPbCCGW7NQ8b01+WRzeggfHYuFw49Mwvr9Dqh97z6HelvvoNr90DW/IuxGrJgKzeP2obbdUn0MRuBNx//JjU9lTDRSCU6i5HR/IJvMdYT9iR7icJa0Wjtl6UXjoZXf2azL1jRLrtG2QtMmUI0PQcg9IDlH1Eiosut2LOcmCCThzR96ubQqjRCAggWJqR1x7qe5+Llkq47jPauWuD1RoH7OLTm1EcVRd68ZtfYPHK8tiu3xZqcmnHC5bHedM9hdwiyDqm/qhdnz5cWA4rlWp4VAFTmJzyt4VsgTTXDbCbRHtU0Ml7E36lg4fMekfql1XT+tiNxmoedAWGOXXbvf8rdyH6EZ2uFBhPtX+2kV66XsNBALAizqwVqpw/wdKVkWGRMsi1p8nE4wj6/WcHBd7YYJBig4UNyqdPTsn2dMhKi3GbzgNvkmNo+V6G3IZ0cqy4tBZkBW3izVfN8e5BEUZ6BR4V8uC/NvTSd7fcJSJZROw/RR2zdTo7qDqgzz27aZrPghDfmLYB26VGn56geSVwEFraDeaUZxa17UiBLtGYPb8rZRQkn0ACsfHu8r2PVt6p/W6oVnZ/05icvP2NjGtWWbzXRpHZEw/5R+cRsfTgi2HpWpJwpqt6Y+dxXb5mpNkwL3tM/p9i1+LKm+jxknkmtCW8rxeHseuvX3WIcyAP7JEBzv76xliKBQP0lpvM53fNcgsJvzgphnwUGAlht+2Y9ZBlKSdGyrft+K4hxYWRcCWMg8+s/VA1VkHl3+JOzwmgfGFx3eVcZnul1Xq58BGWRmVXo0PfSo0VCZuhVuegF2Odqfrx2w57CwKESxDkKGt7h8m7CexJbVHzawJlC5ZbP6iLHCf2rHXIcD/K4Qjl/rbDLvMrHpazSOpaIckYdXFeA3UglEFQ4m4o8OEYJ1rH8z3jQt4A+3L4+8+8tzeTIfnVgA2ouqs00XcoFoPSSvbzZstawJubBR6/pOwYjJAAJTVMPw08X/8qM1NJHSvxgkDlWrm8uUfgGbG6ftkCKlbEuJmWLH2I74zHa0/2j8VtcoHjFkbQc7Slg/pM0IC55ZJhgaZ74qsGJaGz5FpdYx8Fz+riqBTGZA7LpVzsYGl/f4Cmtvf6v0f2pCypb1bMWp+IEn2w7StGVd6oSuc9+wgBi9RWAmlB27MkBvKnx+IuUPpKtUSUG+YM3mFUl9f5A0spMdTiDPdh8Wegqx5S2PLKt61YvoMS8vSQTGxMf3A3WhykYGrFxq8l1zKcZfD7x4oPj398rfBGQPpemuiTuDJeGtsKta8ERgLw1bgrtEu6PWvEf4bSCLYa1x7RkrOihDPyn+jP0v/1Cb7+eQR33tVwNYdN7ZNRzvbn7JHKuci8DrZU2961aA+7t1UMjSaJ06RG4pm120pQckRi3vALxh583KlqpG5a8+DAHq46h16W7PC9IbDdb5YRgAo0QwOnIdFLj4EKt5GefOspnfOWM5gROU33l4J2n+5IsH8d0eOTGXlIVmyye3fXMTTI248DMchBR/xUKisujZZsvjFutxYYX/RGXoIHllmycpAsqnMBPNj6edjPruAqnyDzHpS88Y3eBSv446GA== diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/bogus-forced.xml b/testsuite/Testsrc/Testsbin/bcfg2-crypt/bogus-forced.xml new file mode 100644 index 000000000..d5d0eb8d9 --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/bogus-forced.xml @@ -0,0 +1,5 @@ +<Bogus> + <!-- This is not a real properties file; we use it to test forcing + bcfg2-crypt to treat it like one --> + <Test encrypted="basic">U2FsdGVkX1+uFQUijBDQpGBdTroNS6nl8lPUSeHcWJw=</Test> +</Bogus> diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/bogus.xml b/testsuite/Testsrc/Testsbin/bcfg2-crypt/bogus.xml new file mode 100644 index 000000000..8c83afa69 --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/bogus.xml @@ -0,0 +1,5 @@ +<Bogus> + <!-- This is not a real properties file; we use it to test forcing + bcfg2-crypt to treat it like one --> + <Test encrypted="basic">some text</Test> +</Bogus> diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/complex.crypt b/testsuite/Testsrc/Testsbin/bcfg2-crypt/complex.crypt new file mode 100644 index 000000000..cd4ada4bf --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/complex.crypt @@ -0,0 +1 @@ +U2FsdGVkX1+pB62f8PRmn7i+qzc0nVvezv4sL6KXA4SaTWGvai48caajA480b//AfSkSaLK9C5pDFk0hf0HvoSK/qFdEL/3sChswE2EXJqcZCQxSpexQPhd4t/ES3m40PPO+WmK3AemjvNSv9oQdIXPlgwboVBoBfJ61onPqdQSrDtheDRsaNzOA9kYe5dttl7PwKz9UTj8Ds7rRXvrFdXvmX/KvwWHxQvHmz7RzFfMzBQ7AS+yKo4JPVMSA6peBQRsmu8n/c+G4q0MXntmPRfSELyxX2BG9kIUKJEmCgJHVOnN6Yd/wseKQWbWFut/vB+tq+83yTQKJmEtDx1EAupXv3oBNDpDR2wjLsijvAQCaBkvalHyLEC8/KlC3P6VMUObiaHOMIxcMX9wH5C2jGYCTZ+tlAnzM9RfKkXPEq0sXbTpRy0laRWfV6c39PDFgTE3qBy79Reh4V+YfqG0X2jFR3bbcG1ZFc/QL20X+6LvfP/cxf/zepZ1sa9yJmAw386xtt7tDohdZBmvwOgueqbHhlRnf2hXggSUBHLxspAKsy8kyHzSFBy71qzmXo4j3C96MFf/SCCDJsX1P2yEdny7hKxHKqvQBFR2JlN+m0DFt5rnhiWAkLbpJsBdy66hZ9dmWa3IfKhO9EDkAFf5ts7ZXPgimpdj/A7IjtD19FmBLZ/N29spA+InK661iHwc3HVJ14AorA9gOyeCW6p3GBS/BzENALEDrKk3O9w0Dk4pRcrlRgvEK+HZF6CDVV5jh188Lt3eWReCfBQ9Cwoj/sDY16WNTsqJnA9f+FIUW7WxSpUA5g3bk2IR3pfnzXLfxFpUUGy7v7y7EvOMOWH27550NKtXOszcS72rRzwf4jze7mN51Wt1n9iKKpmoUZNwJ6E6vaU2o/NyuWrWHMWPtTusdQsVMkm2J3TddvMFp6u8mTMIj5ypMuz8+9f2QgLjWrCTY0llVHQfQvELgSSNoNBMfpFKqZt0qk4Zoo0a/CRFj+wEv0zOwF8A0HAtIXrnF4zeiLjDlN4lMrR67hKOuHNLpR6hqkCUSo2blf3d3AiwN6YfLHNuVywuUHINVeoyzls1q6WXuLQuO3HOdpMOWrp4tA6PoDAia36qT8+WhK/hrzinEbiu0Wvd9uv7Ie/veAGBcqW+4SOu0kJqQLgnQfGXvUoxs5TvgbTsrs8TW9CtihR0u0vuuec4El6VEA0xVDKbyI1H2ac3X44qEXwVwWqC2JJdy9O2uNgEQhGah+qLiumvKQElZngMw/ovE76rysF8eO7z1Hkt45raOcodUcAnvTgvGYrVwPKa3a4ShBpT/94IjbsMQ/S9dsnACCps7/TGtqgS96XsookFX8m9x2oycJnpiF59+UlcyyxY6e+vAkl11eIwH/niEpl3JzGKGUi5rs3y7MiLR12ZAoI5R9xBpSY8nm9OGNN5lO242cDnstdjhd7Qz2bR8/KDFQtIVLgxhKrRaUnQr0CFBQ/bW0RS0E1SdFssixvvPohlAXoBQssVOUPHVXbk9On9k8ebQ9n9lhd6L2dK0HydVUFLr0vtVCTv049aLcI3yQ8a6TrOxtVy793hzwY0UZC8X5pjYqT6V/ddyRWzJQRvXnK5KzL4KHWSOXTvwam8TYMp3TdBEi8G+Raydi2kVSeGYEjdjkGSfGgw5kRoeDqkworeZSA== diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext b/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext new file mode 100644 index 000000000..71fa9654e --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext @@ -0,0 +1,9 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla in elit arcu. Morbi interdum fermentum magna non molestie. Curabitur imperdiet, mi eget ultrices porttitor, dolor arcu dictum purus, eu tristique felis felis eget leo. Suspendisse dignissim laoreet velit, id bibendum leo. Etiam faucibus lorem nunc, eget laoreet tortor feugiat at. Fusce at ornare tellus. Donec dui neque, fermentum quis ante ut, sodales commodo magna. Proin nec malesuada risus, ac consectetur mi. Praesent pharetra eleifend lacinia. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Donec odio metus, dictum vel commodo quis, tincidunt in ligula. Aenean et orci non arcu lobortis ultricies. Ut ipsum nisl, luctus sed porta nec, vestibulum pharetra tellus. + + Praesent consectetur condimentum nisl ut cursus. Etiam aliquam nisi + dolor. Mauris aliquet condimentum neque, sodales laoreet lectus + venenatis ac. Morbi mattis justo odio, ac fringilla leo egestas + ut. Integer nec sapien pulvinar, ultrices nulla id, posuere + magna. Quisque in tincidunt sem, sed vehicula orci. Nulla blandit, + nisi vel cursus semper, nibh metus consequat purus, ac ullamcorper + dolor lorem vitae ligula. Maecenas non consectetur nibh. diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext-all.xml b/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext-all.xml new file mode 100644 index 000000000..2dccaa51c --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext-all.xml @@ -0,0 +1,12 @@ +<Properties> + <Foo name="1" encrypted="basic">U2FsdGVkX19CZCt2ydtozka/HuG9Iay3Dpxs/pR7byM=</Foo> + <Bar name="1" encrypted="complex">U2FsdGVkX18RmlLRK6CSIww69iuUTAb1xOkA/2dZw84=</Bar> + <Foo name="2" encrypted="complex">U2FsdGVkX1+h5sBn5Ms1FXe88o69Wc0tE99Nuck++tQ=</Foo> + <Group name="test"> + <Bar name="2" encrypted="basic">U2FsdGVkX1/NcWDYbvU1fUWry44xvFxYQXodBoTs/Ek=</Bar> + <Foo name="3" encrypted="basic">U2FsdGVkX1+02mmcZw+h/QkC+Qr48bjy198xcivfopvaK64xzBe25fEBADvIG7Qab+BxZdZAPWFgX3toBVQFVjQ6M6zf1lrNeciK39LSDj4v3mTIg1/gvew8TUGeQtkrU/xo8ShEAiExma6ILf7Qq6PTc5IdBfuB85bn5YtU1tN4YTiUbK3/DIkTRJI+8YI4GbhFBKeaqMkau5498YdwhxpE0LB0sxTK3Bjw23nwOfcLLXH0uYux1JCxDgYJ9Zalx0qPUthrNnEq2mR9R11lZGmRQD8ToN0/7eS8NkZs3j5TgefbzNdpK7yThbXHFPNwuD6I1AwhQ5oJ//iOkVGpAMXvdPkEZCgKthXnze/X99J0MphTq6oD7XGrY+Sj5EwVzv8X9Mux96QtFylCIGhNllkCAqb3Mzmsr7ZIEmauAr+eTkuRASjJr7XsQKSL5hoLFtF/vKnzTx6YjVETrbXkczZUhA2n7C3HF1OYAozPZmd2WTF7/15jcWCKZB517dfKr5GC1q10NlbiujEUfb/8JnVRg9JfK5r6oXcdfxbODbLchzU+/h2sRRjSVdN6wcXrX+bVMJG9P6cLiPR3oruBUHf/dbZXg06Mp1bqazbOpJY=</Foo> + </Group> + <Group name="test2"> + <Bar name="3" encrypted="basic">U2FsdGVkX1/keWAAgSOnVvhoEDWzmRQWaf3mxOy749o=<Foo name="4" encrypted="basic">U2FsdGVkX1+O10Id9f9FUcavHi8JaQWVNlWm/jwQ4f4=</Foo></Bar> + </Group> +</Properties> diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext-xpath.xml b/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext-xpath.xml new file mode 100644 index 000000000..4bcec1474 --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext-xpath.xml @@ -0,0 +1,12 @@ +<Properties> + <Foo name="1" encrypted="basic">U2FsdGVkX18bAwhcMtr8J02ztT8kBjdCjae9lYnbsRY=</Foo> + <Bar name="1" encrypted="complex">Some text</Bar> + <Foo name="2" encrypted="complex">U2FsdGVkX19+Yq+VwbAfNGUHtnB7hy74V7Fvz0GHsqA=</Foo> + <Group name="test"> + <Bar name="2" encrypted="basic">Some text</Bar> + <Foo name="3" encrypted="basic">U2FsdGVkX1+A3f6lIoFvCNaC6/ctbOLqT0z/YCJ+FkeyTgAnmU1/wk7FlPxOaPPkI2iRfEbNK2sNtUS0rQ8TYT3gIRO3qyrUNIcHaYfGerAZN3Xg9F3CsaL1NQjHxSKyJLdTmdB/1m0AQ3jw13n1eNrokGHF6HU6bD6TIJVFmds126ucOg56Xh+3ffOUukh2EwlBxnvGC/CDQluFixL1MY74xFd5mY5iDcJG9o5qUtjKmiOrtEAXFuM49JLciDHtMjQ2wbX/9lGek7U2Y05I2vU8BGtD3jh/Pt+17Vql80UrNHqVnWx247sxgYmkJIaworCTNowU2KsWEQj46E1bzAXEUVdGF65ltIXIK75KozHf8msKuVFwQDYCJ+lXRZgIygqcZ5glAyjW1WxyigxSFpRfVcZfiHp7d52JfBCU66367j7DvEnAJAuvL7jufJSavd6RxaEGGB3KGAMpz8NQxPy6i2s5RkY5V8eiqUOHsnZN6zHPgkZ90a+dokllLbH+HSYGU26sevJL4TupDCkz2/sRasmBB8fBAF5PGOI+UC7vXncbvpsMLsILFoUYtyWrDZ9cygOElEmWpVJSeECAA09iOhyaXN5rN/tyqkt3+ao=</Foo> + </Group> + <Group name="test2"> + <Bar name="3" encrypted="basic">Some text<Foo name="4" encrypted="basic">U2FsdGVkX1+cSl37JVEVIEV+bqVBlMGQnZdZWsjHPME=</Foo></Bar> + </Group> +</Properties> diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext.xml b/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext.xml new file mode 100644 index 000000000..45d9941c8 --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext.xml @@ -0,0 +1,18 @@ +<Properties> + <Foo name="1" encrypted="basic">Some text</Foo> + <Bar name="1" encrypted="complex">Some text</Bar> + <Foo name="2" encrypted="complex">Some text</Foo> + <Group name="test"> + <Bar name="2" encrypted="basic">Some text</Bar> + <Foo name="3" encrypted="basic">Praesent consectetur condimentum nisl ut cursus. Etiam aliquam nisi +dolor. Mauris aliquet condimentum neque, sodales laoreet lectus +venenatis ac. Morbi mattis justo odio, ac fringilla leo egestas +ut. Integer nec sapien pulvinar, ultrices nulla id, posuere +magna. Quisque in tincidunt sem, sed vehicula orci. Nulla blandit, +nisi vel cursus semper, nibh metus consequat purus, ac ullamcorper +dolor lorem vitae ligula. Maecenas non consectetur nibh.</Foo> + </Group> + <Group name="test2"> + <Bar name="3" encrypted="basic">Some text<Foo name="4" encrypted="basic">Nested text</Foo></Bar> + </Group> +</Properties> diff --git a/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext2.xml b/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext2.xml new file mode 100644 index 000000000..fa63330f0 --- /dev/null +++ b/testsuite/Testsrc/Testsbin/bcfg2-crypt/plaintext2.xml @@ -0,0 +1,18 @@ +<Properties> + <Foo name="1" encrypted="basic">Some text</Foo> + <Bar name="1" encrypted="basic">Some text</Bar> + <Foo name="2" encrypted="basic">Some text</Foo> + <Group name="test"> + <Bar name="2" encrypted="basic">Some text</Bar> + <Foo name="3" encrypted="basic">Praesent consectetur condimentum nisl ut cursus. Etiam aliquam nisi +dolor. Mauris aliquet condimentum neque, sodales laoreet lectus +venenatis ac. Morbi mattis justo odio, ac fringilla leo egestas +ut. Integer nec sapien pulvinar, ultrices nulla id, posuere +magna. Quisque in tincidunt sem, sed vehicula orci. Nulla blandit, +nisi vel cursus semper, nibh metus consequat purus, ac ullamcorper +dolor lorem vitae ligula. Maecenas non consectetur nibh.</Foo> + </Group> + <Group name="test2"> + <Bar name="3" encrypted="basic">Some text<Foo name="4" encrypted="basic">Nested text</Foo></Bar> + </Group> +</Properties> diff --git a/testsuite/Testsrc/Testsbin/test_bcfg2_crypt.py b/testsuite/Testsrc/Testsbin/test_bcfg2_crypt.py new file mode 100644 index 000000000..3eee4415f --- /dev/null +++ b/testsuite/Testsrc/Testsbin/test_bcfg2_crypt.py @@ -0,0 +1,390 @@ +# -*- coding: utf-8 -*- +import os +import sys +import shutil +import difflib +import tempfile +import lxml.etree +import Bcfg2.Options +from Bcfg2.Compat import StringIO, b64decode, u_str +from mock import Mock, MagicMock, patch + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * + +try: + from Bcfg2.Server.Encryption import CLI + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False + + +class TestEncryption(Bcfg2TestCase): + cfg_plaintext = None + known_files = None + basedir = None + + @classmethod + def setUpClass(cls): + basedir = os.path.join(os.path.dirname(__file__), "bcfg2-crypt") + cls.basedir = tempfile.mkdtemp() + for fname in os.listdir(basedir): + shutil.copy(os.path.join(basedir, fname), cls.basedir) + cls.known_files = os.listdir(cls.basedir) + cls.cfg_plaintext = open(os.path.join(cls.basedir, "plaintext")).read() + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.basedir) + + @skipUnless(HAS_CRYPTO, "Encryption libraries not found") + def setUp(self): + set_setup_default("lax_decryption", False) + + def set_options(self): + Bcfg2.Options.setup.algorithm = "aes_256_cbc" + Bcfg2.Options.setup.passphrases = dict( + basic="basic", + complex="1234567890əùÃÿł¢€ñû⸘" * 10) + + def tearDown(self): + # clean up stray files created by tests + for fname in os.listdir(self.basedir): + if fname not in self.known_files: + os.unlink(os.path.join(self.basedir, fname)) + + def assertExists(self, fname): + fpath = os.path.join(self.basedir, fname) + self.assertTrue(os.path.exists(fpath), + "%s does not exist" % fpath) + + def assertNotExists(self, fname): + fpath = os.path.join(self.basedir, fname) + self.assertFalse(os.path.exists(fpath), + "%s exists, but shouldn't" % fpath) + + def assertFilesEqual(self, fname1, fname2): + self.assertExists(fname1) + self.assertExists(fname2) + contents1 = open(os.path.join(self.basedir, fname1)).read().strip() + contents2 = open(os.path.join(self.basedir, fname2)).read().strip() + diff = "\n".join( + difflib.unified_diff(contents1.splitlines(), + contents2.splitlines(), + fname1, fname2)).replace("\n\n", "\n") + self.assertEqual(contents1, contents2, + "Contents of %s and %s do not match:\n%s" % + (fname1, fname2, diff)) + + def assertFilesNotEqual(self, fname1, fname2): + self.assertExists(fname1) + self.assertExists(fname2) + self.assertNotEqual( + open(os.path.join(self.basedir, fname1)).read(), + open(os.path.join(self.basedir, fname2)).read(), + "Contents of %s and %s are unexpectedly identical") + + def _is_encrypted(self, data): + """ Pretty crappy check for whether or not data is encrypted: + just see if it's a valid base64-encoded string whose contents + start with "Salted__". But without decrypting, which rather + begs the question in a set of crypto unit tests, I'm not sure + how to do a better test.""" + try: + return b64decode(data).startswith("Salted__") + except UnicodeDecodeError: + # decoded base64, resulting value contained non-ASCII text + return True + except TypeError: + # couldn't decode base64 + return False + + def assertIsEncrypted(self, data): + if not self._is_encrypted(data): + self.fail("Data is not encrypted: %s" % data) + + def assertNotEncrypted(self, data): + if self._is_encrypted(data): + self.fail("Data is unexpectedly encrypted: %s" % data) + + def _decrypt(self, cli, outfile, expected=None): + self.set_options() + cli.run() + if expected is None: + self.assertExists(outfile) + actual = open(os.path.join(self.basedir, outfile)).read() + self.assertEqual(self.cfg_plaintext, actual) + self.assertNotEncrypted(actual) + else: + self.assertFilesEqual(outfile, expected) + + def _encrypt(self, cli, outfile, original=None): + self.set_options() + cli.run() + if original is None: + self.assertExists(outfile) + actual = open(os.path.join(self.basedir, outfile)).read() + self.assertNotEqual(self.cfg_plaintext, actual) + self.assertIsEncrypted(actual) + else: + self.assertFilesNotEqual(outfile, original) + + def _cfg_decrypt(self, opts, encrypted): + if encrypted.endswith(".crypt"): + decrypted = encrypted[:-6] + else: + self.fail("Could not determine decrypted filename for %s" % + encrypted) + cli = CLI(opts + [os.path.join(self.basedir, encrypted)]) + self._decrypt(cli, decrypted) + + def _cfg_encrypt(self, opts, plaintext): + cli = CLI(opts + [os.path.join(self.basedir, plaintext)]) + self._encrypt(cli, plaintext + ".crypt") + + def _props_decrypt(self, opts, encrypted, expected): + test = os.path.join(self.basedir, "test.xml") + shutil.copy(os.path.join(self.basedir, encrypted), test) + cli = CLI(opts + [test]) + self._decrypt(cli, "test.xml", expected) + try: + xdata = lxml.etree.parse(test) + except: + self.fail("Could not parse decrypted Properties file: %s" % + sys.exc_info()[1]) + for el in xdata.iter(): + if el.tag is not lxml.etree.Comment and el.text.strip(): + self.assertNotEncrypted(el.text) + + def _props_encrypt(self, opts, plaintext, check_all=True): + test = os.path.join(self.basedir, "test.xml") + shutil.copy(os.path.join(self.basedir, plaintext), test) + cli = CLI(opts + [test]) + self._encrypt(cli, "test.xml", plaintext) + try: + xdata = lxml.etree.parse(test) + except: + self.fail("Could not parse encrypted Properties file: %s" % + sys.exc_info()[1]) + if check_all: + for el in xdata.iter(): + if el.tag is not lxml.etree.Comment and el.text.strip(): + self.assertIsEncrypted(el.text) + + def test_decrypt_cfg(self): + """ Decrypt a Cfg file """ + self._cfg_decrypt(["--decrypt", "--cfg", "-p", "basic"], + "basic.crypt") + + def test_decrypt_cfg_complex(self): + """ Decrypt a Cfg file with a passphrase with special characters """ + self._cfg_decrypt(["--decrypt", "--cfg", "-p", "complex"], + "complex.crypt") + + def test_decrypt_cfg_algorithm(self): + """ Decrypt a Cfg file with a non-default algorithm """ + # this can't be done with self._cfg_decrypt or even + # self._decrypt because we have to set the algorithm after + # other options are set, but before the decrypt is performed + cli = CLI(["--decrypt", "--cfg", "-p", "basic", + os.path.join(self.basedir, "basic-des-cbc.crypt")]) + self.set_options() + Bcfg2.Options.setup.algorithm = "des_cbc" + cli.run() + self.assertExists("basic-des-cbc") + actual = open(os.path.join(self.basedir, "basic-des-cbc")).read() + self.assertEqual(self.cfg_plaintext, actual) + self.assertNotEncrypted(actual) + + def test_cfg_auto_passphrase(self): + """ Discover the passphrase to decrypt a Cfg file""" + self._cfg_decrypt(["--decrypt", "--cfg"], "complex.crypt") + + def test_cfg_auto_mode(self): + """ Discover whether to encrypt or decrypt a Cfg file """ + self._cfg_decrypt(["--cfg", "-p", "basic"], "basic.crypt") + self._cfg_encrypt(["--cfg", "-p", "basic"], "plaintext") + + def test_cfg_auto_type(self): + """ Discover a file is a Cfg file """ + self._cfg_decrypt(["--decrypt", "-p", "basic"], "basic.crypt") + self._cfg_encrypt(["--encrypt", "-p", "basic"], "plaintext") + + def test_cfg_multiple(self): + """ Decrypt multiple Cfg files """ + cli = CLI(["--decrypt", "--cfg", "-p", "basic", + os.path.join(self.basedir, "basic.crypt"), + os.path.join(self.basedir, "basic2.crypt")]) + self.set_options() + cli.run() + self.assertExists("basic") + self.assertExists("basic2") + actual1 = open(os.path.join(self.basedir, "basic")).read() + actual2 = open(os.path.join(self.basedir, "basic2")).read() + self.assertEqual(self.cfg_plaintext, actual1) + self.assertEqual(self.cfg_plaintext, actual2) + self.assertNotEncrypted(actual1) + self.assertNotEncrypted(actual2) + + def test_cfg_auto_all(self): + """ Discover all options to encrypt/decrypt Cfg files """ + self._cfg_decrypt([], "complex.crypt") + self._cfg_encrypt(["-p", "basic"], "plaintext") + + def test_cfg_stdout(self): + """ Decrypt a Cfg file to stdout """ + cli = CLI(["--decrypt", "--cfg", "-p", "basic", "--stdout", + os.path.join(self.basedir, "basic.crypt")]) + self.set_options() + old_stdout = sys.stdout + sys.stdout = StringIO() + cli.run() + output = sys.stdout.getvalue() + sys.stdout = old_stdout + + self.assertNotExists("basic") + self.assertEqual(self.cfg_plaintext.strip(), output.strip()) + self.assertNotEncrypted(output) + + def test_encrypt_cfg(self): + """ Encrypt a Cfg file """ + self._cfg_encrypt(["--encrypt", "--cfg", "-p", "basic"], "plaintext") + os.rename(os.path.join(self.basedir, "plaintext.crypt"), + os.path.join(self.basedir, "test.crypt")) + self._cfg_decrypt(["--decrypt", "--cfg", "-p", "basic"], + "test.crypt") + + def test_encrypt_props_as_cfg(self): + """ Encrypt an XML file as a Cfg file """ + cli = CLI(["--encrypt", "--cfg", "-p", "basic", + os.path.join(self.basedir, "plaintext.xml")]) + self._encrypt(cli, "plaintext.xml.crypt", "plaintext.xml") + + os.rename(os.path.join(self.basedir, "plaintext.xml.crypt"), + os.path.join(self.basedir, "test.xml.crypt")) + cli = CLI(["--decrypt", "--cfg", "-p", "basic", + os.path.join(self.basedir, "test.xml.crypt")]) + self._decrypt(cli, "test.xml", "plaintext.xml") + + def test_cfg_remove(self): + """ Encrypt and remove a Cfg file """ + test = os.path.join(self.basedir, "test") + shutil.copy(os.path.join(self.basedir, "plaintext"), test) + self._cfg_encrypt(["--encrypt", "--remove", "--cfg", "-p", "basic"], + test) + self.assertNotExists("test") + + def test_decrypt_props(self): + """ Decrypt a Properties file """ + self._props_decrypt(["--decrypt", "--properties", "-p", "basic"], + "all-basic.xml", "plaintext2.xml") + + def test_props_decrypt_multiple_passphrases(self): + """ Decrypt a Properties file with multiple passphrases""" + self._props_decrypt(["--decrypt", "--properties"], + "plaintext-all.xml", "plaintext.xml") + + def test_props_decrypt_mixed(self): + """ Decrypt a Properties file with mixed encrypted content""" + self._props_decrypt(["--decrypt", "--properties"], + "plaintext-xpath.xml", "plaintext.xml") + + def test_props_decrypt_bogus(self): + """ Decrypt a malformed Properties file """ + self._props_decrypt(["--decrypt", "--properties"], + "bogus-forced.xml", "bogus.xml") + + def test_props_decrypt_auto_type(self): + """ Discover an encrypted file is a Properties file """ + self._props_decrypt(["--decrypt"], + "all-basic.xml", "plaintext2.xml") + + def test_props_decrypt_auto_mode(self): + """ Discover whether to encrypt or decrypt an encrypted Properties file """ + self._props_decrypt(["--properties"], + "all-basic.xml", "plaintext2.xml") + + def test_props_decrypt_auto_all(self): + """ Discover all options to decrypt a Properties file """ + self._props_decrypt([], "all-basic.xml", "plaintext2.xml") + + def test_props_encrypt_cli_passphrase(self): + """ Encrypt a Properties file with passphrase on the CLI""" + self._props_encrypt(["--encrypt", "--properties", "-p", "basic"], + "plaintext2.xml") + os.rename(os.path.join(self.basedir, "test.xml"), + os.path.join(self.basedir, "encrypted.xml")) + self._props_decrypt(["--decrypt", "--properties", "-p", "basic"], + "encrypted.xml", "plaintext2.xml") + + def test_props_encrypt_file_passphrase(self): + """ Encrypt a Properties file with passphrase in the file """ + self._props_encrypt(["--encrypt", "--properties"], "plaintext2.xml") + os.rename(os.path.join(self.basedir, "test.xml"), + os.path.join(self.basedir, "encrypted.xml")) + self._props_decrypt(["--decrypt", "--properties"], + "encrypted.xml", "plaintext2.xml") + + def test_props_encrypt_multiple_passphrases(self): + """ Encrypt a Properties file with multiple passphrases """ + self._props_encrypt(["--encrypt", "--properties"], "plaintext.xml") + os.rename(os.path.join(self.basedir, "test.xml"), + os.path.join(self.basedir, "encrypted.xml")) + self._props_decrypt(["--decrypt", "--properties"], + "encrypted.xml", "plaintext.xml") + + def test_props_encrypt_xpath(self): + """ Encrypt a Properties file with --xpath """ + test = os.path.join(self.basedir, "test.xml") + self._props_encrypt(["--encrypt", "--properties", "--xpath", "//Foo"], + "plaintext.xml", check_all=False) + xdata = lxml.etree.parse(test) + for el in xdata.iter(): + if el.tag is not lxml.etree.Comment and el.text.strip(): + if el.tag == "Foo": + self.assertIsEncrypted(el.text) + else: + self.assertNotEncrypted(el.text) + + os.rename(test, os.path.join(self.basedir, "encrypted.xml")) + self._props_decrypt(["--decrypt", "--properties"], + "encrypted.xml", "plaintext.xml") + + def test_props_encrypt_bogus(self): + """ Decrypt a malformed Properties file """ + self._props_encrypt(["--encrypt", "--properties"], "bogus.xml") + os.rename(os.path.join(self.basedir, "test.xml"), + os.path.join(self.basedir, "encrypted.xml")) + self._props_decrypt(["--decrypt", "--properties"], + "encrypted.xml", "bogus.xml") + + def test_props_encrypt_auto_type(self): + """ Discover if a file is a Properties file """ + self._props_encrypt(["--encrypt"], "plaintext2.xml") + os.rename(os.path.join(self.basedir, "test.xml"), + os.path.join(self.basedir, "encrypted.xml")) + self._props_decrypt(["--decrypt"], + "encrypted.xml", "plaintext2.xml") + + def test_props_encrypt_auto_mode(self): + """ Discover whether to encrypt or decrypt a Properties file """ + self._props_encrypt(["--properties"], "plaintext2.xml") + os.rename(os.path.join(self.basedir, "test.xml"), + os.path.join(self.basedir, "encrypted.xml")) + self._props_decrypt(["--properties"], + "encrypted.xml", "plaintext2.xml") + + def test_props_encrypt_auto_all(self): + """ Discover all options to encrypt a Properties file """ + self._props_encrypt([], "plaintext.xml") + os.rename(os.path.join(self.basedir, "test.xml"), + os.path.join(self.basedir, "encrypted.xml")) + self._props_decrypt([], "encrypted.xml", "plaintext.xml") diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index d9f985104..79eff7959 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -39,6 +39,7 @@ contingent_checks = { ("yum",): {"lib/Bcfg2/Client/Tools": ["YUM.py"]}, ("genshi",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgGenshiGenerator.py"]}, ("Cheetah",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgCheetahGenerator.py"]}, + ("jinja2",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgJinja2Generator.py"]}, ("M2Crypto",): {"lib/Bcfg2": ["Encryption.py"], "lib/Bcfg2/Server/Plugins/Cfg": ["CfgEncryptedGenerator.py"]}, @@ -46,15 +47,16 @@ contingent_checks = { ["CfgEncryptedGenshiGenerator.py"]}, ("M2Crypto", "Cheetah"): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgEncryptedCheetahGenerator.py"]}, + ("M2Crypto", "jinja2"): {"lib/Bcfg2/Server/Plugins/Cfg": + ["CfgEncryptedJinja2Generator.py"]}, } # perform only error checking on the listed files error_checks = { - "sbin": ["bcfg2-build-reports"], - "lib/Bcfg2": ["Proxy.py", "SSLServer.py", "Reporting"], - "lib/Bcfg2/Server": ["Reports", "SchemaUpdater"], - "lib/Bcfg2/Server/Admin": ["Compare.py", - "Snapshots.py"], + "lib/Bcfg2": ["Reporting"], + "lib/Bcfg2/Client": ["Proxy.py"], + "lib/Bcfg2/Server": ["Reports", "SchemaUpdater", "SSLServer.py"], + "lib/Bcfg2/Server/Admin": ["Compare.py"], "lib/Bcfg2/Client/Tools": ["OpenCSW.py", "Blast.py", "FreeBSDInit.py", @@ -71,18 +73,11 @@ no_checks = { "lib/Bcfg2/Server": ["Snapshots", "Hostbase"], "lib/Bcfg2": ["manage.py"], "lib/Bcfg2/Server/Reports": ["manage.py"], - "lib/Bcfg2/Server/Plugins": ["Account.py", - "Base.py", - "Editor.py", - "Hostbase.py", - "Snapshots.py", - "Statistics.py", - "TCheetah.py", - "TGenshi.py"], + "lib/Bcfg2/Server/Plugins": ["Base.py"], } if sys.version_info < (2, 6): # multiprocessing core requires py2.6 - no_checks['lib/Bcfg2/Server'].append('MultiprocessingCore.py') + no_checks['lib/Bcfg2/Server'] = ['MultiprocessingCore.py'] try: any diff --git a/testsuite/before_install.sh b/testsuite/before_install.sh deleted file mode 100755 index 2c80036cd..000000000 --- a/testsuite/before_install.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -ex - -# before_install script for Travis-CI - -PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') - -sudo apt-get update -qq -sudo apt-get install -qq swig libxml2-utils -if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then - if [[ ${PYVER:0:1} == "2" ]]; then - sudo apt-get install -y yum libaugeas0 augeas-lenses libacl1-dev \ - libssl-dev - fi -fi diff --git a/testsuite/common.py b/testsuite/common.py index 7471795a6..5a08f8db5 100644 --- a/testsuite/common.py +++ b/testsuite/common.py @@ -12,15 +12,13 @@ import os import re import sys import codecs -import unittest import lxml.etree +import Bcfg2.Options from mock import patch, MagicMock, _patch, DEFAULT -from Bcfg2.Compat import wraps - -#: The path to the Bcfg2 specification root for the tests. Using the -#: root directory exposes a lot of potential problems with building -#: paths. -datastore = "/" +try: + from unittest import skip, skipIf, skipUnless, TestCase +except ImportError: + from unittest2 import skip, skipIf, skipUnless, TestCase #: The XInclude namespace name XI_NAMESPACE = "http://www.w3.org/2001/XInclude" @@ -34,19 +32,43 @@ inPy3k = False if sys.hexversion >= 0x03000000: inPy3k = True + +#: A function to set a default config option if it's not already set +def set_setup_default(option, value=None): + if not hasattr(Bcfg2.Options.setup, option): + setattr(Bcfg2.Options.setup, option, value) + +Bcfg2.Options.Parser.unit_test = True + try: - from django.core.management import setup_environ + import django.conf has_django = True - os.environ['DJANGO_SETTINGS_MODULE'] = "Bcfg2.settings" - - import Bcfg2.settings - Bcfg2.settings.DATABASE_NAME = \ - os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.sqlite") - Bcfg2.settings.DATABASES['default']['NAME'] = Bcfg2.settings.DATABASE_NAME + set_setup_default("db_engine", "sqlite3") + set_setup_default("db_name", + os.path.join(os.path.dirname(os.path.abspath(__file__)), + "test.sqlite")) + set_setup_default("db_user") + set_setup_default("db_password") + set_setup_default("db_host") + set_setup_default("db_port") + set_setup_default("db_opts", dict()) + set_setup_default("db_schema") + set_setup_default("timezone") + set_setup_default("web_debug", False) + set_setup_default("web_prefix") + + import Bcfg2.DBSettings + Bcfg2.DBSettings.finalize_django_config() except ImportError: has_django = False +#: The path to the Bcfg2 specification root for the tests. Using the +#: root directory exposes a lot of potential problems with building +#: paths. +datastore = "/" + +set_setup_default("repository", datastore) try: from mock import call @@ -91,180 +113,12 @@ else: return codecs.unicode_escape_decode(s)[0] -#: Whether or not skipping tests is natively supported by -#: :mod:`unittest`. If it isn't, then we have to make tests that -#: would be skipped succeed instead. -can_skip = False - -if hasattr(unittest, "skip"): - can_skip = True - - #: skip decorator from :func:`unittest.skip` - skip = unittest.skip - - #: skipIf decorator from :func:`unittest.skipIf` - skipIf = unittest.skipIf - - #: skipUnless decorator from :func:`unittest.skipUnless` - skipUnless = unittest.skipUnless -else: - # we can't actually skip tests, we just make them pass - can_skip = False - - def skip(msg): - """ skip decorator used when :mod:`unittest` doesn't support - skipping tests. Replaces the decorated function with a - no-op. """ - def decorator(func): - return lambda *args, **kwargs: None - return decorator - - def skipIf(condition, msg): - """ skipIf decorator used when :mod:`unittest` doesn't support - skipping tests """ - if not condition: - return lambda f: f - else: - return skip(msg) - - def skipUnless(condition, msg): - """ skipUnless decorator used when :mod:`unittest` doesn't - support skipping tests """ - if condition: - return lambda f: f - else: - return skip(msg) - - -def _count_diff_all_purpose(actual, expected): - '''Returns list of (cnt_act, cnt_exp, elem) triples where the - counts differ''' - # elements need not be hashable - s, t = list(actual), list(expected) - m, n = len(s), len(t) - NULL = object() - result = [] - for i, elem in enumerate(s): - if elem is NULL: - continue - cnt_s = cnt_t = 0 - for j in range(i, m): - if s[j] == elem: - cnt_s += 1 - s[j] = NULL - for j, other_elem in enumerate(t): - if other_elem == elem: - cnt_t += 1 - t[j] = NULL - if cnt_s != cnt_t: - diff = (cnt_s, cnt_t, elem) - result.append(diff) - - for i, elem in enumerate(t): - if elem is NULL: - continue - cnt_t = 0 - for j in range(i, n): - if t[j] == elem: - cnt_t += 1 - t[j] = NULL - diff = (0, cnt_t, elem) - result.append(diff) - return result - - -def _assertion(predicate, default_msg=None): - @wraps(predicate) - def inner(self, *args, **kwargs): - if 'msg' in kwargs: - msg = kwargs['msg'] - del kwargs['msg'] - else: - try: - msg = default_msg % args - except TypeError: - # message passed as final (non-keyword) argument? - msg = args[-1] - args = args[:-1] - assert predicate(*args, **kwargs), msg - return inner - - -def _regex_matches(val, regex): - if hasattr(regex, 'search'): - return regex.search(val) - else: - return re.search(regex, val) - - -class Bcfg2TestCase(unittest.TestCase): +class Bcfg2TestCase(TestCase): """ Base TestCase class that inherits from - :class:`unittest.TestCase`. This class does a few things: - - * Adds :func:`assertXMLEqual`, a useful assertion method given all - the XML used by Bcfg2; - - * Defines convenience methods that were (mostly) added in Python - 2.7. + :class:`unittest.TestCase`. This class adds + :func:`assertXMLEqual`, a useful assertion method given all the + XML used by Bcfg2. """ - if not hasattr(unittest.TestCase, "assertItemsEqual"): - # TestCase in Py3k lacks assertItemsEqual, but has the other - # convenience methods. this code is (mostly) cribbed from the - # py2.7 unittest library - def assertItemsEqual(self, expected_seq, actual_seq, msg=None): - """ Implementation of - :func:`unittest.TestCase.assertItemsEqual` for python - versions that lack it """ - first_seq, second_seq = list(actual_seq), list(expected_seq) - differences = _count_diff_all_purpose(first_seq, second_seq) - - if differences: - standardMsg = 'Element counts were not equal:\n' - lines = ['First has %d, Second has %d: %r' % diff - for diff in differences] - diffMsg = '\n'.join(lines) - standardMsg += diffMsg - if msg is None: - msg = standardMsg - else: - msg = "%s : %s" % (standardMsg, msg) - self.fail(msg) - - if not hasattr(unittest.TestCase, "assertRegexpMatches"): - # Some versions of TestCase in Py3k seem to lack - # assertRegexpMatches, but have the other convenience methods. - assertRegexpMatches = _assertion(lambda s, r: _regex_matches(s, r), - "%s does not contain /%s/") - - if not hasattr(unittest.TestCase, "assertNotRegexpMatches"): - # Some versions of TestCase in Py3k seem to lack - # assertNotRegexpMatches even though they have - # assertRegexpMatches - assertNotRegexpMatches = \ - _assertion(lambda s, r: not _regex_matches(s, r), - "%s contains /%s/") - - if not hasattr(unittest.TestCase, "assertIn"): - # versions of TestCase before python 2.7 and python 3.1 lacked - # a lot of the really handy convenience methods, so we provide - # them -- at least the easy ones and the ones we use. - assertIs = _assertion(lambda a, b: a is b, "%s is not %s") - assertIsNot = _assertion(lambda a, b: a is not b, "%s is %s") - assertIsNone = _assertion(lambda x: x is None, "%s is not None") - assertIsNotNone = _assertion(lambda x: x is not None, "%s is None") - assertIn = _assertion(lambda a, b: a in b, "%s is not in %s") - assertNotIn = _assertion(lambda a, b: a not in b, "%s is in %s") - assertIsInstance = _assertion(isinstance, "%s is not instance of %s") - assertNotIsInstance = _assertion(lambda a, b: not isinstance(a, b), - "%s is instance of %s") - assertGreater = _assertion(lambda a, b: a > b, - "%s is not greater than %s") - assertGreaterEqual = _assertion(lambda a, b: a >= b, - "%s is not greater than or equal to %s") - assertLess = _assertion(lambda a, b: a < b, "%s is not less than %s") - assertLessEqual = _assertion(lambda a, b: a <= b, - "%s is not less than or equal to %s") - def assertXMLEqual(self, el1, el2, msg=None): """ Test that the two XML trees given are equal. """ if msg is None: @@ -308,15 +162,18 @@ class Bcfg2TestCase(unittest.TestCase): class DBModelTestCase(Bcfg2TestCase): """ Test case class for Django database models """ models = [] + __test__ = False @skipUnless(has_django, "Django not found, skipping") def test_syncdb(self): """ Create the test database and sync the schema """ - setup_environ(Bcfg2.settings) - import django.core.management - django.core.management.call_command("syncdb", interactive=False, - verbosity=0) - self.assertTrue(os.path.exists(Bcfg2.settings.DATABASE_NAME)) + if self.models: + import django.core.management + django.core.management.call_command("syncdb", interactive=False, + verbosity=0) + self.assertTrue( + os.path.exists( + django.conf.settings.DATABASES['default']['NAME'])) @skipUnless(has_django, "Django not found, skipping") def test_cleandb(self): diff --git a/testsuite/install.sh b/testsuite/install.sh index 9de5b8c6d..bbbd9ae76 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -2,13 +2,22 @@ # install script for Travis-CI -pip install -r testsuite/requirements.txt --use-mirrors +sudo apt-get update -qq +sudo apt-get install swig libxml2-utils + +pip install -r testsuite/requirements.txt PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') +if [[ ${PYVER:0:1} == "2" && $PYVER != "2.7" ]]; then + pip install --use-mirrors unittest2 +fi + if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then - pip install --use-mirrors genshi PyYAML pyinotify boto 'django<1.5' \ - pylibacl + sudo apt-get install -y yum libaugeas0 augeas-lenses libacl1-dev libssl-dev + + pip install --use-mirrors PyYAML pyinotify boto pylibacl 'django<1.5' \ + Jinja2 easy_install https://fedorahosted.org/released/python-augeas/python-augeas-0.4.1.tar.gz if [[ ${PYVER:0:1} == "2" ]]; then # django supports py3k, but South doesn't, and the django bits diff --git a/testsuite/pylintrc.conf b/testsuite/pylintrc.conf index 94904877b..1d3ba8c88 100644 --- a/testsuite/pylintrc.conf +++ b/testsuite/pylintrc.conf @@ -154,7 +154,7 @@ ignore-mixin-members=yes # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). -ignored-classes=ForeignKey,Interaction,git.cmd.Git +ignored-classes=ForeignKey,Interaction,git.cmd.Git,argparse.Namespace,Namespace # When zope mode is activated, add a predefined set of Zope acquired attributes # to generated-members. diff --git a/testsuite/requirements.txt b/testsuite/requirements.txt index 898249389..d7eaa1ac9 100644 --- a/testsuite/requirements.txt +++ b/testsuite/requirements.txt @@ -5,3 +5,5 @@ sphinx pylint<1.0 pep8 python-daemon +genshi +argparse diff --git a/tools/README b/tools/README index 5505573c8..3c705b660 100644 --- a/tools/README +++ b/tools/README @@ -3,9 +3,6 @@ This directory contains repository maintenance tools. basebuilder.py <image directory> - builds v2 base.xml from bcfg1 repo -batchadd.py <filename> - - Add records to Hostbase - bcfg2-completion.bash - Bash tab completion for bcfg2-admin @@ -20,9 +17,6 @@ bcfg2_local.py - Perform a full Bcfg2 run against a local repository instead of against a remote server -bcfg2-profile-templates.py [<template>] - - Benchmark template rendering times - bcfg2_svnlog.py - Send intelligent log messages about changes made to your Bcfg2 repository from SVN postcommit @@ -61,15 +55,6 @@ generate-manpages.bash git_commit.py - Trigger script to commit local changes back to a git repository -hostbasepush.py - - Call the Hostbase.rebuildState XML-RPC method - -hostbase.py {-l|-c} <hostname> - - Display or create host information for Hostbase - -hostinfo.py {-q <query>|--showfields} - - Query the hostbase databse - pkgmgr_gen.py - Generate Pkgmgr XML files from a list of directories that contain RPMS diff --git a/tools/batchadd.py b/tools/batchadd.py deleted file mode 100755 index e8008b330..000000000 --- a/tools/batchadd.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/python - -from datetime import date -import os -import sys - -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings' -from Bcfg2.Server.Hostbase.hostbase.models import * -from Bcfg2.Server.Hostbase.settings import DEFAULT_MX, PRIORITY -import Bcfg2.Server.Hostbase.regex - -host_attribs = ['administrator', - 'comments', - 'csi', - 'expiration_date', - 'hostname', - 'location', - 'netgroup', - 'outbound_smtp', - 'primary_user', - 'printq', - 'security_class', - 'support', - 'whatami'] - - -def handle_error(field): - if '-f' in sys.argv: - return - print("Error: %s is already defined in hostbase" % field) - if '-s' in sys.argv: - sys.exit(1) - - -def checkformat(values, indices): - """Ensures file contains all necessary attributes in order """ - filelist = [pair[0] for pair in values] - - # lines = len(filelist) - - filelist = filelist[indices[0]:] - - for index in indices: - if filelist[0:13] != host_attribs: - # figure out what to do here - return False - else: - # process rest of host attributes - try: - next = filelist[1:].index('hostname') - remaining = filelist[13:next + 1] - filelist = filelist[next + 1:] - except: - remaining = filelist[13:] - needfields = ['mac_addr', 'hdwr_type', 'ip_addr'] - if [item for item in needfields if item not in remaining]: - return False - return True - - -if __name__ == '__main__': - - # argument handling for batchadd - try: - fd = open(sys.argv[1], 'r') - except (IndexError, IOError): - print("\nUsage: batchadd.py filename\n") - sys.exit() - - lines = fd.readlines() - # splits and strips lines into (attribute, value) - info = [[item.strip() for item in line.split("->")] for line in lines - if line.lstrip(' ')[0] != '#' and line != '\n'] - - if info[0][0] == 'mx' and info[1][0] == 'priority': - mx, created = MX.objects.get_or_create(mx=info[0][1], - priority=info[1][1]) - info = info[2:] - else: - mx, created = MX.objects.get_or_create(mx=DEFAULT_MX, - priority=PRIORITY) - if created: - mx.save() - - hostindices = [num for num in range(0, len(info)) - if info[num][0] == 'hostname'] - - if not checkformat(info, hostindices): - print("Error: file format") - sys.exit() - -################# - - for host in hostindices: - try: - host = Host.objects.get(hostname=info[host][1]) - handle_error(info[host][1]) - except: - # do something here - pass - - macindices = [num for num in range(0, len(info)) - if info[num][0] == 'mac_addr'] - for mac_addr in macindices: - try: - host = Interface.objects.get(mac_addr=info[mac_addr][1]) - handle_error(info[mac_addr][1]) - except: - # do something here - pass - - for host in hostindices: - blank = Host() - for attrib in host_attribs: - pair = info.pop(0) - if pair[0] == 'outbound_smtp': - if pair[1] == 'y': - blank.__dict__[pair[0]] = 1 - else: - blank.__dict__[pair[0]] = 0 - elif pair[0] == 'expiration_date': - (year, month, day) = pair[1].split("-") - blank.expiration_date = date(int(year), - int(month), - int(day)) - else: - blank.__dict__[pair[0]] = pair[1] - blank.status = 'active' - blank.save() - newhostname = blank.hostname.split(".")[0] - newdomain = blank.hostname.split(".", 1)[1] - while info and info[0][0] != 'hostname': - if info[0][0] == 'mac_addr': - pair = info.pop(0) - inter = Interface.objects.create(host=blank, - mac_addr=pair[1], - hdwr_type='eth') - if not pair[1]: - inter.dhcp = False - inter.save() - elif info[0][0] == 'hdwr_type': - pair = info.pop(0) - inter.hdwr_type = pair[1] - inter.save() - elif info[0][0] == 'ip_addr': - pair = info.pop(0) - ip = IP.objects.create(interface=inter, ip_addr=pair[1]) - hostnamenode = Name(ip=ip, - name=blank.hostname, - dns_view='global', - only=False) - hostnamenode.save() - namenode = Name(ip=ip, - name=".".join([newhostname + "-" + inter.hdwr_type, - newdomain]), - dns_view="global", only=False) - namenode.save() - subnetnode = Name(ip=ip, name=newhostname + "-" + - ip.ip_addr.split(".")[2] + "." + - newdomain, dns_view="global", only=False) - subnetnode.save() - hostnamenode.mxs.add(mx) - namenode.mxs.add(mx) - subnetnode.mxs.add(mx) - elif info[0][0] == 'cname': - pair = info.pop(0) - cname = CName.objects.create(name=hostnamenode, cname=pair[1]) - cname.save() diff --git a/tools/bcfg2-completion.bash b/tools/bcfg2-completion.bash index 99cfb695b..b9e799cf8 100644 --- a/tools/bcfg2-completion.bash +++ b/tools/bcfg2-completion.bash @@ -1,14 +1,13 @@ # TODO: Add completion for each admin mode _bcfg2-admin() { - local cur prev sedcmd possibles + local cur prev possibles COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" - sedcmd='sed -n -e s/^[[:space:]][[:space:]][[:space:]]\([[:alpha:]]\+\)[[:space:]]\+.*$/\1/p' if [[ ${COMP_CWORD} -eq 1 ]] || [[ -n "${prev}" && ${prev} == -* ]] then - possibles="$(bcfg2-admin help | ${sedcmd})" + possibles="$(bcfg2-admin help | awk '{print $1}')" #elif bcfg2-admin ${prev} help &>/dev/null ; then # possibles=$(bcfg2-admin ${prev} help | ${sedcmd}) fi @@ -18,4 +17,21 @@ _bcfg2-admin() { return 0 } +_bcfg2-info() { + local cur prev possibles + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [[ ${COMP_CWORD} -eq 1 ]] || [[ -n "${prev}" && ${prev} == -* ]] + then + possibles="$(bcfg2-info help | awk '{print $1}')" + fi + + [[ -n "${possibles}" ]] && \ + COMPREPLY=( $(compgen -W "${possibles}" -- ${cur}) ) + + return 0 +} complete -F _bcfg2-admin bcfg2-admin +complete -F _bcfg2-info bcfg2-info diff --git a/tools/bcfg2-import-config b/tools/bcfg2-import-config index d6273f0c6..fec007e7e 100755 --- a/tools/bcfg2-import-config +++ b/tools/bcfg2-import-config @@ -11,7 +11,6 @@ usage() { echo "$0: tool to import files in to bcfg2-server repository" echo " -s Copy SSH Key files" - echo " -p Create :info files with current file permissions" echo " -n No suffix. Generate global files" echo " --debian Run debsums to detect changed configuration files" echo " ** debsums is only able to detect part of changes!" @@ -28,7 +27,6 @@ eval set -- "$TEMP" ## Start Defaults NEEDSSH=0 -NEEDPERM=0 DEBSUMS=0 NOSUFFIX=0 # End Defaults @@ -37,7 +35,6 @@ NOSUFFIX=0 while true ; do case "$1" in -s) NEEDSSH=1; shift ;; - -p) NEEDPERM=1; shift ;; --debian) DEBSUMS=1; shift ;; -n) NOSUFFIX=1; shift ;; -h|--help) @@ -102,11 +99,6 @@ get_files() { FILE=$(basename $i) mkdir -p $CFGREPO/$i cp $i $CFGREPO/$i/${FILE}${SUFFIX} - if [ $NEEDPERM -ne 0 ]; then - # Get permissions for the file - echo -n "(permissions) " - find $i -printf "owner:%u\ngroup:%g\nperms:%#m\n" > "$CFGREPO/$i/:info" - fi echo "OK" else echo "$i: Not a file" @@ -126,7 +118,7 @@ get_debsums() { } ## End Functions -if [ $(($NEEDPERM + $NEEDSSH + $DEBSUMS)) -eq 0 -a -z "$FILES" ]; then usage ; exit 0; fi +if [ $(($NEEDSSH + $DEBSUMS)) -eq 0 -a -z "$FILES" ]; then usage ; exit 0; fi init_temp_repo get_debsums diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py deleted file mode 100755 index 2b0ca6d63..000000000 --- a/tools/bcfg2-profile-templates.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/python -Ott -# -*- coding: utf-8 -*- -""" Benchmark template rendering times """ - -import sys -import time -import math -import signal -import logging -import operator -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Core - - -def stdev(nums): - mean = float(sum(nums)) / len(nums) - return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums))) - - -def get_sigint_handler(core): - """ Get a function that handles SIGINT/Ctrl-C by shutting down the - core and exiting properly.""" - - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - core.shutdown() - os._exit(1) # pylint: disable=W0212 - - return hdlr - - -def main(): - optinfo = dict( - client=Bcfg2.Options.Option("Benchmark templates for one client", - cmd="--client", - odesc="<client>", - long_arg=True, - default=None), - runs=Bcfg2.Options.Option("Number of rendering passes per template", - cmd="--runs", - odesc="<runs>", - long_arg=True, - default=5, - cook=int)) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging("bcfg2-test", - to_console=setup['verbose'] or setup['debug'], - to_syslog=False, - to_file=setup['logging'], - level=level) - logger = logging.getLogger(sys.argv[0]) - - core = Bcfg2.Server.Core.BaseCore(setup) - signal.signal(signal.SIGINT, get_sigint_handler(core)) - logger.info("Bcfg2 server core loaded") - core.load_plugins() - logger.debug("Plugins loaded") - core.block_for_fam_events(handle_events=True) - logger.debug("Repository events processed") - - if setup['args']: - templates = setup['args'] - else: - templates = [] - - if setup['client'] is None: - clients = [core.build_metadata(c) for c in core.metadata.clients] - else: - clients = [core.build_metadata(setup['client'])] - - times = dict() - client_count = 0 - for metadata in clients: - client_count += 1 - logger.info("Rendering templates for client %s (%s/%s)" % - (metadata.hostname, client_count, len(clients))) - structs = core.GetStructures(metadata) - struct_count = 0 - for struct in structs: - struct_count += 1 - logger.info("Rendering templates from structure %s:%s (%s/%s)" % - (struct.tag, struct.get("name"), struct_count, - len(structs))) - entries = struct.xpath("//Path") - entry_count = 0 - for entry in entries: - entry_count += 1 - if templates and entry.get("name") not in templates: - continue - logger.info("Rendering Path:%s (%s/%s)..." % - (entry.get("name"), entry_count, len(entries))) - ptimes = times.setdefault(entry.get("name"), []) - for i in range(setup['runs']): - start = time.time() - try: - core.Bind(entry, metadata) - ptimes.append(time.time() - start) - except: - break - if ptimes: - avg = sum(ptimes) / len(ptimes) - if avg: - logger.debug(" %s: %.02f sec" % - (metadata.hostname, avg)) - - # print out per-file results - tmpltimes = [] - for tmpl, ptimes in times.items(): - try: - mean = float(sum(ptimes)) / len(ptimes) - except ZeroDivisionError: - continue - ptimes.sort() - median = ptimes[len(ptimes) / 2] - std = stdev(ptimes) - if mean > 0.01 or median > 0.01 or std > 1 or templates: - tmpltimes.append((tmpl, mean, median, std)) - print("%-50s %-9s %-11s %6s" % - ("Template", "Mean Time", "Median Time", "σ")) - for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): - print("%-50s %9.02f %11.02f %6.02f" % info) - core.shutdown() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index 78a46ba5c..5e5bca777 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -6,19 +6,19 @@ the server core, then uses that to get probes, run them, and so on.""" import sys import socket import Bcfg2.Options -from Bcfg2.Client.Client import Client -from Bcfg2.Server.Core import BaseCore +from Bcfg2.Client import Client +from Bcfg2.Server.Core import Core -class LocalCore(BaseCore): +class LocalCore(Core): """ Local server core similar to the one started by bcfg2-info """ - def __init__(self, setup): - saved = (setup['syslog'], setup['logging']) - setup['syslog'] = False - setup['logging'] = None - Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) - setup['syslog'], setup['logging'] = saved + def __init__(self): + #saved = (setup['syslog'], setup['logging']) + #setup['syslog'] = False + #setup['logging'] = None + Bcfg2.Server.Core.BaseCore.__init__(self) + #setup['syslog'], setup['logging'] = saved self.load_plugins() self.block_for_fam_events(handle_events=True) @@ -60,26 +60,22 @@ class LocalClient(Client): """ A version of the Client class that uses LocalProxy instead of an XML-RPC proxy to make its calls """ - def __init__(self, setup, proxy): - Client.__init__(self, setup) + def __init__(self, proxy): + Client.__init__(self) self._proxy = proxy def main(): - optinfo = Bcfg2.Options.CLIENT_COMMON_OPTIONS - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - if 'bundle_quick' in optinfo: - # CLIENT_BUNDLEQUICK option uses -Q, just like the server repo - # option. the server repo is more important for this - # application. - optinfo['bundle_quick'] = Bcfg2.Options.Option('bundlequick', - default=False) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - core = LocalCore(setup) + parser = Bcfg2.Options.Parser( + description="Run a Bcfg2 client against a local repository without a " + "server", + conflict_handler="resolve", + components=[LocalCore, LocalProxy, LocalClient]) + parser.parse() + + core = LocalCore() try: - LocalClient(setup, LocalProxy(core)).run() + LocalClient(LocalProxy(core)).run() finally: core.shutdown() diff --git a/tools/export.py b/tools/export.py index 6b0238bbb..df3c810d5 100755 --- a/tools/export.py +++ b/tools/export.py @@ -257,13 +257,11 @@ E.G. 1.2.0pre1 is a valid version. startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2.spec', 'BuildRoot', - 'BuildRoot: %%{_tmppath}/%%{name}-%%{version}%s-%%{release}-root-%%(%%{__id_u} -n)\n' % - version_info['build'], + 'BuildRoot: %{_tmppath}/%{name}-%{version}%{?_pre_rc}-%{release}-root-%(%{__id_u} -n)\n', startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2-selinux.spec', 'BuildRoot', - 'BuildRoot: %%{_tmppath}/%%{name}-%%{version}%s-%%{release}-root-%%(%%{__id_u} -n)\n' % - version_info['build'], + 'BuildRoot: %{_tmppath}/%{name}-%{version}%{?_pre_rc}-%{release}-root-%(%{__id_u} -n)\n', startswith=True, dryrun=options.dryrun) # fix pre problem noted in diff --git a/tools/hostbase.py b/tools/hostbase.py deleted file mode 100755 index 7474e68b7..000000000 --- a/tools/hostbase.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/python -import os -from getopt import getopt, GetoptError -from re import split -import sys - -os.environ['DJANGO_SETTINGS_MODULE'] = 'Hostbase.settings' -from Hostbase.hostbase.models import Host - -attribs = ['administrator', - 'comments', - 'csi', - 'dhcp', - 'expiration_date', - 'hostname', - 'last', - 'location', - 'netgroup', - 'outbound_smtp', - 'primary_user', - 'printq', - 'security_class', - 'support', - 'status', - 'whatami'] - -already_exists = None -#here's my attempt at making the command line idiot proof -#you must supply and arugument and hostname for hostbase.py to run -try: - (opts, args) = getopt(sys.argv[1:], 'l:c:') - sys.argv[1] - if len(split("\.", opts[0][1])) == 1: - hosttouse = opts[0][1] + ".mcs.anl.gov" - else: - hosttouse = opts[0][1] -except (GetoptError, IndexError): - print("\nUsage: hostbase.py -flag (hostname)\n") - print("Flags:") - print("\t-l look (hostname)\n") -# print("\t-c copy (hostname)\n") - sys.exit() - -try: - host = Host.objects.get(hostname=hosttouse) -except: - print("Error: host %s not in hostbase" % hosttouse) - sys.exit(1) -interfaces = [] -for interface in host.interface_set.all(): - interfaces.append([interface, interface.ip_set.all()]) -hostinfo = "\n" -for attrib in attribs: - if not (opts[0][0] == '-c' and attrib in ['status', 'last']): - if attrib == 'dhcp' or attrib == 'outbound_smtp': - if host.__dict__[attrib]: - hostinfo += "%-32s-> %s\n" % (attrib, 'y') - else: - hostinfo += "%-32s-> %s\n" % (attrib, 'n') - else: - hostinfo += "%-32s-> %s\n" % (attrib, host.__dict__[attrib]) -for interface in interfaces: - hostinfo += "\n%-32s-> %s\n" % ('mac_addr', interface[0].mac_addr) - hostinfo += "%-32s-> %s\n" % ('hdwr_type', interface[0].hdwr_type) - for ip in interface[1]: - hostinfo += "%-32s-> %s\n" % ('ip_addr', ip.ip_addr) - -if opts[0][0] == '-l': - """Displays general host information""" - print(hostinfo) - -if opts[0][0] == '-c': - """Provides pre-filled template to copy a host record""" - fd = open('/tmp/hostbase.%s.tmp' % host.id, 'w') - fd.write(hostinfo) - fd.close() - os.system('vi + /tmp/hostbase.%s.tmp' % host.id) - os.system('batchadd.py /tmp/hostbase.%s.tmp' % host.id) - os.system('rm /tmp/hostbase.%s.tmp' % host.id) diff --git a/tools/hostbasepush.py b/tools/hostbasepush.py deleted file mode 100755 index 02b7a582f..000000000 --- a/tools/hostbasepush.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python - -import os -import Bcfg2.Client.Proxy - -if not os.getuid() == 0: - print("this command must be run as root") - raise SystemExit - -proxy = Bcfg2.Client.Proxy.bcfg2() -print("building files...") -proxy.run_method('Hostbase.rebuildState', ()) -print("running bcfg...") -os.system('bcfg2 -q -d -v') diff --git a/tools/hostinfo.py b/tools/hostinfo.py deleted file mode 100755 index 8ae5c4df6..000000000 --- a/tools/hostinfo.py +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/python -"""Hostinfo queries the hostbase database according to user-defined data""" - -from os import system, environ -environ['DJANGO_SETTINGS_MODULE'] = 'Hostbase.settings' -from getopt import gnu_getopt, GetoptError -from django.db import connection -import sys - -logic_ops = ["and", "or"] -host_attribs = ["hostname", "whatami", "netgroup", "security_class", - "support", "csi", "memory", "printq", "dhcp", "outbound_smtp", - "primary_user", "administrator", "location", - "comments", "last", "expiration_date"] -dispatch = {'mac_addr': ' i.', - 'hdwr_type': ' i.', - 'ip_addr': ' p.', - 'name': ' n.', - 'dns_view': ' n.', - 'cname': ' c.', - 'mx': ' m.', - 'priority': ' m.'} - - -def pinger(hosts): - """Function that uses fping to ping multiple hosts in parallel""" - hostnames = "" - for each in hosts: - hostnames += each[0] + " " - system("fping -r 1" + hostnames) - sys.exit() - - -def get_query(arguments): - """Parses the command line options and returns the necessary - data for an SQL query""" - logic = None - resultset = [] - querystring = '' - while 1: - notflag = False - if arguments[0] == 'not': - notflag = True - querypos = 1 - elif arguments[0] in logic_ops: - logic = arguments[0] - if arguments[1] == 'not': - notflag = True - querypos = 2 - else: - querypos = 1 - else: - querypos = 0 - if len(arguments[querypos].split("==")) > 1: - operator = "=" - if notflag: - operator = "<>" - querysplit = arguments[querypos].split("==") - if querysplit[0] in host_attribs: - querystring = " h.%s%s\'%s\'" % (querysplit[0], - operator, - querysplit[1]) - elif querysplit[0] in dispatch: - querystring = dispatch[querysplit[0]] - querystring += "%s%s\'%s\'" % (querysplit[0], - operator, - querysplit[1]) - elif len(arguments[querypos].split("=")) > 1: - notstring = '' - if notflag: - notstring = 'NOT ' - querysplit = arguments[querypos].split("=") - if querysplit[0] in host_attribs: - querystring = " h.%s %sLIKE \'%%%%%s%%%%\'" % (querysplit[0], - notstring, - querysplit[1]) - elif querysplit[0] in dispatch: - querystring = dispatch[querysplit[0]] - querystring += "%s %sLIKE \'%%%%%s%%%%\'" % (querysplit[0], - notstring, - querysplit[1]) - else: - print("ERROR: bad query format") - sys.exit() - if not querystring: - print("ERROR: bad query format") - sys.exit() - resultset.append((querystring, logic)) - arguments = arguments[querypos + 1:] - if arguments == [] or arguments[0] not in logic_ops: - break - return resultset - -try: - (opts, args) = gnu_getopt(sys.argv[1:], - 'q:', ['showfields', 'fields', 'ping', 'summary']) - cursor = connection.cursor() - if ('--showfields', '') in opts: - print("\nhost fields:\n") - for field in host_attribs: - print(field) - for field in dispatch: - print(field) - print("") - sys.exit() - if opts[0][0] == '-q': - results = get_query(sys.argv[2:]) - queryoptions = "" - for result in results: - if result[1] == 'and': - queryoptions += " AND " + result[0] - elif result[1] == 'or': - queryoptions += " OR " + result[0] - else: - queryoptions += result[0] - if ('--summary', '') in opts: - fields = "h.hostname, h.whatami, h.location, h.primary_user" - query = """SELECT DISTINCT %s FROM (((((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - INNER JOIN hostbase_name_mxs x ON x.name_id = n.id) - INNER JOIN hostbase_mx m ON m.id = x.mx_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE %s ORDER BY h.hostname - """ % (fields, queryoptions) - cursor.execute(query) - results = cursor.fetchall() - if not results: - print("No matches were found for your query") - sys.exit() - print("\n%-32s %-10s %-10s %-10s" % ('Hostname', 'Type', 'Location', 'User')) - print("================================ ========== ========== ==========") - for host in results: - print("%-32s %-10s %-10s %-10s" % (host)) - print("") - elif ('--fields', '') in opts: - tolook = [arg for arg in args if arg in host_attribs or arg in dispatch] - fields = "" - fields = ", ".join(tolook) - if not fields: - print("No valid fields were entered. exiting...") - sys.exit() - query = """SELECT DISTINCT %s FROM (((((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - INNER JOIN hostbase_name_mxs x ON x.name_id = n.id) - INNER JOIN hostbase_mx m ON m.id = x.mx_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE %s ORDER BY h.hostname - """ % (fields, queryoptions) - - cursor.execute(query) - results = cursor.fetchall() - - last = results[0] - for field in results[0]: - print(repr(field) + "\t") - for host in results: - if not host == last: - for field in host: - print(repr(field) + "\t") - last = host - print("") - else: - basequery = """SELECT DISTINCT h.hostname FROM (((((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - INNER JOIN hostbase_name_mxs x ON x.name_id = n.id) - INNER JOIN hostbase_mx m ON m.id = x.mx_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE - """ - cursor.execute(basequery + queryoptions + " ORDER BY h.hostname") - results = cursor.fetchall() - - if not results: - print("No matches were found for your query") - sys.exit() - - if ("--ping", '') in opts: - pinger(results) - - for host in results: - print(host[0]) - - -except (GetoptError, IndexError): - print("\nUsage: hostinfo.py -q <field>=[=]<value> [and/or <field>=<value> [--long option]]") - print(" hostinfo.py --showfields\tshows all data fields") - print("\n long options:") - print("\t --fields f1 f2 ...\tspecifies the fields displayed from the queried hosts") - print("\t --summary\t\tprints out a predetermined set of fields") - print("\t --ping\t\t\tuses fping to ping all queried hosts\n") - sys.exit() diff --git a/tools/posixusers_baseline.py b/tools/posixusers_baseline.py index 4c78a757e..06925beed 100755 --- a/tools/posixusers_baseline.py +++ b/tools/posixusers_baseline.py @@ -5,74 +5,51 @@ import sys import logging import lxml.etree import Bcfg2.Logger +import Bcfg2.Options from Bcfg2.Client.Tools.POSIXUsers import POSIXUsers -from Bcfg2.Options import OptionParser, Option, get_bool, CLIENT_COMMON_OPTIONS -def get_setup(): - optinfo = CLIENT_COMMON_OPTIONS - optinfo['nouids'] = Option("Do not include UID numbers for users", - default=False, - cmd='--no-uids', - long_arg=True, - cook=get_bool) - optinfo['nogids'] = Option("Do not include GID numbers for groups", - default=False, - cmd='--no-gids', - long_arg=True, - cook=get_bool) - setup = OptionParser(optinfo) - setup.parse(sys.argv[1:]) +class CLI(object): + options = [ + Bcfg2.Options.BooleanOption( + "--no-uids", help="Do not include UID numbers for users"), + Bcfg2.Options.BooleanOption( + "--no-gids", help="Do not include GID numbers for groups")] - if setup['args']: - print("posixuser_[baseline.py takes no arguments, only options") - print(setup.buildHelpMessage()) - raise SystemExit(1) - level = 30 - if setup['verbose']: - level = 20 - if setup['debug']: - level = 0 - Bcfg2.Logger.setup_logging('posixusers_baseline.py', - to_syslog=False, - level=level, - to_file=setup['logging']) - return setup - - -def main(): - setup = get_setup() - if setup['file']: - config = lxml.etree.parse(setup['file']).getroot() - else: + def __init__(self): + Bcfg2.Options.get_parser( + description="Generate a bundle with a baseline of POSIX users and " + "groups", + components=[self, POSIXUsers]).parse() config = lxml.etree.Element("Configuration") - logger = logging.getLogger('posixusers_baseline.py') - users = POSIXUsers(logger, setup, config) - - baseline = lxml.etree.Element("Bundle", name="posixusers_baseline") - for entry in users.FindExtra(): - data = users.existing[entry.tag][entry.get("name")] - for attr, idx in users.attr_mapping[entry.tag].items(): - if (entry.get(attr) or - (attr == 'uid' and setup['nouids']) or - (attr == 'gid' and setup['nogids'])): - continue - entry.set(attr, str(data[idx])) - if entry.tag == 'POSIXUser': - try: - entry.set("group", grp.getgrgid(data[3])[0]) - except KeyError: - logger.warning("User %s is a member of nonexistent group %s" % - (entry.get("name"), data[3])) - entry.set("group", str(data[3])) - for group in users.user_supplementary_groups(entry): - memberof = lxml.etree.SubElement(entry, "MemberOf", - group=group[0]) - - entry.tag = "Bound" + entry.tag - baseline.append(entry) - - print(lxml.etree.tostring(baseline, pretty_print=True)) + self.users = POSIXUsers(config) + self.logger = logging.getLogger('posixusers_baseline.py') + + def run(self): + baseline = lxml.etree.Element("Bundle", name="posixusers_baseline") + for entry in self.users.FindExtra(): + data = self.users.existing[entry.tag][entry.get("name")] + for attr, idx in self.users.attr_mapping[entry.tag].items(): + if (entry.get(attr) or + (attr == 'uid' and Bcfg2.Options.setup.no_uids) or + (attr == 'gid' and Bcfg2.Options.setup.no_gids)): + continue + entry.set(attr, str(data[idx])) + if entry.tag == 'POSIXUser': + try: + entry.set("group", grp.getgrgid(data[3])[0]) + except KeyError: + self.logger.warning( + "User %s is a member of nonexistent group %s" % + (entry.get("name"), data[3])) + entry.set("group", str(data[3])) + for group in self.users.user_supplementary_groups(entry): + lxml.etree.SubElement(entry, "MemberOf", group=group[0]) + + entry.tag = "Bound" + entry.tag + baseline.append(entry) + + print(lxml.etree.tostring(baseline, pretty_print=True)) if __name__ == "__main__": - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py index 507a16f43..ad2a40426 100755 --- a/tools/selinux_baseline.py +++ b/tools/selinux_baseline.py @@ -1,41 +1,18 @@ #!/usr/bin/env python import sys -import logging import lxml.etree - import Bcfg2.Logger import Bcfg2.Options -from Bcfg2.Client.Tools.SELinux import * - -LOGGER = None - -def get_setup(): - global LOGGER - optinfo = Bcfg2.Options.CLIENT_COMMON_OPTIONS - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) +from Bcfg2.Client.Tools.SELinux import SELinux - if setup['args']: - print("selinux_baseline.py takes no arguments, only options") - print(setup.buildHelpMessage()) - raise SystemExit(1) - level = 30 - if setup['verbose']: - level = 20 - if setup['debug']: - level = 0 - Bcfg2.Logger.setup_logging('selinux_base', - to_syslog=False, - level=level, - to_file=setup['logging']) - LOGGER = logging.getLogger('bcfg2') - return setup def main(): - setup = get_setup() + Bcfg2.Options.get_parser( + description="Get a baseline bundle of SELinux entries", + components=[SELinux]).parse() config = lxml.etree.Element("Configuration") - selinux = SELinux(LOGGER, setup, config) + selinux = SELinux(config) baseline = lxml.etree.Element("Bundle", name="selinux_baseline") for etype, handler in selinux.handlers.items(): diff --git a/tools/upgrade/1.1/posixunified.py b/tools/upgrade/1.1/posixunified.py index 8eb4ed734..b6ce7bc90 100644..100755 --- a/tools/upgrade/1.1/posixunified.py +++ b/tools/upgrade/1.1/posixunified.py @@ -17,12 +17,13 @@ NOTE: This script takes a conservative approach when it comes to """ if __name__ == '__main__': - opts = { - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - } - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) - repo = setup['repo'] + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.0-style POSIX entries to 1.1-style " + "unified Path entries") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + repo = Bcfg2.Options.setup.repository unifiedposixrules = "%s/Rules/unified-rules.xml" % repo rulesroot = lxml.etree.Element("Rules") diff --git a/tools/upgrade/1.2/nagiosgen-convert.py b/tools/upgrade/1.2/nagiosgen-convert.py index 2c2142735..eb10cd4ea 100755 --- a/tools/upgrade/1.2/nagiosgen-convert.py +++ b/tools/upgrade/1.2/nagiosgen-convert.py @@ -7,10 +7,13 @@ import lxml.etree import Bcfg2.Options def main(): - opts = {'repo': Bcfg2.Options.SERVER_REPOSITORY} - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) - repo = setup['repo'] + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.1-style Properties-based NagiosGen " + "configuration to standalone 1.2-style") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + repo = Bcfg2.Options.setup.repository oldconfigfile = os.path.join(repo, 'Properties', 'NagiosGen.xml') newconfigpath = os.path.join(repo, 'NagiosGen') newconfigfile = os.path.join(newconfigpath, 'config.xml') @@ -32,11 +35,11 @@ def main(): if host.tag == lxml.etree.Comment: # skip comments continue - + if host.tag == 'default': print("default tag will not be converted; use a suitable Group tag instead") continue - + newhost = lxml.etree.Element("Client", name=host.tag) for opt in host: newopt = lxml.etree.Element("Option", name=opt.tag) diff --git a/tools/upgrade/1.2/packages-convert.py b/tools/upgrade/1.2/packages-convert.py index d65ce90a2..eb1f2f7de 100755 --- a/tools/upgrade/1.2/packages-convert.py +++ b/tools/upgrade/1.2/packages-convert.py @@ -30,10 +30,13 @@ def place_source(xdata, source, groups): return xdata def main(): - opts = {'repo': Bcfg2.Options.SERVER_REPOSITORY} - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) - repo = setup['repo'] + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.1-style Packages configuration to " + "1.2-style") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + repo = Bcfg2.Options.setup.repository configpath = os.path.join(repo, 'Packages') oldconfigfile = os.path.join(configpath, 'config.xml') newconfigfile = os.path.join(configpath, 'packages.conf') @@ -78,7 +81,7 @@ def main(): if el.tag == lxml.etree.Comment or el.tag == 'Config': # skip comments and Config continue - + if el.tag == XI + 'include': oldsources.append(os.path.join(configpath, el.get('href'))) newsource.append(el) @@ -98,7 +101,7 @@ def main(): newel.set(tag.lower(), el.find(tag).text) except AttributeError: pass - + for child in el.getchildren(): if child.tag in ['Component', 'Blacklist', 'Whitelist', 'Arch']: newel.append(child) diff --git a/tools/upgrade/1.3/migrate_configs.py b/tools/upgrade/1.3/migrate_configs.py index 76b2392e7..547ce61e4 100755 --- a/tools/upgrade/1.3/migrate_configs.py +++ b/tools/upgrade/1.3/migrate_configs.py @@ -5,6 +5,7 @@ import sys from Bcfg2.Compat import ConfigParser import Bcfg2.Options + def copy_section(src_file, tgt_cfg, section, newsection=None): if newsection is None: newsection = section @@ -16,13 +17,13 @@ def copy_section(src_file, tgt_cfg, section, newsection=None): tgt_cfg.add_section(newsection) except ConfigParser.DuplicateSectionError: print("[%s] section already exists in %s, adding options" % - (newsection, setup['configfile'])) + (newsection, Bcfg2.Options.setup.config)) for opt in cfg.options(section): val = cfg.get(section, opt) if tgt_cfg.has_option(newsection, opt): print("%s in [%s] already populated in %s, skipping" % - (opt, newsection, setup['configfile'])) - print(" %s: %s" % (setup['configfile'], + (opt, newsection, Bcfg2.Options.setup.config)) + print(" %s: %s" % (Bcfg2.Options.setup.config, tgt_cfg.get(newsection, opt))) print(" %s: %s" % (src_file, val)) else: @@ -30,47 +31,50 @@ def copy_section(src_file, tgt_cfg, section, newsection=None): tgt_cfg.set(newsection, opt, val) def main(): - opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY, - configfile=Bcfg2.Options.CFILE) - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.2 per-plugin config files to 1.3 " + "unified config file") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + repo = Bcfg2.Options.setup.repository + cfp = ConfigParser.ConfigParser() + cfp.read(Bcfg2.Options.setup.config) # files that you should remove manually remove = [] # move rules config out of rules.conf and into bcfg2.conf - rules_conf = os.path.join(setup['repo'], 'Rules', 'rules.conf') + rules_conf = os.path.join(repo, 'Rules', 'rules.conf') if os.path.exists(rules_conf): remove.append(rules_conf) - copy_section(rules_conf, setup.cfp, "rules") + copy_section(rules_conf, cfp, "rules") # move packages config out of packages.conf and into bcfg2.conf - pkgs_conf = os.path.join(setup['repo'], 'Packages', 'packages.conf') + pkgs_conf = os.path.join(repo, 'Packages', 'packages.conf') if os.path.exists(pkgs_conf): remove.append(pkgs_conf) - copy_section(pkgs_conf, setup.cfp, "global", newsection="packages") + copy_section(pkgs_conf, cfp, "global", newsection="packages") for section in ["apt", "yum", "pulp"]: - copy_section(pkgs_conf, setup.cfp, section, + copy_section(pkgs_conf, cfp, section, newsection="packages:" + section) # move reports database config into [database] section - if setup.cfp.has_section("statistics"): - if not setup.cfp.has_section("database"): - setup.cfp.add_section("database") - for opt in setup.cfp.options("statistics"): + if cfp.has_section("statistics"): + if not cfp.has_section("database"): + cfp.add_section("database") + for opt in cfp.options("statistics"): if opt.startswith("database_"): newopt = opt[9:] - if setup.cfp.has_option("database", newopt): + if cfp.has_option("database", newopt): print("%s in [database] already populated, skipping" % newopt) else: - setup.cfp.set("database", newopt, - setup.cfp.get("statistics", opt)) - setup.cfp.remove_option("statistics", opt) + cfp.set("database", newopt, cfp.get("statistics", opt)) + cfp.remove_option("statistics", opt) - print("Writing %s" % setup['configfile']) + print("Writing %s" % Bcfg2.Options.setup.config) try: - setup.cfp.write(open(setup['configfile'], "w")) + cfp.write(open(Bcfg2.Options.setup.config, "w")) if len(remove): print("Settings were migrated, but you must remove these files " "manually:") @@ -78,7 +82,7 @@ def main(): print(" %s" % path) except IOError: err = sys.exc_info()[1] - print("Could not write %s: %s" % (setup['configfile'], err)) + print("Could not write %s: %s" % (Bcfg2.Options.setup.config, err)) if __name__ == '__main__': sys.exit(main()) diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py index 34430e3df..de8aef973 100755 --- a/tools/upgrade/1.3/migrate_dbstats.py +++ b/tools/upgrade/1.3/migrate_dbstats.py @@ -2,17 +2,14 @@ import os os.environ['BCFG2_LEGACY_MODELS'] = '1' -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' import sys import logging import time import Bcfg2.Logger import Bcfg2.Options -from django.core.cache import cache -from django.db import connection, backend - -from Bcfg2.Server.Admin.Reports import Reports +from django.db import connection, transaction, backend +from Bcfg2.Server.Admin import UpdateReports from Bcfg2.Reporting import models as new_models from Bcfg2.Reporting.utils import BatchFetch from Bcfg2.Reporting.Compat import transaction @@ -282,17 +279,10 @@ def _restructure(): if __name__ == '__main__': - Bcfg2.Logger.setup_logging('bcfg2-report-collector', - to_console=logging.INFO, - level=logging.INFO) - - optinfo = dict() - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - #sync! - Reports(setup).__call__(['update']) + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.2 DBStats plugin to 1.3 Reporting " + "subsystem", + components=[UpdateReports]) + UpdateReports().run(Bcfg2.Options.setup) _restructure() diff --git a/tools/upgrade/1.3/migrate_info.py b/tools/upgrade/1.3/migrate_info.py index 3ccbf0285..7f3bb9a29 100755 --- a/tools/upgrade/1.3/migrate_info.py +++ b/tools/upgrade/1.3/migrate_info.py @@ -5,7 +5,16 @@ import re import sys import lxml.etree import Bcfg2.Options -from Bcfg2.Server.Plugin import INFO_REGEX + +INFO_REGEX = re.compile(r'owner:\s*(?P<owner>\S+)|' + + r'group:\s*(?P<group>\S+)|' + + r'mode:\s*(?P<mode>\w+)|' + + r'secontext:\s*(?P<secontext>\S+)|' + + r'paranoid:\s*(?P<paranoid>\S+)|' + + r'sensitive:\s*(?P<sensitive>\S+)|' + + r'encoding:\s*(?P<encoding>\S+)|' + + r'important:\s*(?P<important>\S+)|' + + r'mtime:\s*(?P<mtime>\w+)') PERMS_REGEX = re.compile(r'perms:\s*(?P<perms>\w+)') @@ -32,16 +41,17 @@ def convert(info_file): def main(): - opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY, - configfile=Bcfg2.Options.CFILE, - plugins=Bcfg2.Options.SERVER_PLUGINS) - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.2 info/:info files to 1.3 info.xml") + parser.add_options([Bcfg2.Options.Common.repository, + Bcfg2.Options.Common.plugins]) + parser.parse() - for plugin in setup['plugins']: + for plugin in Bcfg2.Options.setup.plugins: if plugin not in ['SSLCA', 'Cfg', 'TGenshi', 'TCheetah', 'SSHbase']: continue - for root, dirs, files in os.walk(os.path.join(setup['repo'], plugin)): + datastore = os.path.join(Bcfg2.Options.setup.repository, plugin) + for root, dirs, files in os.walk(datastore): for fname in files: if fname in [":info", "info"]: convert(os.path.join(root, fname)) diff --git a/tools/upgrade/1.3/migrate_perms_to_mode.py b/tools/upgrade/1.3/migrate_perms_to_mode.py index ee440bc8e..2dfb70388 100755 --- a/tools/upgrade/1.3/migrate_perms_to_mode.py +++ b/tools/upgrade/1.3/migrate_perms_to_mode.py @@ -4,13 +4,14 @@ import lxml.etree import os import sys from fnmatch import fnmatch -from Bcfg2.Compat import any +from Bcfg2.Compat import any # pylint: disable=W0622 +from Bcfg2.Server.FileMonitor import FileMonitor import Bcfg2.Options def setmodeattr(elem): """Set the mode attribute for a given element.""" - if elem.attrib.has_key('perms'): + if 'perms' in elem.attrib: elem.set('mode', elem.get('perms')) del elem.attrib['perms'] return True @@ -54,33 +55,34 @@ def convertstructure(structfile): writefile(structfile, xdata) -def skip_path(path, setup): +def skip_path(path): return any(fnmatch(path, p) or fnmatch(os.path.basename(path), p) - for p in setup['ignore']) + for p in Bcfg2.Options.setup.ignore_files) def main(): - opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY, - configfile=Bcfg2.Options.CFILE, - ignore=Bcfg2.Options.SERVER_FAM_IGNORE, - plugins=Bcfg2.Options.SERVER_PLUGINS) - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) - repo = setup['repo'] + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.2 'perms' attribute to 1.3 'mode' " + "attribute", + components=FileMonitor) + parser.add_options([Bcfg2.Options.Common.repository, + Bcfg2.Options.Common.plugins]) + parser.parse() + repo = Bcfg2.Options.setup.repository - for plugin in setup['plugins']: + for plugin in Bcfg2.Options.setup.plugins: if plugin in ['Base', 'Bundler', 'Rules']: - for root, dirs, files in os.walk(os.path.join(repo, plugin)): - if skip_path(root, setup): + for root, _, files in os.walk(os.path.join(repo, plugin)): + if skip_path(root): continue for fname in files: - if skip_path(fname, setup): + if skip_path(fname): continue convertstructure(os.path.join(root, fname)) if plugin not in ['Cfg', 'TGenshi', 'TCheetah', 'SSHbase', 'SSLCA']: continue for root, dirs, files in os.walk(os.path.join(repo, plugin)): - if skip_path(root, setup): + if skip_path(root): continue for fname in files: if fname == 'info.xml': diff --git a/tools/upgrade/1.3/migrate_probe_groups_to_db.py b/tools/upgrade/1.3/migrate_probe_groups_to_db.py index 73339e787..f9abbf982 100755 --- a/tools/upgrade/1.3/migrate_probe_groups_to_db.py +++ b/tools/upgrade/1.3/migrate_probe_groups_to_db.py @@ -4,16 +4,13 @@ and Probe plugins. Does not migrate individual probe return data. Assumes migration to BOTH Metadata and Probe to database backends. """ import os -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' - -import lxml.etree import sys +import lxml.etree import Bcfg2.Options +import Bcfg2.DBSettings -from Bcfg2.Server.Plugins.Metadata import MetadataClientModel -from Bcfg2.Server.Plugins.Probes import ProbesGroupsModel -def migrate(xclient): +def migrate(xclient, MetadataClientModel, ProbesGroupsModel): """ Helper to do the migration given a <Client/> XML element """ client_name = xclient.get('name') try: @@ -32,9 +29,11 @@ def migrate(xclient): group_name = xgroup.get('name') cgroups.append(group_name) try: - group = ProbesGroupsModel.objects.get(hostname=client_name, group=group_name) + group = ProbesGroupsModel.objects.get(hostname=client_name, + group=group_name) except ProbesGroupsModel.DoesNotExist: - group = ProbesGroupsModel(hostname=client_name, group=group_name) + group = ProbesGroupsModel(hostname=client_name, + group=group_name) group.save() ProbesGroupsModel.objects.filter( @@ -46,6 +45,7 @@ def migrate(xclient): return False return True + def main(): """ Main """ opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY) @@ -59,10 +59,15 @@ def main(): except lxml.etree.XMLSyntaxError: err = sys.exc_info()[1] print("Could not parse %s, skipping: %s" % (probefile, err)) - + + # these must be loaded after option parsing is complete + from Bcfg2.Server.Plugins.Metadata import MetadataClientModel + from Bcfg2.Server.Plugins.Probes import ProbesGroupsModel + for xclient in xdata.findall('Client'): - print "Migrating Metadata and Probe groups for %s" % xclient.get('name') - migrate(xclient) + print("Migrating Metadata and Probe groups for %s" % + xclient.get('name')) + migrate(xclient, MetadataClientModel, ProbesGroupsModel) if __name__ == '__main__': sys.exit(main()) diff --git a/tools/upgrade/1.3/service_modes.py b/tools/upgrade/1.3/service_modes.py index 0c458c3a9..d8e3c9e6f 100755 --- a/tools/upgrade/1.3/service_modes.py +++ b/tools/upgrade/1.3/service_modes.py @@ -6,14 +6,18 @@ import glob import lxml.etree import Bcfg2.Options + def main(): - opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY) - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.2 Service modes to 1.3-style " + "granular Service specification") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() files = [] for plugin in ['Bundler', 'Rules', 'Default']: - files.extend(glob.glob(os.path.join(setup['repo'], plugin, "*"))) + files.extend(glob.glob(os.path.join(Bcfg2.Options.setup.repository, + plugin, "*"))) for bfile in files: bdata = lxml.etree.parse(bfile) diff --git a/tools/upgrade/1.4/README b/tools/upgrade/1.4/README new file mode 100644 index 000000000..b03cb9b74 --- /dev/null +++ b/tools/upgrade/1.4/README @@ -0,0 +1,14 @@ +This directory contains scripts to help with upgrading from Bcfg2 1.3 +to 1.4. + +migrate_decisions.py + - Convert old group- and host-specific whitelist and blacklist + files into structured XML + +convert_bundles.py + - Remove deprecated explicit bundle names, rename .genshi bundles + to .xml + +migrate_sslca.py + - Migrate from the standalone SSLCA plugin to the built-in SSL + certificate generation abilities of the Cfg plugin
\ No newline at end of file diff --git a/tools/upgrade/1.4/convert_bundles.py b/tools/upgrade/1.4/convert_bundles.py new file mode 100755 index 000000000..b9cb483f2 --- /dev/null +++ b/tools/upgrade/1.4/convert_bundles.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import os +import sys +import lxml.etree +import Bcfg2.Options + + +def main(): + parser = Bcfg2.Options.get_parser("Tool to remove bundle names") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + bundler_dir = os.path.join(Bcfg2.Options.setup.repository, "Bundler") + if os.path.exists(bundler_dir): + for root, _, files in os.walk(bundler_dir): + for fname in files: + bpath = os.path.join(root, fname) + newpath = bpath + if newpath.endswith(".genshi"): + newpath = newpath[:-6] + "xml" + print("Converting %s to %s" % (bpath, newpath)) + else: + print("Converting %s" % bpath) + xroot = lxml.etree.parse(bpath) + xdata = xroot.getroot() + if 'name' in xdata.attrib: + del xdata.attrib['name'] + xroot.write(bpath) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/upgrade/1.4/migrate_decisions.py b/tools/upgrade/1.4/migrate_decisions.py new file mode 100755 index 000000000..d0915f202 --- /dev/null +++ b/tools/upgrade/1.4/migrate_decisions.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +import os +import re +import sys +import glob +import lxml.etree +import Bcfg2.Options + + +SPECIFIC = re.compile(r'.*\/(white|black)list' + r'(\.(H_(?P<host>.*)|G\d+_(?P<group>.*)))?$') + + +def convert(files, xdata): + hosts = [] + groups = [] + for oldfile in files: + spec = SPECIFIC.match(oldfile) + if spec and spec.group('host'): + hosts.append(spec.group('host')) + elif spec and spec.group('group'): + groups.append(spec.group('group')) + + for oldfile in files: + print("Converting %s" % oldfile) + spec = SPECIFIC.match(oldfile) + if not spec: + print("Skipping unknown file %s" % oldfile) + continue + + parent = xdata + if spec.group('host'): + for host in hosts: + if host != spec.group('host'): + parent = lxml.etree.SubElement(parent, "Client", + name=host, negate="true") + parent = lxml.etree.SubElement(parent, "Client", + name=spec.group('host')) + elif spec.group('group'): + for host in hosts: + parent = lxml.etree.SubElement(parent, "Client", + name=host, negate="true") + for group in groups: + if group != spec.group('group'): + parent = lxml.etree.SubElement(parent, "Group", + name=group, negate="true") + parent = lxml.etree.SubElement(parent, "Group", + name=spec.group('group')) + parent.append(lxml.etree.Comment("Converted from %s" % oldfile)) + olddata = lxml.etree.parse(oldfile, parser=Bcfg2.Server.XMLParser) + for decision in olddata.xpath('//Decision'): + parent.append(decision) + return xdata + + +def main(): + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.3 Decisions list format to 1.4 " + "format") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + datadir = os.path.join(Bcfg2.Options.setup.repository, 'Decisions') + whitelist = lxml.etree.Element("Decisions") + blacklist = lxml.etree.Element("Decisions") + if os.path.exists(datadir): + convert(glob.glob(os.path.join(datadir, 'whitelist*')), + whitelist) + convert(glob.glob(os.path.join(datadir, 'blacklist*')), + blacklist) + + print("Writing %s" % os.path.join(datadir, "whitelist.xml")) + open(os.path.join(datadir, "whitelist.xml"), + 'w').write(lxml.etree.tostring(whitelist, pretty_print=True)) + print("Writing %s" % os.path.join(datadir, "blacklist.xml")) + open(os.path.join(datadir, "blacklist.xml"), + 'w').write(lxml.etree.tostring(blacklist, pretty_print=True)) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/upgrade/1.4/migrate_sslca.py b/tools/upgrade/1.4/migrate_sslca.py new file mode 100755 index 000000000..958228c86 --- /dev/null +++ b/tools/upgrade/1.4/migrate_sslca.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +import os +import sys +import shutil +import Bcfg2.Options + + +def main(): + parser = Bcfg2.Options.get_parser( + description="Migrate from the SSLCA plugin to built-in Cfg SSL cert " + "generation") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + sslcadir = os.path.join(Bcfg2.Options.setup.repository, 'SSLCA') + cfgdir = os.path.join(Bcfg2.Options.setup.repository, 'Cfg') + for root, _, files in os.walk(sslcadir): + if not files: + continue + newpath = cfgdir + root[len(sslcadir):] + if not os.path.exists(newpath): + print("Creating %s and copying contents from %s" % (newpath, root)) + shutil.copytree(root, newpath) + else: + print("Copying contents from %s to %s" % (root, newpath)) + for fname in files: + newfpath = os.path.exists(os.path.join(newpath, fname)) + if newfpath: + print("%s already exists, skipping" % newfpath) + else: + shutil.copy(os.path.join(root, fname), newpath) + cert = os.path.join(newpath, "cert.xml") + newcert = os.path.join(newpath, "sslcert.xml") + key = os.path.join(newpath, "key.xml") + newkey = os.path.join(newpath, "sslkey.xml") + if os.path.exists(cert): + os.rename(cert, newcert) + if os.path.exists(key): + os.rename(key, newkey) + + +if __name__ == '__main__': + sys.exit(main()) |