diff options
181 files changed, 4668 insertions, 1286 deletions
diff --git a/.travis.yml b/.travis.yml index 73b8a9594..54f2215de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,17 @@ language: python python: - - "2.5" - "2.6" - "2.7" env: - WITH_OPTIONAL_DEPS=yes - WITH_OPTIONAL_DEPS=no -before_install: - - testsuite/before_install.sh install: - testsuite/install.sh - pip install --use-mirrors -e . script: - nosetests testsuite +after_failure: + - pip freeze branches: except: - maint-1.2 @@ -157,7 +157,8 @@ add themselves to this file. See LICENSE for the full license. - Zach Lowry <zach@mcs.anl.gov> wrote Solaris support and general hardening. -- Michael Fenn <fennm@deshawresearch.com> fixed various small bugs +- Michael Fenn <fennm@deshawresearch.com> implemented the database router + for separately storing the reporting database and fixed various small bugs related to bcfg2 on CentOS 5 - Alexander Sulfrian <alexander@sulfrian.net> fixed various bugs. @@ -21,11 +21,11 @@ Installation ------------ For details about the installation of Bcfg2 please refer to the -following pages in the Bcfg2 wiki. +following pages in the Bcfg2 online documentation: -* Prerequisites: http://bcfg2.org/wiki/Prereqs -* Download: http://bcfg2.org/wiki/Download -* Installation: http://bcfg2.org/wiki/Install +* Prerequisites: http://docs.bcfg2.org/installation/prerequisites.html +* Download: http://bcfg2.org/download/ +* Installation: http://docs.bcfg2.org/installation/index.html Need help --------- diff --git a/debian/bcfg2.install b/debian/bcfg2.install index fc43c3ecb..a240dac53 100644 --- a/debian/bcfg2.install +++ b/debian/bcfg2.install @@ -1,6 +1,7 @@ debian/tmp/usr/bin/bcfg2 usr/sbin debian/tmp/usr/lib/python*/*-packages/Bcfg2/*.py debian/tmp/usr/lib/python*/*-packages/Bcfg2/Client/* +debian/tmp/usr/lib/python*/*-packages/Bcfg2/Options/* debian/tmp/usr/share/man/man1/* debian/tmp/usr/share/man/man5/* examples/bcfg2.conf usr/share/bcfg2 diff --git a/debian/changelog b/debian/changelog index 7f6e2f637..e30fba546 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +bcfg2 (1.4.0pre1-0.0) unstable; urgency=low + + * New upstream release + + -- Sol Jerome <sol.jerome@gmail.com> Mon, 16 Jun 2014 09:36:13 -0500 + +bcfg2 (1.3.4-0.0) unstable; urgency=low + + * New upstream release + + -- Sol Jerome <sol.jerome@gmail.com> Tue, 25 Feb 2014 13:25:16 -0600 + bcfg2 (1.3.3-0.0) unstable; urgency=low * New upstream release diff --git a/debian/control b/debian/control index 1b1ca8f61..8eb4343c7 100644 --- a/debian/control +++ b/debian/control @@ -33,9 +33,9 @@ Description: Configuration management client Package: bcfg2-server Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, python-lxml (>= 0.9), libxml2-utils (>= 2.6.23), lsb-base (>= 3.1-9), ucf, bcfg2 (= ${binary:Version}), openssl, python (>= 2.6), python-pyinotify | python-gamin, python-daemon +Depends: ${python:Depends}, ${misc:Depends}, python-lxml (>= 0.9), libxml2-utils (>= 2.6.23), lsb-base (>= 3.1-9), ucf, bcfg2 (= ${binary:Version}), openssl, python (>= 2.6), python-pyinotify | python-gamin, python-daemon, python-genshi (>= 0.4.4) Recommends: graphviz, patch -Suggests: python-cheetah, python-genshi (>= 0.4.4), python-profiler, python-django, mail-transport-agent, bcfg2-doc (= ${binary:Version}) +Suggests: python-cheetah, python-profiler, python-django, mail-transport-agent, bcfg2-doc (= ${binary:Version}) Description: Configuration management server Bcfg2 is a configuration management system that generates configuration sets for clients bound by client profiles. diff --git a/doc/appendix/guides/authentication.txt b/doc/appendix/guides/authentication.txt index b8ec82590..93a34c9bc 100644 --- a/doc/appendix/guides/authentication.txt +++ b/doc/appendix/guides/authentication.txt @@ -37,7 +37,6 @@ This is a :ref:`Cheetah template per-client bcfg2.conf from the per-client metadata:: [communication] - protocol = xmlrpc/ssl #if $self.metadata.uuid != None user = $self.metadata.uuid #end if diff --git a/doc/appendix/guides/centos.txt b/doc/appendix/guides/centos.txt index 3a35627a8..44ee08777 100644 --- a/doc/appendix/guides/centos.txt +++ b/doc/appendix/guides/centos.txt @@ -132,7 +132,6 @@ upon connection:: [communication] - protocol = xmlrpc/ssl password = N41lMNeW ca = /etc/bcfg2.crt @@ -230,10 +229,11 @@ should look something like this When editing your xml files by hand, it is useful to occasionally run `bcfg2-lint` to ensure that your xml validates properly. -The final thing we need is for the client to have the proper -arch group membership. For this, we will make use of the -:ref:`unsorted-dynamic_groups` capabilities of the Probes plugin. Add -Probes to your plugins line in ``bcfg2.conf`` and create the Probe.:: +The final thing we need is for the client to have the proper arch +group membership. For this, we will make use of the +:ref:`server-plugins-probes-dynamic-groups` capabilities of the Probes +plugin. Add Probes to your plugins line in ``bcfg2.conf`` and create +the Probe.:: [root@centos ~]# grep plugins /etc/bcfg2.conf plugins = Bundler,Cfg,...,Probes diff --git a/doc/appendix/guides/import-existing-ssh-keys.txt b/doc/appendix/guides/import-existing-ssh-keys.txt index 6ce41ba60..4e2282044 100644 --- a/doc/appendix/guides/import-existing-ssh-keys.txt +++ b/doc/appendix/guides/import-existing-ssh-keys.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _appendix-guides-import-existing-ssh-keys: @@ -21,11 +22,24 @@ Add a bundle for ssh After verifying that SSHbase is listed on the plugins line in ``/etc/bcfg2.conf``, you need to create a bundle containing the -appropriate entries. +appropriate entries. In general, you can use a path glob: .. code-block:: xml <Bundle> + <Path glob="/etc/ssh/*"/> + </Bundle> + +If you need more granular control -- e.g., other entries in +``/etc/ssh`` are specified in other bundles -- you can also list the +files explicity: + +.. code-block:: xml + + <Bundle> + <!-- requires a version of openssh that can generate ecdsa keys --> + <Path name="/etc/ssh/ssh_host_ecdsa_key"/> + <Path name="/etc/ssh/ssh_host_ecdsa_key.pub"/> <Path name='/etc/ssh/ssh_host_dsa_key'/> <Path name='/etc/ssh/ssh_host_rsa_key'/> <Path name='/etc/ssh/ssh_host_dsa_key.pub'/> @@ -83,7 +97,7 @@ Now, we pull the ssh host key data for the client out of the uploaded stats and insert it as host-specific copies of these files in ``/var/lib/bcfg2/SSHBase``.:: - for key in ssh_host_rsa_key ssh_host_dsa_key ssh_host_key; do + for key in ssh_host_ecdsa_key ssh_host_rsa_key ssh_host_dsa_key ssh_host_key; do sudo bcfg2-admin pull <clientname> Path /etc/ssh/$key sudo bcfg2-admin pull <clientname> Path /etc/ssh/$key.pub done diff --git a/doc/appendix/guides/sslca_howto.txt b/doc/appendix/guides/sslca_howto.txt index 9c939dcd3..8ee0b2b42 100644 --- a/doc/appendix/guides/sslca_howto.txt +++ b/doc/appendix/guides/sslca_howto.txt @@ -148,7 +148,6 @@ Here's what a functional **[communication]** section in a ``bcfg2.conf`` genshi template for clients might look like.:: [communication] - protocol = xmlrpc/ssl {% if metadata.uuid != None %}\ user = ${metadata.uuid} {% end %}\ diff --git a/doc/appendix/guides/ubuntu.txt b/doc/appendix/guides/ubuntu.txt index 60f8e3a41..24bebf023 100644 --- a/doc/appendix/guides/ubuntu.txt +++ b/doc/appendix/guides/ubuntu.txt @@ -172,7 +172,6 @@ Replace Pkgmgr with Packages in the plugins line of ``bcfg2.conf``:: transport = LocalFilesystem [communication] - protocol = xmlrpc/ssl password = secret certificate = /etc/ssl/bcfg2.crt key = /etc/ssl/bcfg2.key @@ -253,10 +252,11 @@ that our client is able to obtain these sources. When editing your xml files by hand, it is useful to occasionally run ``bcfg2-lint -v`` to ensure that your xml validates properly. -The last thing we need is for the client to have the proper -arch group membership. For this, we will make use of the -:ref:`unsorted-dynamic_groups` capabilities of the Probes plugin. Add -Probes to your plugins line in ``bcfg2.conf`` and create the Probe. +The last thing we need is for the client to have the proper arch group +membership. For this, we will make use of the +:ref:`server-plugins-probes-dynamic-groups` capabilities of the Probes +plugin. Add Probes to your plugins line in ``bcfg2.conf`` and create +the Probe. .. code-block:: sh diff --git a/doc/client/metadata.txt b/doc/client/metadata.txt index 27870ba9a..0dec5e3a7 100644 --- a/doc/client/metadata.txt +++ b/doc/client/metadata.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _client-metadata: @@ -24,12 +25,12 @@ interaction: This construction process spans several server plugins. The :ref:`server-plugins-grouping-metadata` is responsible for initial instance creation, including the client hostname, -profile, and basic group memberships. After this initial creation, -Connector plugins (such as :ref:`server-plugins-probes-index` or -:ref:`server-plugins-connectors-properties`) can add additional group -memberships for clients. These memberships are merged into the instance; -that is, the new group memberships are treated as if they were included -in groups.xml. If any of these groups are defined in groups.xml, +profile, and basic group memberships. After this initial +creation, Connector plugins (such as :ref:`server-plugins-probes` +or :ref:`server-plugins-connectors-properties`) can add additional +group memberships for clients. These memberships are merged into the +instance; that is, the new group memberships are treated as if they were +included in groups.xml. If any of these groups are defined in groups.xml, then groups included there are included in the ClientMetadata instance group list. At the end of this process, the ClientMetadata instance has its complete set of group memberships. At this point, each connector diff --git a/doc/client/tools/augeas.txt b/doc/client/tools/augeas.txt new file mode 100644 index 000000000..6fed5f5ce --- /dev/null +++ b/doc/client/tools/augeas.txt @@ -0,0 +1,95 @@ +.. -*- mode: rst -*- + +.. _client-tools-augeas: + +======== + Augeas +======== + +The Augeas tool provides a way to use `Augeas +<http://www.augeas.net>`_ to edit files that may not be completely +managed. + +In the simplest case, you simply tell Augeas which path to edit, and +give it a sequence of commands: + +.. code-block:: xml + + <Path type="augeas" name="/etc/hosts" owner="root" group="root" + mode="0644"> + <Set path="01/ipaddr" value="192.168.0.1"/> + <Set path="01/canonical" value="pigiron.example.com"/> + <Set path="01/alias[1]" value="pigiron"/> + <Set path="01/alias[2]" value="piggy"/> + </Path> + +The commands are run in document order. There's no need to do an +explicit ``save`` at the end. + +These commands will be run if any of the paths do not already +have the given setting. In other words, if any command has not +already been run, they will all be run. + +So, if the first host already has all of the specified settings, then +that Path will verify successfully and nothing will be changed. But +suppose the first host looks like this:: + + 192.168.0.1 pigiron.example.com pigiron + +All that is missing is the second alias, ``piggy``. The entire Augeas +script will be run in this case. It's important, then, to ensure that +all commands you use are idempotent. (For instance, the ``Move`` and +``Insert`` commands are unlikely to be useful.) + +The Augeas paths are all relative to ``/files/etc/hosts``. + +The Augeas tool understands a subset of ``augtool`` commands. Valid +tags are: ``Remove``, ``Move``, ``Set``, ``Clear``, ``SetMulti``, and +``Insert``. Refer to the official Augeas docs or the `Schema`_ below +for details on the commands. + +The Augeas tool also supports one additional directive, ``Initial``, +for setting initial file content when a file does not exist. For +instance, the ``Xml`` lens fails to parse a file that does not exist, +and, as a result, you cannot add content to it. You can use +``Initial`` to circumvent this issue: + +.. code-block:: xml + + <Path type="augeas" name="/etc/test.xml" lens="Xml" + owner="root" group="root" mode="0640"> + <Initial><Test/></Initial> + <Set path="Test/#text" value="text content"/> + </Path> + +Editing files outside the default load path +=========================================== + +If you're using Augeas to edit files outside of its default load path, +you must manually specify the lens. For instance: + +.. code-block:: xml + + <Path type="augeas" name="/opt/jenkins/home/config.xml" lens="Xml" + owner="jenkins" group="jenkins" mode="0640"> + <Set path="hudson/systemMessage/#text" + value="This is a Jenkins server."/> + </Path> + +Note that there's no need to manually modify the load path by setting +``/augeas/load/<lens>/incl``, nor do you have to call ``load`` +explicitly. + +Schema +====== + +.. xml:group:: augeasCommands + + +Performance +=========== + +The Augeas tool is quite slow to initialize. For each ``<Path +type="augeas" ... >`` entry you have, it creates a new Augeas object +internally, which can take several seconds. It's thus important to +use this tool sparingly. diff --git a/doc/conf.py b/doc/conf.py index 0e4009cd3..1da6b3b01 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -64,9 +64,9 @@ else: # built documents. # # The short X.Y version. -version = '1.3' +version = '1.4' # The full version, including alpha/beta/rc tags. -release = '1.3.3' +release = '1.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/contents.txt b/doc/contents.txt index 8220d0d1d..e7df568f9 100644 --- a/doc/contents.txt +++ b/doc/contents.txt @@ -21,6 +21,7 @@ Bcfg2 documentation |release| glossary appendix/index man/index + releases/index unsorted/index diff --git a/doc/development/caching.txt b/doc/development/caching.txt index 47d627278..83ec0290f 100644 --- a/doc/development/caching.txt +++ b/doc/development/caching.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _development-cache: @@ -44,10 +45,10 @@ Currently known caches are: | Metadata | Hostname | :class:`ClientMetadata | The :ref:`Metadata cache <server-caching>` | | | | <Bcfg2.Server.Plugins.Metadata.ClientMetadata>` | | +-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ -| Probes, | Hostname | ``list`` of group names | Groups set by :ref:`server-plugins-probes-index` | +| Probes, | Hostname | ``list`` of group names | Groups set by :ref:`server-plugins-probes` | | probegroups | | | | +-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ -| Probes, | Hostname | ``dict`` of ``<probe name>``: | Other data set by :ref:`server-plugins-probes-index` | +| Probes, | Hostname | ``dict`` of ``<probe name>``: | Other data set by :ref:`server-plugins-probes` | | probedata | | :class:`ProbeData | | | | | <Bcfg2.Server.Plugins.Probes.ProbeData>` | | +-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ diff --git a/doc/development/cfg.txt b/doc/development/cfg.txt index f93bb42c7..4e967368b 100644 --- a/doc/development/cfg.txt +++ b/doc/development/cfg.txt @@ -64,9 +64,11 @@ Generators .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator.CfgPlaintextGenerator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.CfgGenshiGenerator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.CfgCheetahGenerator +.. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.CfgJinja2Generator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator.CfgEncryptedGenerator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator.CfgEncryptedGenshiGenerator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator.CfgEncryptedCheetahGenerator +.. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator.CfgEncryptedJinja2Generator .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.CfgAuthorizedKeysGenerator Creators diff --git a/doc/development/plugins.txt b/doc/development/plugins.txt index e4f16b84d..5993c4e29 100644 --- a/doc/development/plugins.txt +++ b/doc/development/plugins.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _development-plugins: @@ -147,9 +148,8 @@ For examples, see: * :func:`Bcfg2.Server.Plugins.Probes.ReceiveData` takes a copy of the groups that have been assigned to a client by - :ref:`server-plugins-probes-index`, and if that data changes when - new probe data is received, it invalidates the cache for that - client. + :ref:`server-plugins-probes`, and if that data changes when new probe + data is received, it invalidates the cache for that client. * :func:`Bcfg2.Server.Plugins.GroupPatterns.Index` expires the entire cache whenever a FAM event is received for the :ref:`server-plugins-grouping-grouppatterns` config file. diff --git a/doc/development/testing.txt b/doc/development/testing.txt index f00193574..b6db98cca 100644 --- a/doc/development/testing.txt +++ b/doc/development/testing.txt @@ -69,8 +69,8 @@ Server Testing Entry: fs13.bgl.mcs.anl.gov.xml Entry: fs13.bgl.mcs.anl.gov.xml good Entry: login1.bgl.mcs.anl.gov.xml - ConfigFile /bin/whatami contents differ - ConfigFile /bin/whatami differs (in bundle softenv) + Path /bin/whatami contents differ + Path /bin/whatami differs (in bundle softenv) Entry: login1.bgl.mcs.anl.gov.xml bad This can be used to compare configurations for single clients, or diff --git a/doc/exts/xmlschema.py b/doc/exts/xmlschema.py index c26aed81e..89104c2a6 100644 --- a/doc/exts/xmlschema.py +++ b/doc/exts/xmlschema.py @@ -784,7 +784,7 @@ class XMLDomain(Domain): def clear_doc(self, docname): to_del = [] for dtype in self.types.keys(): - for key, (doc, _) in self.data[dtype].iteritems(): + for key, (doc, _) in self.data[dtype].items(): if doc == docname: to_del.append((dtype, key)) for dtype, key in to_del: @@ -803,7 +803,7 @@ class XMLDomain(Domain): def get_objects(self): for dtype in self.types.keys(): - for name, (docname, tgtid) in self.data[dtype].iteritems(): + for name, (docname, tgtid) in self.data[dtype].items(): yield (name, name, dtype, docname, tgtid, self.object_types[dtype].attrs['searchprio']) diff --git a/doc/getting_started/index.txt b/doc/getting_started/index.txt index 9b69bf65a..f619447e2 100644 --- a/doc/getting_started/index.txt +++ b/doc/getting_started/index.txt @@ -223,7 +223,7 @@ you will find that we now have a correct entry:: Done! Now we just have 242 (or more) entries to take care of! -:ref:`server-plugins-structures-bundler-index` is a +:ref:`server-plugins-structures-bundler` is a relatively easy directory to populate. You can find many samples of Bundles in the :ref:`Bundler Example Repository <server-plugins-structures-bundler-index-examples>`, many of which can @@ -255,6 +255,10 @@ Once you have the server setup, you may be interested in Platform-specific Quickstart Notes ================================== -* :ref:`appendix-guides-centos` -* :ref:`appendix-guides-ubuntu` -* :ref:`getting_started-macosx-notes` +.. toctree:: + :maxdepth: 1 + + CentOS </appendix/guides/centos> + Ubuntu </appendix/guides/ubuntu> + Gentoo </appendix/guides/gentoo> + Mac OS X <macosx/notes> diff --git a/doc/installation/building-packages.txt b/doc/installation/building-packages.txt new file mode 100644 index 000000000..b3b775869 --- /dev/null +++ b/doc/installation/building-packages.txt @@ -0,0 +1,228 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _installation-building-packages: + +============================= +Building packages from source +============================= + +Building RPMs +============= + +Building from a tarball +----------------------- + +* Create a directory structure for rpmbuild:: + + rpmdev-setuptree + +* Copy the tarball to ``~/rpmbuild/SOURCES/`` +* Extract another copy of it somewhere else (eg: ``/tmp``) and retrieve + the ``misc/bcfg2.spec`` file +* Run the following:: + + rpmbuild -ba bcfg2.spec + +* The resulting RPMs will be in ``~/rpmbuild/RPMS/`` and SRPMs + in ``~/rpmbuild/SRPMS/``. + +Building Debian packages +======================== + +The Bcfg2 project provides a ``debian`` subdirectory with the project's +source that enables users to create their own Debian/Ubuntu compatible +packages (`.deb` files). + +Build deps +---------- + +If the distribution you are building on already has packaged bcfg2 +(even an older version), the following command will likely install the +necessary build dependencies:: + + apt-get build-dep bcfg2 bcfg2-server + +Install source code +------------------- + +Depending on which version of bcfg2 you want build, you can obtain the +source code from the Download_ page or from the project's git repository. +To create a local anonymous working copy of the latest version of the +bcfg2 source code, use a command like the following:: + + git clone git://git.mcs.anl.gov/bcfg2.git + +Update the changelog +-------------------- + +The next step is to update the ``debian/changelog`` file with an +appropriate package version string. Debian packages contain a version +that is extracted from the latest entry in this file. An appropriate +version will help you distinguish your locally built package from one +provided by your distribution. It also helps the packaging system know +when a newer version of the package is available to install. + +It is possible to skip this step, but the packages you build will have +the same version as the source distribution and will be easy to confuse +with other similarly named (but maybe not equivalent) packages. + +The basic format of the package version string to use is this:: + + <UPSTREAM VER>~<UPSTREAM PRE-VER>+<GIT-ID>-0.1+<LOCAL VER> + +.. note:: + The '+', and '-' characters have significance in determining when + one package is newer than another. The following format is believed + to do the right thing in all common situations. + +The components of the package version string are explained below: + +.. glossary:: + + <UPSTREAM VER> + This is the version of the Bcfg source code you are working + from. It will likely be something like `0.9.6` or `1.0`. + + <UPSTREAM PRE-VER> + If you are using a published pre-release of Bcfg2, it will have + a name like `pre1` or `rc1`. Use that string here, otherwise + drop this component from the package version string. + + +<GIT-ID> + If you are building from a local working copy of the git + repository, it is useful to include the revision in the package + version. If you are building from a downloaded copy of the source, + drop this component (including the preceding plus-sign (`+`) + from the package version string. + + +<LOCAL VER> + This is a locally relevant name like your last name or your + domain name, plus the digit `1`. For example, if your family + name is ''Smith'', you could use `smith1`. If you work for + ''Example Inc'', you could use `example1`. + +Here are some examples: + +* If you are building packages for revision 6c681bd from git, and the + latest published version is 1.2.0rc1, the version string should be + `1.2.0rc1+6c681bd-0.1+example1`. +* If you are building packages for the published 1.0 rc1 version, the + version string should be `1.0rc1-0.1+example1`. +* If you are building packages for the published 1.0 version, the version + string should be `1.0-0.1+example1`. + +If you are working on a git working copy of 1.0 pre5 and have the +``devscripts`` package installed, the following command is a convenient +way to create a well formatted changelog entry:: + + REV=$(git log --oneline | head -n 1 | cut -d' ' -f1) + debchange --force-bad-version --preserve --newversion "1.0~pre5+${REV}-0.1+example1" git revision $REV + +Building the package +-------------------- + +With the preliminaries out of the way, building the package is simple.:: + + cd .. # Change into the top level of the source directory + fakeroot dpkg-buildpackage -uc -us + +The freshly built packages will be deposited in the parent of the +current directory (``..``). Examine the output of ``dpkg-buildpackage`` +for details. + +External build systems +---------------------- + +This section describes how to build bcfg2 and deps via external build +systems (Currently only a PPA). Some other possibilities are: + + * #651 Look into project-builder to make more native-system bcfg2 packages available + * http://en.opensuse.org/Build_Service/Deb_builds + +Launchpad PPA +^^^^^^^^^^^^^ + +https://launchpad.net/~bcfg2 + +To upload to the PPA you need to be on the active member list of `Bcfg2 +in Launchpad`_. + +Note that **after each successful upload**, you should wait until the PPA +is built, and then **install it locally** using ``sudo aptitude update; +sudo aptitude install (packagename)`` so the next build doesn't fail on +your local machine. If you don't want to wait for a PPA binary build to +complete, you can "apt-get source (packagename)" and do a local build +before the PPA build is done. + +setup gpg-agent +""""""""""""""" + +Setting up gpg-agent and pinentry prevents you from having to type your +passphrase repeatedly.:: + + sudo aptitude install gnupg-agent pinentry-gtk2 pinentry-curses + # replace 0xAA95C349 with your GPG Key ID + export GPGKEY=0xAA95C349 + killall -q gpg-agent + eval $(gpg-agent --daemon) + +setup debuild +""""""""""""" + +Tell dpkg-buildpackage who you are, for example:: + + export DEBEMAIL="dclark@pobox.com" + export DEBFULLNAME="Daniel Joseph Barnhart Clark" + +upload bcfg2 to ppa +""""""""""""""""""" + +A ``dists`` file contains a space-separated list of all distributions +you want to build PPA packages for. + +.. code-block:: sh + + #!/bin/sh + + . ./dists + + # Replace 0xAA95C349 with your GnuPG Key ID + export GPGKEY=0xAA95C349 + + sudo apt-get build-dep bcfg2 bcfg2-server + sudo aptitude install git + + VERSION=1.3.2-1 + if [ ! -d testing ]; then + mkdir testing + fi + DATE=$(date +%F-%H%M) + ppa="testing" # "testing" or "ppa" (for stable) + + # download source + cd testing + git clone git://git.mcs.anl.gov/bcfg2 + cd bcfg2 + GITID=$(git log --oneline | head -n 1 | cut -d' ' -f1) + cp debian/changelog ../changelog.orig + + for dist in $DISTS + do + cp ../changelog.orig debian/changelog + (cd debian && dch --distribution ${dist} \ + --force-bad-version \ + --preserve \ + --force-distribution \ + --newversion "${VERSION}~${ppa}~${dist}${DATE}+${GITID}" \ + "bcfg2 backport for ${dist} release ${VERSION} git commit ${GITID}") + debuild --no-tgz-check -rfakeroot -I -S -k${GPGKEY} + done + + for dist in $DISTS + do + dput ppa:bcfg2/${dist}testing ../bcfg2_${VERSION}~${ppa}~${dist}${DATE}+${GITID}_source.changes + done + +.. _Download: http://bcfg2.org/download/ +.. _Bcfg2 in Launchpad: https://launchpad.net/~bcfg2 diff --git a/doc/installation/distributions.txt b/doc/installation/distributions.txt index 9db111682..306439485 100644 --- a/doc/installation/distributions.txt +++ b/doc/installation/distributions.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _distributions: @@ -103,10 +104,12 @@ section will try and meet the dependencies using packages from EPEL_ [#f1]_. The *el5* and the *el6* package should be compatible with `CentOS`_ 5.x/6.x and `Scientific Linux`_. -EPEL_ for 5.x :: +EPEL_ for 5.x:: + [root@centos ~]# rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-4.noarch.rpm -EPEL_ for 6.x :: +EPEL_ for 6.x:: + [root@centos ~]# rpm -Uvh http://download.fedoraproject.org/pub/epel/6/i386/epel-release-6-5.noarch.rpm Install the bcfg2-server and bcfg2 RPMs:: diff --git a/doc/installation/index.txt b/doc/installation/index.txt index 9f04d4b52..9bcf8be15 100644 --- a/doc/installation/index.txt +++ b/doc/installation/index.txt @@ -19,5 +19,5 @@ needs to be installed on any machine you plan to manage by Bcfg2. prerequisites source - packages + building-packages distributions diff --git a/doc/installation/packages.txt b/doc/installation/packages.txt deleted file mode 100644 index b23a870cf..000000000 --- a/doc/installation/packages.txt +++ /dev/null @@ -1,81 +0,0 @@ -.. -*- mode: rst -*- - -.. _packages: - -.. _CentOS: http://www.centos.org/ -.. _Red Hat/RHEL: http://www.redhat.com/rhel/ -.. _Scientific Linux: http://www.scientificlinux.org/ -.. _EPEL: http://fedoraproject.org/wiki/EPEL -.. _RPMForge: https://rpmrepo.org/RPMforge - - -Building RPM packages from source -================================= - -The Bcfg2 distribution contains two different spec files. - -Building from Tarball ---------------------- - -* Copy the tarball to ``/usr/src/packages/SOURCES/`` -* Extract another copy of it somewhere else (eg: ``/tmp``) and retrieve - the ``misc/bcfg2.spec`` file -* Run :: - - rpmbuild -ba bcfg2.spec - -* The resulting RPMs will be in ``/usr/src/packages/RPMS/`` and SRPMs - in ``/usr/src/packages/SRPMS`` - -Building from an GIT Checkout ------------------------------ - -* Change to the ``redhat/`` directory in the working copy -* Run :: - - make - -* The resulting RPMs will be in ``/usr/src/redhat/RPMS/`` and SRPMs - in ``/usr/src/redhat/SRPMS`` and will have the SVN revision appended - -Building RPM packages with ``rpmbuild`` ---------------------------------------- - -While you can go about building all these things from source, this -how to will try and meet the dependencies using packages from EPEL_. -The *el5* and the *el6* package should be compatible with CentOS 5.x. - -* Installation of the EPEL_ repository package :: - - [root@centos ~]# rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-6.noarch.rpm - -* Now you can install the rest of the prerequisites :: - - [root@centos ~]# yum install python-genshi python-cheetah python-lxml - -* After installing git, check out the master branch :: - - [root@centos redhat]# git clone git://git.mcs.anl.gov/bcfg2.git - -* Install the ``fedora-packager`` package :: - - [root@centos ~]# yum install fedora-packager - -* A directory structure for the RPM build process has to be established. :: - - [you@centos ~]$ rpmdev-setuptree - -* Change to the *redhat* directory of the checked out Bcfg2 source:: - - [you@centos ~]$ cd bcfg2/redhat/ - -* In the particular directory is a ``Makefile`` which will do the job of - building the RPM packages. You can do this as root, but it's not - recommended:: - - [you@centos redhat]$ make - -* Now the new RPM package can be installed. Please adjust the path to - your RPM package :: - - [root@centos ~]# rpm -ihv /home/YOU/rpmbuild/RPMS/noarch/bcfg2-server-1.0.0-0.2r5835.noarch.rpm diff --git a/doc/installation/prerequisites.txt b/doc/installation/prerequisites.txt index 81ac12632..a30a3b26b 100644 --- a/doc/installation/prerequisites.txt +++ b/doc/installation/prerequisites.txt @@ -21,7 +21,7 @@ Bcfg2 Client +----------------------------+------------------------+--------------------------------+ | libxslt (if lxml is used) | Any | libxml2 | +----------------------------+------------------------+--------------------------------+ -| python | 2.4 and greater [#f1] | | +| python | 2.4 and greater [#f1]_ | | +----------------------------+------------------------+--------------------------------+ | lxml or elementtree [#f2]_ | Any | lxml: libxml2, libxslt, python | +----------------------------+------------------------+--------------------------------+ @@ -30,6 +30,8 @@ Bcfg2 Client | debsums (if APT tool | Any | | | driver is used) | | | +----------------------------+------------------------+--------------------------------+ +| python-setuptools | Any | | ++----------------------------+------------------------+--------------------------------+ .. [#f1] python 2.5 and later works with elementtree. @@ -54,10 +56,25 @@ Bcfg2 Server +-------------------------------+----------+--------------------------------+ | python-gamin or pyinotify | Any | gamin or inotify, python | +-------------------------------+----------+--------------------------------+ -| M2crypto or python-ssl (note | Any | python, openssl | -| that the ssl module is | | | -| included in python versions | | | -| 2.6 and later | | | +| python-ssl (note | Any | python, backported ssl module | ++-------------------------------+----------+--------------------------------+ +| python-setuptools | Any | | ++-------------------------------+----------+--------------------------------+ +| python-genshi | Any | | ++-------------------------------+----------+--------------------------------+ + +Bcfg2 Reporting +--------------- + +A webserver capabable of running wsgi applications is required for web +reporting, such as Apache + mod_wsgi or nginx. + ++-------------------------------+----------+--------------------------------+ +| Software | Version | Requires | ++===============================+==========+================================+ +| django | 1.2.0+ | | ++-------------------------------+----------+--------------------------------+ +| south | 0.7.5+ | | +-------------------------------+----------+--------------------------------+ Bcfg2 Reporting diff --git a/doc/installation/source.txt b/doc/installation/source.txt index 1406a5ceb..9bf023fbc 100644 --- a/doc/installation/source.txt +++ b/doc/installation/source.txt @@ -1,8 +1,9 @@ .. -*- mode: rst -*- +.. vim: ft=rst -.. _GPG1: http://pgp.mit.edu:11371/pks/lookup?op=get&search=0x75BF2C177F7D197E -.. _GPG2: http://pgp.mit.edu:11371/pks/lookup?op=get&search=0x80B8492FA88FFF4B -.. _Download: http://trac.mcs.anl.gov/projects/bcfg2/wiki/Download +.. _7F7D197E: http://pgp.mit.edu:11371/pks/lookup?op=get&search=0x75BF2C177F7D197E +.. _A88FFF4B: http://pgp.mit.edu:11371/pks/lookup?op=get&search=0x80B8492FA88FFF4B +.. _Download: http://bcfg2.org/download/ .. _source: @@ -17,8 +18,8 @@ Tarball The Bcfg2 source tarball can be grabbed from the `Download`_ page. -All tarballs are signed with GPG keys `7F7D197E <GPG1>`_ or `A88FFF4B -<GPG2>`_. You can verify your download by importing the keys and running :: +All tarballs are signed with GPG keys `7F7D197E`_ or `A88FFF4B`_. You +can verify your download by importing the keys and running :: gpg --recv-keys 0x75bf2c177f7d197e 0x80B8492FA88FFF4B gpg --verify bcfg2-<version>.tar.gz.gpg bcfg2-<version>.tar.gz diff --git a/doc/man/bcfg2-server.txt b/doc/man/bcfg2-server.txt index 3f8f3ea21..33d0df6cf 100644 --- a/doc/man/bcfg2-server.txt +++ b/doc/man/bcfg2-server.txt @@ -11,7 +11,7 @@ Synopsis **bcfg2-server** [-d] [-v] [-C *configfile*] [-D *pidfile*] [-E *encoding*] [-Q *repo path*] [-S *server url*] [-o *logfile*] [-x -*password*] [--ssl-key=\ *ssl key*] +*password*] [--ssl-key=\ *ssl key*] [--no-fam-blocking] Description ----------- @@ -22,19 +22,20 @@ configurations to clients based on the data in its repository. Options ------- --C configfile Specify alternate bcfg2.conf location. --D pidfile Daemonize, placing the program pid in *pidfile*. --E encoding Specify the encoding of config files. --Q path Specify the path to the server repository. --S server Manually specify the server location (as opposed to - using the value in bcfg2.conf). This should be in - the format "https://server:port" --d Enable debugging output. --v Run in verbose mode. --h Print usage information. ---ssl-key=key Specify the path to the SSL key. +-C configfile Specify alternate bcfg2.conf location. +-D pidfile Daemonize, placing the program pid in *pidfile*. +-E encoding Specify the encoding of config files. +-Q path Specify the path to the server repository. +-S server Manually specify the server location (as opposed to + using the value in bcfg2.conf). This should be in + the format "https://server:port" +-d Enable debugging output. +-v Run in verbose mode. +-h Print usage information. +--ssl-key=key Specify the path to the SSL key. +--no-fam-blocking Synonym for fam_blocking = False in bcfg2.conf See Also -------- -:manpage:`bcfg2(1)`, :manpage:`bcfg2-lint(8)` +:manpage:`bcfg2(1)`, :manpage:`bcfg2-lint(8)`, :manpage:`bcfg2.conf(5)` diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index df49f3d4a..825ab2121 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -49,7 +49,7 @@ fam_blocking Whether the server should block at startup until the file monitor backend has processed all events. This can cause a slower startup, but ensure that all files are recognized before the first client - is handled. + is handled. Defaults to True. ignore_files A comma-separated list of globs that should be ignored by the file @@ -447,7 +447,7 @@ settings used for client-server communication. sets the password to use to connect to the server. protocol - Communication protocol to use. Defaults to xmlrpc/ssl. + Communication protocol to use. Defaults to xmlrpc/tlsv1. retries A client-only option. Number of times to retry network @@ -537,6 +537,10 @@ Packages options The following options are specified in the **[packages]** section. + backends + Comma separated list of backends for the dependency resolution. + Default is "Yum,Apt,Pac,Pkgng". + resolver Enable dependency resolution. Default is 1 (true). @@ -632,7 +636,7 @@ Server-only, specified in the **[database]** section. These options control the database connection of the server. engine - The database engine used by the statistics module. One of the + The database engine used by server plugins. One of the following:: postgresql @@ -641,9 +645,9 @@ control the database connection of the server. ado_mssql name - The name of the database to use for statistics data. If + The name of the database to use for server data. If 'database_engine' is set to 'sqlite3' this is a file path to - the sqlite file and defaults to ``$REPOSITORY_DIR/etc/brpt.sqlite``. + the sqlite file and defaults to ``$REPOSITORY_DIR/etc/bcfg2.sqlite``. user User for database connections. Not used for sqlite3. @@ -662,6 +666,41 @@ control the database connection of the server. expected as multiple key=value pairs, separated with commas. The concrete value depends on the database engine. + reporting_engine + The database engine used by the Reporting plugin. One of the + following:: + + postgresql + mysql + sqlite3 + ado_mssql + + If reporting_engine is not specified, the Reporting plugin uses + the same database as the other server plugins. + + reporting_name + The name of the database to use for reporting data. If + 'database_engine' is set to 'sqlite3' this is a file path to + the sqlite file and defaults to + ``$REPOSITORY_DIR/etc/reporting.sqlite``. + + reporting_user + User for reporting database connections. Not used for sqlite3. + + reporting_password + Password for reporting database connections. Not used for sqlite3. + + reporting_host + Host for reporting database connections. Not used for sqlite3. + + reporting_port + Port for reporting database connections. Not used for sqlite3. + + reporting_options + Various options for the reporting database connection. The value + is expected as multiple key=value pairs, separated with commas. + The concrete value depends on the database engine. + Reporting options ----------------- diff --git a/doc/releases/1.3.4.txt b/doc/releases/1.3.4.txt new file mode 100644 index 000000000..f6bc13436 --- /dev/null +++ b/doc/releases/1.3.4.txt @@ -0,0 +1,49 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.3.4: + +1.3.4 +===== + +We are happy to announce the release of Bcfg2 1.3.4. It is available for +download at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +This is primarily a bugfix release. + +* New probes.allowed_groups option to restrict group assignments + +* Bundler fixes: + + * Fix parsing XML template output with encoding declaration + +* bcfg2-lint: + + * Resolve XIncludes when parsing XML for validation + * New TemplateAbuse plugin to detect templated scripts + * New ValidateJSON plugin + +* bcfg2-crypt fixes: + + * Fix logic + * Improve debugging/error handling with Properties files + * Fix exception handling + * Handle error when encrypting properties with multiple keys + +* Add new Augeas client tool driver: + http://docs.bcfg2.org/client/tools/augeas.html +* Restored bcfg2-admin client add functionality +* Migration tool fixes +* Schema fixes +* Add Django 1.6 support +* Use 'public' default pgsql database schema +* Refresh essential packages during Packages.Refresh +* Allow lxml.etree XML implementation to parse very large documents +* Support ACLs without a specific user/group +* Explicitly close database connections at the end of each client run +* Fix verification of symlinks + +Special thanks to the following contributors for this release: Matt Baker, +Simon Ruderich, Michael Fenn, Dan Foster, Richard Connon, John Morris. diff --git a/doc/releases/1.4.0pre1.txt b/doc/releases/1.4.0pre1.txt new file mode 100644 index 000000000..779873f41 --- /dev/null +++ b/doc/releases/1.4.0pre1.txt @@ -0,0 +1,182 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.4.0pre1: + +1.4.0pre1 +========= + +The first prerelease for Bcfg2 1.4.0 is now available at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +Bcfg2 1.4.0pre1 is a prerelease, and contains many new features, +including some that are backwards-incompatible with Bcfg2 1.3.x and +earlier. Please read the release notes thoroughly. This is a prerelease +and as such is not likely suitable for general production deployment. +That said, please help us test the release in non- and preproduction +environments. + +backwards-incompatible user-facing changes +------------------------------------------ + +* Completely rewrote option parser + + Many single character options now have long equivalents. Some + subcommand interfaces (``bcfg2-info``, ``bcfg2-admin``) have been + reorganized to some degree. ``bcfg2-reports`` syntax is completely + different. + +* Added new :ref:`server-plugins-misc-acl` plugin + + Default ACLs only allow clients to perform bcfg2 client runs, and only + permit `bcfg2-admin xcmd` calls from localhost. If you want to change + this, you must enable the ACL plugin and configure your own ACLs. + +* Added genshi requirement for the server + +* :ref:`server-plugins-generators-decisions` + + * Switch plugin to use StructFile instead of host- or group-specific XML + files (this allows a single e.g. whitelist.xml file with <Group> tags) + + You can convert your existing decisions using + ``tools/upgrade/1.4/migrate_decisions.py``. + + +deprecated features (will be removed in a future release, likely 1.5) +--------------------------------------------------------------------- + +* :ref:`server-plugins-structures-bundler` + + * Deprecated use of an explicit name attribute + + You can convert your existing bundles using + ``tools/upgrade/1.4/convert_bundles.py``. + + * Deprecated :ref:`.genshi bundles + <server-plugins-structures-bundler-index-genshi-templates>` (use + .xml bundles and specify the genshi namespace instead) + +* SSLCA + + * Deprecated plugin + * SSLCA functionality has been added to the Cfg plugin: + see :ref:`server-plugins-generators-cfg-ssl-certificates` + +deprecated plugins and features which have been removed +------------------------------------------------------- + +Plugins +^^^^^^^ + +* PostInstall +* TGenshi +* TCheetah +* Account +* Hostbase +* Snapshots +* Statistics +* Editor +* Base + +Client tools +^^^^^^^^^^^^ + +* RPMng +* YUM24 +* YUMng + +Other features +^^^^^^^^^^^^^^ + +* FAM filemonitor +* Removed mode="inherit" support +* Removed support for .cat/.diff files +* Removed support for info/:info files +* Removed "magic" groups (for the Packages plugin) + +other fixes and new features +---------------------------- + +* Added :ref:`inter-bundle dependencies + <server-plugins-structures-bundler-index-dependencies>` +* Added support for :ref:`independent bundles + <server-plugins-structures-bundler-index-disabling-magic>` (replaces + the functionality of Base): +* Added support for wildcard XIncludes +* Add Solaris 11 IPS Package support +* Add bcfg2-report-collector init script to debian package +* Git VCS plugin enhancements +* Removed deprecated plugins + +* :ref:`server-plugins-structures-bundler` + + * Deprecated use of an explicit name attribute + * Deprecated .genshi bundles + * Added path globbing + +* :ref:`server-plugins-grouping-metadata` + + * Allow setting global default authentication type + +* :ref:`server-plugins-generators-packages` + + * Add yum group support to internal resolver + * Change location of plugin-generated APT sources + * Add new Pkgng plugin + * Add ability for per-package recommended flag override + +* :ref:`server-plugins-statistics-reporting` + + * Add support for POSIX user/group entries + * Add support for Django > 1.4 + * Add support for separate reporting database + +* Added option to periodically dump performance stats to logs +* Added option to force server to wait until all FAM events are + processed + +* :ref:`server-plugins-generators-sshbase` + + * Add support for IPv6 addresses in known_hosts file + * Add support for :ref:`encryption of generated ssh keys + <server-plugins-generators-sshbase-encryption>` + +* APT + + * Allow specification of deb-src lines (resolves + http://trac.mcs.anl.gov/projects/bcfg2/ticket/1148) + +* SSLCA + + * Rewrote SSLCA as Cfg handler + + Existing SSLCA installations will need to migrate to the new format + using ``tools/upgrade/1.4/migrate_sslca.py``. + +* :ref:`server-plugins-generators-nagiosgen` + + * Migrate configuration to conf.d + +* :ref:`server-plugins-probes` + + * Rewritten to improve caching + * Add probes.allowed_groups option to restrict group assignments: + see :ref:`server-plugins-probes-dynamic-groups` + + +Thanks +------ + +Special thanks to the following contributors for this release + + * Alexander Sulfrain + * Chris Brinker + * Duncan Hutty + * Jason Kincl + * John Morris + * Matt Schwager + * Michael Fenn + * Stéphane Graber + * Tim Laszlo diff --git a/doc/releases/1.4.0pre2.txt b/doc/releases/1.4.0pre2.txt new file mode 100644 index 000000000..7bbed5603 --- /dev/null +++ b/doc/releases/1.4.0pre2.txt @@ -0,0 +1,37 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.4.0pre2: + +1.4.0pre2 +========= + +The second prerelease for Bcfg2 1.4.0 is now available at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +Bcfg2 1.4.0pre2 is a prerelease, and contains many new features, +including some that are backwards-incompatible with Bcfg2 1.3.x and +earlier. Please read the release notes thoroughly. This is a prerelease +and as such is not likely suitable for general production deployment. +That said, please help us test the release in non- and preproduction +environments. + +backwards-incompatible user-facing changes +------------------------------------------ + +* Changed default communication protocol to xmlrpc/tlsv1 + +* Diff output from files sent to the Reports plugin from the client will now be + in a unified diff format rather than the previous n-diff format. + + This fixes potentially long client runs when comparing files that have + diverged significantly. + +Thanks +------ + +Special thanks to the following contributors for this release + + * Alexander Sulfrain + * Matt Kemp diff --git a/doc/releases/index.txt b/doc/releases/index.txt new file mode 100644 index 000000000..479aa19de --- /dev/null +++ b/doc/releases/index.txt @@ -0,0 +1,13 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-index: + +===================== +Release Announcements +===================== + +.. toctree:: + + 1.4.0pre1 + 1.3.4 diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 6b8a1f467..53bdef24e 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -25,6 +25,7 @@ configuration. Specific features in the new system include: users to drill down to find out about a :ref:`specific host <reports-client-detail>`, rather than only having one huge page with too much information. +* Ability to store reporting data separately from other server data. Installation ============ @@ -214,8 +215,8 @@ database ^^^^^^^^ If you choose to use a different database, you'll need to edit -``/etc/bcfg2.conf``. These fields should be updated in the [database] -section: +``/etc/bcfg2.conf``. These fields should be updated in the +``[database]`` section: * engine @@ -228,11 +229,27 @@ section: * host * port (optional) +To store reporting data separately from the main server data, use +the following options: + +* reporting_engine + + * ex: reporting_engine = mysql + * ex: reporting_engine = postgresql_psycopg2 + +* reporting_name +* reporting_user +* reporting_password +* reporting_host +* reporting_port (optional) + .. warning:: If mysql is used as a backend, it is recommended to use InnoDB for the `storage engine <http://dev.mysql.com/doc/refman/5.1/en/storage-engine-setting.html>`_. +Refer to :ref:`server-database` for a full listing of +available options. statistics ^^^^^^^^^^ diff --git a/doc/server/caching.txt b/doc/server/caching.txt index 32be684db..3557bf0f3 100644 --- a/doc/server/caching.txt +++ b/doc/server/caching.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-caching: @@ -42,15 +43,15 @@ descending order of reliability. That is, odds are higher that biggest speed boost. ``off`` will never result in stale data, but it gives no speed boost. -In addition to the :ref:`server-plugins-grouping-metadata` plugin, -Bcfg2 includes three plugins that can set additional groups, and thus -may affect the caching behavior. They are -:ref:`server-plugins-grouping-grouppatterns`, -:ref:`server-plugins-probes-index`, and +In addition to the :ref:`server-plugins-grouping-metadata` +plugin, Bcfg2 includes three plugins that can set additional +groups, and thus may affect the caching behavior. They +are :ref:`server-plugins-grouping-grouppatterns`, +:ref:`server-plugins-probes`, and :ref:`server-plugins-connectors-puppetenc`. All of those plugins -**except** for PuppetENC fully support all caching levels. PuppetENC -is incompatible with ``aggressive``, and may result in some stale data -with ``cautious``. +**except** for PuppetENC fully support all caching levels. PuppetENC is +incompatible with ``aggressive``, and may result in some stale data with +``cautious``. If you are not using the PuppetENC plugin, and do not have any custom plugins that provide additional groups, then all four modes should be diff --git a/doc/server/configuration.txt b/doc/server/configuration.txt index d3fa42601..79d732f6d 100644 --- a/doc/server/configuration.txt +++ b/doc/server/configuration.txt @@ -216,3 +216,46 @@ To select which backend to use, set the ``backend`` option in the * ``best`` (the default; currently the same as ``builtin``) ``best`` may change in future releases. + +Multiprocessing core configuration +---------------------------------- + +If you use the multiprocessing core, there are other bits you may wish +to twiddle. + +By default, the server spawns as many children as the host has CPUs. +(This is determined by ``multiprocessing.cpu_count()``.) To change +this, set: + +.. code-block:: ini + + [server] + children = 4 + +The optimal number of children may vary depending on your workload. +For instance, if you are using :ref:`native yum +library support <native-yum-libraries>`, then a separate process is +spawned for each client to resolve its package dependencies, so +keeping the children at or below the CPU count is likely a good idea. +If you're not using native yum library support, though, you may wish +to oversubscribe the core slightly. It's recommended that you test +various configurations and use what works best for your workload. + +Secondly, if ``tmpwatch`` is enabled, you must either disable it or +exclude the pattern ``/tmp/pymp-\*``. For instance, on RHEL or CentOS +you may have a line like the following in +``/etc/cron.daily/tmpwatch``: + +.. code-block:: bash + + /usr/sbin/tmpwatch -x /tmp/.X11-unix -x /tmp/.XIM-unix -x /tmp/.font-unix \ + -x /tmp/.ICE-unix -x /tmp/.Test-unix 240 /tmp + +You would need to add ``-X /tmp/pymp-\*`` to it, like so: + +.. code-block:: bash + + /usr/sbin/tmpwatch -x /tmp/.X11-unix -x /tmp/.XIM-unix -x /tmp/.font-unix \ + -x /tmp/.ICE-unix -x /tmp/.Test-unix -X /tmp/pymp-\* 240 /tmp + +See https://bugzilla.redhat.com/show_bug.cgi?id=1058310 for more information. diff --git a/doc/server/database.txt b/doc/server/database.txt index 3c8970f68..986914171 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-database: @@ -9,51 +10,86 @@ Global Database Settings .. versionadded:: 1.3.0 Several Bcfg2 plugins, including -:ref:`server-plugins-grouping-metadata`, -:ref:`server-plugins-probes-index`, and -:ref:`server-plugins-statistics-reporting`, can connect use a -relational database to store data. They use the global database -settings in ``bcfg2.conf``, described in this document, to connect. +:ref:`server-plugins-grouping-metadata`, :ref:`server-plugins-probes`, and +:ref:`server-plugins-statistics-reporting`, can connect use a relational +database to store data. They use the global database settings in +``bcfg2.conf``, described in this document, to connect. .. note:: Although SQLite is supported as a database, it may cause - significant thread contention (and a performance penalty) if you - use SQLite with :ref:`server-plugins-grouping-metadata` or - :ref:`server-plugins-probes-index`. If you are using the - database-backed features of either of those plugins, it's - recommended that you use a higher performance database backend. + significant thread contention (and a performance penalty) if + you use SQLite with :ref:`server-plugins-grouping-metadata` or + :ref:`server-plugins-probes`. If you are using the database-backed + features of either of those plugins, it's recommended that you use + a higher performance database backend. +Separate Reporting Database +=========================== + +.. versionadded:: 1.4.0 + +Bcfg2 supports storing the data generated by the +:ref:`server-plugins-statistics-reporting` in a separate +database from the data generated by the other plugins (e.g. +:ref:`server-plugins-grouping-metadata` and :ref:`server-plugins-probes`). +To activate this support, set the ``reporting_engine``, +``reporting_name``, ``reporting_user``, etc. options in the +``[database]`` section of the config file. The valid values for the +``reporting_*`` options are the same as for the standard database +options. See :ref:`server-database-configuration-options` for a full +listing. + +.. _server-database-configuration-options: + Configuration Options ===================== All of the following options should go in the ``[database]`` section of ``/etc/bcfg2.conf``. -+-------------+------------------------------------------------------------+-------------------------------+ -| Option name | Description | Default | -+=============+============================================================+===============================+ -| engine | The name of the Django database backend to use. See | "sqlite3" | -| | https://docs.djangoproject.com/en/dev/ref/settings/#engine | | -| | for available options (note that django.db.backends is not | | -| | included in the engine name) | | -+-------------+------------------------------------------------------------+-------------------------------+ -| name | The name of the database | "/var/lib/bcfg2/bcfg2.sqlite" | -+-------------+------------------------------------------------------------+-------------------------------+ -| user | The user to connect to the database as | None | -+-------------+------------------------------------------------------------+-------------------------------+ -| password | The password to connect to the database with | None | -+-------------+------------------------------------------------------------+-------------------------------+ -| host | The host to connect to | "localhost" | -+-------------+------------------------------------------------------------+-------------------------------+ -| port | The port to connect to | None | -+-------------+------------------------------------------------------------+-------------------------------+ -| options | Extra parameters to use when connecting to the database. | None | -| | Available parameters vary depending on your database | | -| | backend. The parameters are supplied as comma separated | | -| | key=value pairs. | | -+-------------+------------------------------------------------------------+-------------------------------+ ++--------------------+------------------------------------------------------------+---------------------------------------+ +| Option name | Description | Default | ++====================+============================================================+=======================================+ +| engine | The name of the Django database backend to use. See | "sqlite3" | +| | https://docs.djangoproject.com/en/dev/ref/settings/#engine | | +| | for available options (note that django.db.backends is not | | +| | included in the engine name) | | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| name | The name of the database | "/var/lib/bcfg2/etc/bcfg2.sqlite" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| user | The user to connect to the database as | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| password | The password to connect to the database with | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| host | The host to connect to | "localhost" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| port | The port to connect to | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| options | Extra parameters to use when connecting to the database. | None | +| | Available parameters vary depending on your database | | +| | backend. The parameters are supplied as comma separated | | +| | key=value pairs. | | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_engine | The name of the Django database backend to use for the | None | +| | reporting database. Takes the same values as ``engine``. | | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_name | The name of the reporting database | "/var/lib/bcfg2/etc/reporting.sqlite" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_user | The user to connect to the reporting database as | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_password | The password to connect to the reporting database with | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_host | The host to connect to for the reporting database | "localhost" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_port | The port to connect to for the reporting database | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_options | Extra parameters to use when connecting to the reporting | None | +| | database. Available parameters vary depending on your | | +| | database backend. The parameters are supplied as comma | | +| | separated key=value pairs. | | ++--------------------+------------------------------------------------------------+---------------------------------------+ Database Schema Sync diff --git a/doc/server/encryption.txt b/doc/server/encryption.txt index b657deb8c..db5e2ae29 100644 --- a/doc/server/encryption.txt +++ b/doc/server/encryption.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-encryption: @@ -54,7 +55,7 @@ In general, Properties encryption is preferred for a few reasons: Other types of data that can be encrypted are: * Text content of Path tags in - :ref:`server-plugins-structures-bundler-index` + :ref:`server-plugins-structures-bundler` * Passphrases in XML description files for generated :ref:`server-plugins-generators-cfg-sshkeys` diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index 7a404c824..8b49e244b 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-plugins-generators-cfg: @@ -29,8 +30,8 @@ in ``Cfg/etc/passwd/passwd``, while the ssh pam module config file, ``/etc/pam.d/sshd``, goes in ``Cfg/etc/pam.d/sshd/sshd``. The reason for the like-name directory is to allow multiple versions of each file to exist, as described below. Note that these files are exact copies of what -will appear on the client machine (except when using Genshi or Cheetah -templating -- see below). +will appear on the client machine (except when using templates -- see +below). Group-Specific Files ==================== @@ -241,6 +242,27 @@ comment to appear in the final config file.:: # This is a comment in my template which will be stripped when it's processed through Cheetah \# This comment will appear in the generated config file. +.. _server-plugins-generators-cfg-jinja2: + +Jinja2 Templates +----------------- + +Jinja2 templates allow you to use the `jinja2 templating system +<http://jinja.pocoo.org/>`_. Jinja2 templates should be +named with a ``.jinja2`` extension, e.g.:: + + % ls Cfg/etc/motd + info.xml motd.jinja2 + +Examples +~~~~~~~~ + +.. toctree:: + :glob: + :maxdepth: 1 + + examples/jinja2/* + Inside Templates ---------------- @@ -262,10 +284,10 @@ Several variables are pre-defined inside templates: | repo | The path to the Bcfg2 repository on the filesystem | +-------------+--------------------------------------------------------+ | path | In Genshi templates, ``path`` is a synonym for | -| | ``source_path``. In Cheetah templates, it's a synonym | -| | for ``name``. For this reason, use of ``path`` is | -| | discouraged, and it may be deprecated in a future | -| | release. | +| | ``source_path``. In Cheetah templates and Jinja2 | +| | templates, it's a synonym for ``name``. For this | +| | reason, use of ``path`` is discouraged, and it may be | +| | deprecated in a future release. | +-------------+--------------------------------------------------------+ To access these variables in a Genshi template, you can simply use the @@ -273,6 +295,10 @@ name, e.g.:: Path to this file: ${name} +Similarly, in a Jinja2 template:: + + Path to this file: {{ name }} + In a Cheetah template, the variables are properties of ``self``, e.g.:: @@ -282,15 +308,15 @@ Notes on Using Templates ------------------------ Templates can be host and group specific as well. Deltas will not be -processed for any Genshi or Cheetah base file. +processed for any Genshi, Cheetah, or Jinja2 base file. .. note:: If you are using templating in combination with host-specific or group-specific files, you will need to ensure that the ``.genshi`` - or ``.cheetah`` extension is at the **end** of the filename. Using the - examples from above for *host.example.com* and group *server* you would - have the following:: + ``.cheetah`` or ``.jinja2`` extension is at the **end** of the filename. + Using the examples from above for *host.example.com* and group *server* + you would have the following:: Cfg/etc/fstab/fstab.H_host.example.com.genshi Cfg/etc/fstab/fstab.G50_server.cheetah @@ -344,7 +370,7 @@ An encrypted file should end with ``.crypt``, e.g.:: Cfg/etc/foo.conf/foo.conf.crypt Cfg/etc/foo.conf/foo.conf.G10_foo.crypt -Encrypted Genshi or Cheetah templates can have the extensions in +Encrypted Genshi, Cheetah, and Jinja2 templates can have the extensions in either order, e.g.:: Cfg/etc/foo.conf/foo.conf.crypt.genshi diff --git a/doc/server/plugins/generators/examples/jinja2/simple.txt b/doc/server/plugins/generators/examples/jinja2/simple.txt new file mode 100644 index 000000000..b4ab844fb --- /dev/null +++ b/doc/server/plugins/generators/examples/jinja2/simple.txt @@ -0,0 +1,53 @@ +.. -*- mode: rst -*- + +========================= + Basic Jinja2 Templates +========================= + +This simple example demonstrates basic usage of Jinja2 templates. + +``/var/lib/bcfg2/Cfg/foo/foo.jinja2`` + +.. code-block:: none + + Hostname is {{ metadata.hostname }} + Filename is {{ name }} + Template is {{ source_path }} + Groups: + {% for group in metadata.groups -%} + * {{ group }} + {% endfor %} + Categories: + {% for category in metadata.categories -%} + * {{ category }} -- {{ metadata.categories[category] }} + {% endfor %} + + Probes: + {% for probe in metadata.Probes -%} + * {{ probe }} -- {{ metadata.Probes[probe] }} + {% endfor %} + +Output +====== + +.. code-block:: xml + + <Path type="file" name="/foo" owner="root" mode="0644" group="root"> + Hostname is topaz.mcs.anl.gov + Filename is /foo + Template is /var/lib/bcfg2/Cfg/foo/foo.jinja2 + Groups: + * desktop + * mcs-base + * ypbound + * workstation + * xserver + * debian-sarge + * debian + * a + Categories: + * test -- a + + Probes: + * os -- debian + </Path> diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index 8b317552f..2fe71f895 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -217,10 +217,6 @@ something like this: <Source type="apt" recommended="true" ...> - .. warning:: You must regenerate the Packages cache when adding or - removing the recommended attribute (``bcfg2-admin xcmd - Packages.Refresh``). - .. [#f1] Bcfg2 will by default add **Essential** packages to the client specification. You can disable this behavior by setting the :xml:attribute:`SourceType:essential` @@ -347,9 +343,85 @@ This is done automatically any time `sources.xml`_ is updated. Availability ============ -Support for clients using yum and apt is currently available. Support for +Support for the following clients is currently available. Support for other package managers (Portage, Zypper, IPS, etc) remain to be added. +apt +--- + +All dpkg based clients (for example Debian, Ubuntu or Nexenta) could be +handled with the apt module: + +.. code-block:: xml + + <Source type="apt" + url="http://us.archive.ubuntu.com/ubuntu" + version="intrepid"> + <Component>main</Component> + <Component>universe</Component> + <Arch>i386</Arch> + <Arch>amd64</Arch> + </Source> + + +pac +--- + +For Arch Linux or Parabola GNU/Linux-libre you could use the pac module +for packages. You do not need to supply a version attribute as the mirrors +are rolling release and does not supply different versions. + +.. code-block:: xml + + <Source type="pac" + url="http://mirrors.kernel.org/archlinux/"> + <Component>core</Component> + <Component>extra</Component> + <Component>community</Component> + <Arch>i686</Arch> + <Arch>x86_64</Arch> + </Source> + + +pkgng +----- + +The support for the Next Generation package management tool for FreeBSD +is called pkgng. It downloads the packagesite file from the mirror +and parses the dependencies out of it. It currently does not use the +DNS SRV record lookup mechanism to get the correct mirror and does +not verify the signature inside the packagesite file. + +.. code-block:: xml + + <Source type="pkgng" + url="http://pkg.freebsd.org/" + version="10"> + <Component>latest</Component> + <Arch>x86:64</Arch> + <Arch>x86:32</Arch> + </Source> + + +yum +--- + +Rpm based clients (for example RedHat, CentOS or Fedora) could be handled +with the yum module: + +.. code-block:: xml + + <Source type="yum" + url="http://mirror.centos.org/centos/" + version="5.2"> + <Component>os</Component> + <Component>updates</Component> + <Component>extras</Component> + <Arch>i386</Arch> + <Arch>x86_64</Arch> + </Source> + + Package Checking and Verification ================================= @@ -645,6 +717,9 @@ It understands the following directives: +-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ | Name | Description | Values | Default | +=============+======================================================+==========+===================================================================+ +| backends | List of backends that should be loaded for the | List | Yum,Apt,Pac,Pkgng | +| | dependency resolution. | | | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ | resolver | Enable dependency resolution | Boolean | True | +-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ | metadata | Enable metadata processing. Disabling ``metadata`` | Boolean | True | diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index 64dbc8597..86478a5ae 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -118,6 +118,20 @@ Attributes common to all Path tags: :onlyattrs: name,type +augeas +^^^^^^ + +Run `Augeas <http://www.augeas.net>`_ commands. See +:ref:`client-tools-augeas` for more details. + +.. xml:type:: PathType + :nochildren: + :noattributegroups: + :nodoc: + :notext: + :onlyattrs: owner,group,mode,secontext,lens + :requiredattrs: owner,group,mode + device ^^^^^^ @@ -263,6 +277,7 @@ child ``<ACL>`` tags. For instance: mode="0775"> <ACL type="default" scope="user" user="foouser" perms="rw"/> <ACL type="default" scope="group" group="users" perms="rx"/> + <ACL type="default" scope="other" perms="r"/> </Path> .. xml:element:: ACL @@ -271,6 +286,9 @@ It is not currently possible to manually set an effective rights mask; the mask will be automatically calculated from the given ACLs when they are applied. +For directories either no default ACL entries or at least an entry for +the owner, owning group and other must be defined. + Note that it is possible to set ACLs that demand different permissions on a file than those specified in the ``perms`` attribute on the ``Path`` tag. For instance: diff --git a/doc/server/plugins/generators/sshbase.txt b/doc/server/plugins/generators/sshbase.txt index 641b9c598..540cc1e06 100644 --- a/doc/server/plugins/generators/sshbase.txt +++ b/doc/server/plugins/generators/sshbase.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-plugins-generators-sshbase: @@ -13,8 +14,9 @@ record for the current system. It has two functions: -* Generating new ssh keys -- When a client requests a dsa, rsa, or v1 key, - and there is no existing key in the repository, one is generated. +* Generating new ssh keys -- When a client requests a ecdsa, dsa, rsa, + or v1 key, and there is no existing key in the repository, one is + generated. * Maintaining the ``ssh_known_hosts`` file -- all current known public keys (and extra public key stores) are integrated into a single @@ -31,7 +33,7 @@ Interacting with SSHbase ``<repo>/SSHbase/<key filename>.H_<hostname>`` * Pre-seeding can also be performed using ``bcfg2-admin pull - ConfigFile /name/of/ssh/key`` + Path /name/of/ssh/key`` * Revoking existing keys -- deleting ``<repo>/SSHbase/\*.H_<hostname>`` will remove keys for an existing @@ -160,6 +162,9 @@ in order to permit :ref:`pulling with bcfg2-admin <server-admin-pull>`. You should almost certainly set ``sensitive`` to "true" in ``info.xml``. + +.. _server-plugins-generators-sshbase-encryption: + Encryption ========== diff --git a/doc/server/plugins/index.txt b/doc/server/plugins/index.txt index f3d6daa73..b39be0786 100644 --- a/doc/server/plugins/index.txt +++ b/doc/server/plugins/index.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-plugins-index: @@ -13,7 +14,7 @@ perform one of several tasks: #. Generating configuration entry contents for clients #. Probing client-side state (like hardware inventory, etc) -- the generic client probing mechanism is described at - :ref:`server-plugins-probes-index`. + :ref:`server-plugins-probes`. #. Automating administrative tasks (e.g. :ref:`server-plugins-generators-sshbase` which automates ssh key management) diff --git a/doc/server/plugins/probes/index.txt b/doc/server/plugins/probes/index.txt index 306a752b6..434ce20a8 100644 --- a/doc/server/plugins/probes/index.txt +++ b/doc/server/plugins/probes/index.txt @@ -1,6 +1,7 @@ .. -*- mode: rst -*- +.. vim: ft=rst -.. _server-plugins-probes-index: +.. _server-plugins-probes: ====== Probes @@ -13,6 +14,9 @@ the system disk, you would want to know this information to correctly generate an `/etc/auto.master` autofs config file for each type. Here we will look at how to do this. +Probes also allow dynamic group assignment for clients, see +:ref:`server-plugins-probes-dynamic-groups`. + First, create a ``Probes`` directory in our toplevel repository location:: @@ -119,6 +123,45 @@ is to add the ``/etc/auto.master`` to a Bundle: <Path name='/etc/auto.master'/> +.. _server-plugins-probes-dynamic-groups: + +Dynamic Group Assignment +======================== + +The output lines of the probe matching "group:" are used to +dynamically assign hosts to groups. These dynamic groups need not already +exist in ``Metadata/groups.xml``. If a dynamic group is defined in +``Metadata/groups.xml``, clients that include this group will also get +all included groups and bundles. + +Consider the following output of a probe:: + + group:debian-wheezy + group:amd64 + +This assigns the client to the groups debian-wheezy and amd64. + +To prevent clients from manipulating the probe output and choosing +unexpected groups (and receiving their potential sensitive files) you +can use the ``allowed_groups`` option in the ``[probes]`` section of +``bcfg2.conf`` on the server. This whitespace-separated list of +anchored regular expressions (must match the complete group name) +controls dynamic group assignments. Only matching groups are +allowed. The default allows all groups. + +.. versionadded:: 1.3.4 + +Example: + +.. code-block:: ini + + [probes] + allowed_groups = debian-(squeeze|wheezy|sid) i386 + +This allows the groups `debian-squeeze`, `debian-wheezy`, `debian-sid` +and `i386`. With the probe output from above, this setting would +disallow the group `amd64`. + Handling Probe Output ===================== diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt index 0b6b8eb50..31faeaf17 100644 --- a/doc/server/plugins/structures/bundler/index.txt +++ b/doc/server/plugins/structures/bundler/index.txt @@ -1,6 +1,7 @@ .. -*- mode: rst -*- +.. vim: ft=rst -.. _server-plugins-structures-bundler-index: +.. _server-plugins-structures-bundler: ======= Bundler @@ -91,6 +92,9 @@ split these entries into two bundles. See :ref:`server-plugins-structures-bundler-bcfg2-server` for an example of this. + +.. _server-plugins-structures-bundler-index-disabling-magic: + Disabling Magic --------------- @@ -111,6 +115,9 @@ entries in independent bundles are only executed if ``when="always"``. (I.e., an Action entry in an independent bundle with ``when="modified"`` is useless.) + +.. _server-plugins-structures-bundler-index-genshi-templates: + Genshi templates ================ @@ -138,6 +145,9 @@ entries in the bundle. See :ref:`bcfg2-info <server-bcfg2-info>` for more details. + +.. _server-plugins-structures-bundler-index-dependencies: + Dependencies ============ diff --git a/doc/server/plugins/structures/bundler/nagios.txt b/doc/server/plugins/structures/bundler/nagios.txt index d25e1cf0a..47a61b898 100644 --- a/doc/server/plugins/structures/bundler/nagios.txt +++ b/doc/server/plugins/structures/bundler/nagios.txt @@ -27,29 +27,14 @@ the clients. <Path name='/etc/hosts.deny'/> <Path name='/etc/services'/> <Path name='/etc/snmpd.conf'/> - <Path name='/usr/lib/nagios/plugins/check_disks_scratchgpfs1.tg'/> - <Path name='/usr/lib/nagios/plugins/check_fs.mds'/> - <Path name='/usr/lib/nagios/plugins/check_gm_network.tg'/> - <Path name='/usr/lib/nagios/plugins/check_gpfs_wan.tg'/> - <Path name='/usr/lib/nagios/plugins/check_hung_jobs.tg'/> - <Path name='/usr/lib/nagios/plugins/check_mem.mds'/> - <Path name='/usr/lib/nagios/plugins/check_mem.tg'/> - <Path name='/usr/lib/nagios/plugins/check_nvidia_acceleration.tg'/> - <Path name='/usr/lib/nagios/plugins/check_os.mds'/> - <Path name='/usr/lib/nagios/plugins/check_procinfo.mds'/> - <Path name='/usr/lib/nagios/plugins/check_torque.tg'/> - <Path name='/usr/lib/nagios/plugins/check_uname_r.tg'/> - <Path name='/usr/lib/nagios/plugins/check_uname_r.tg.conf'/> + <Path glob='/usr/lib/nagios/plugins/*'/> <Service name='snmpd'/> <Group name='nagios-server'> <Package name='nagios'/> <Package name='nagios-devel'/> <Package name='nagios-www'/> <Path name='/etc/httpd/conf.d/nagios.conf'/> - <Path name='/etc/nagios/cgi.cfg'/> - <Path name='/etc/nagios/checkcommands.cfg'/> - <Path name='/etc/nagios/nagios.cfg'/> - <Path name='/etc/nagios/resource.cfg'/> + <Path glob='/etc/nagios/*'/> </Group> </Bundle> diff --git a/doc/server/plugins/structures/bundler/torque.txt b/doc/server/plugins/structures/bundler/torque.txt index f6349df6e..01316f3a3 100644 --- a/doc/server/plugins/structures/bundler/torque.txt +++ b/doc/server/plugins/structures/bundler/torque.txt @@ -29,9 +29,7 @@ A longer Bundle that includes many group-specific entries. <BoundPath type='directory' owner='root' group='root' mode='0755' name='/var/spool/torque/mom_logs'/> <BoundPath type='directory' owner='root' group='root' mode='0755' name='/var/spool/torque/mom_priv'/> <BoundPath type='directory' owner='root' group='root' mode='0755' name='/var/spool/torque/mom_priv/jobs'/> - <Path name='/var/spool/torque/mom_priv/config'/> - <Path name='/var/spool/torque/mom_priv/prologue'/> - <Path name='/var/spool/torque/mom_priv/epilogue'/> + <Path glob='/var/spool/torque/mom_priv/*'/> </Group> <Group name='torque-server'> <Service name='torque_server'/> diff --git a/doc/server/xml-common.txt b/doc/server/xml-common.txt index 3aacfd468..5302a59e4 100644 --- a/doc/server/xml-common.txt +++ b/doc/server/xml-common.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _xml-features: @@ -76,20 +77,12 @@ Or, more compactly: Manage the abstract service "bar" As an example, consider the following :ref:`bundle -<server-plugins-structures-bundler-index>`: +<server-plugins-structures-bundler>`: .. code-block:: xml <Bundle> - <Path name='/etc/ssh/ssh_host_dsa_key'/> - <Path name='/etc/ssh/ssh_host_rsa_key'/> - <Path name='/etc/ssh/ssh_host_dsa_key.pub'/> - <Path name='/etc/ssh/ssh_host_rsa_key.pub'/> - <Path name='/etc/ssh/ssh_host_key'/> - <Path name='/etc/ssh/ssh_host_key.pub'/> - <Path name='/etc/ssh/sshd_config'/> - <Path name='/etc/ssh/ssh_config'/> - <Path name='/etc/ssh/ssh_known_hosts'/> + <Path glob='/etc/ssh/*'/> <Group name='rpm'> <Package name='openssh'/> <Package name='openssh-askpass'/> @@ -122,23 +115,7 @@ any other clients. +------------------------+-----------------------------------+ | Group/Hostname | Entry | +========================+===================================+ -| all | ``/etc/ssh/ssh_host_dsa_key`` | -+------------------------+-----------------------------------+ -| all | ``/etc/ssh/ssh_host_rsa_key`` | -+------------------------+-----------------------------------+ -| all | ``/etc/ssh/ssh_host_dsa_key.pub`` | -+------------------------+-----------------------------------+ -| all | ``/etc/ssh/ssh_host_rsa_key.pub`` | -+------------------------+-----------------------------------+ -| all | ``/etc/ssh/ssh_host_key`` | -+------------------------+-----------------------------------+ -| all | ``/etc/ssh/ssh_host_key.pub`` | -+------------------------+-----------------------------------+ -| all | ``/etc/ssh/sshd_config`` | -+------------------------+-----------------------------------+ -| all | ``/etc/ssh/ssh_config`` | -+------------------------+-----------------------------------+ -| all | ``/etc/ssh/ssh_known_hosts`` | +| all | ``/etc/ssh/*`` | +------------------------+-----------------------------------+ | ``rpm`` | Package ``openssh`` | +------------------------+-----------------------------------+ @@ -333,7 +310,7 @@ Feature Matrix | <server-plugins-misc-acl>` | | | | | +---------------------------------------------------+--------------+--------+------------+------------+ | :ref:`Bundler | Yes | Yes | Yes | Yes | -| <server-plugins-structures-bundler-index>` | | | | | +| <server-plugins-structures-bundler>` | | | | | +---------------------------------------------------+--------------+--------+------------+------------+ | :ref:`info.xml <server-info>` | Yes [#f1]_ | Yes | Yes | Yes | +---------------------------------------------------+--------------+--------+------------+------------+ diff --git a/doc/unsorted/bcfg2.conf-options.txt b/doc/unsorted/bcfg2.conf-options.txt deleted file mode 100644 index 57e26cbd2..000000000 --- a/doc/unsorted/bcfg2.conf-options.txt +++ /dev/null @@ -1,19 +0,0 @@ -.. -*- mode: rst -*- - -.. _unsorted-bcfg2.conf-options: - -========== -bcfg2.conf -========== - -This page documents the various options available in bcfg2.conf. The -various sections correspond to the sections in the file itself. - -components -========== - -logging -------- - -Specify an alternate path for the lockfile used by the bcfg2 client. -Default value is ``/var/lock/bcfg2.run`` diff --git a/doc/unsorted/dynamic_groups.txt b/doc/unsorted/dynamic_groups.txt deleted file mode 100644 index 11535dc8b..000000000 --- a/doc/unsorted/dynamic_groups.txt +++ /dev/null @@ -1,27 +0,0 @@ -.. -*- mode: rst -*- - -.. _unsorted-dynamic_groups: - -============== -Dynamic Groups -============== - -Bcfg2 supports the use of dynamic groups. These groups are not included -in a client's profile group, but instead are derived from the results -of probes executed on the client. These dynamic groups need not already -exist in ``Metadata/groups.xml``. If a dynamic group is defined in -``Metadata/groups.xml``, clients that include this group will also get -all included groups and bundles. - -Setting up dynamic groups -========================= - -In order to define a dynamic group, setup a probe that outputs the text -based on system properties:: - - group:groupname - -This output is processed by the Bcfg2 server, and results in dynamic -group membership in groupname for the client. See the :ref:`Probes -<server-plugins-probes-index>` page for a more thorough description -of probes. diff --git a/doc/unsorted/emacs_snippet.txt b/doc/unsorted/emacs_snippet.txt index b9f7fd25b..4eefb4583 100644 --- a/doc/unsorted/emacs_snippet.txt +++ b/doc/unsorted/emacs_snippet.txt @@ -31,7 +31,7 @@ More snippets are under development. ("<Group" "<Group name='${1:groupname}> $0 </Group>" nil) - ("<Config" "<ConfigFile name='${1:filename}'/> + ("<Path" "<Path name='${1:filename}'/> $0" nil) ("<Service" "<Service name='${1:svcname}'/> $0" nil) diff --git a/doc/unsorted/howtos.txt b/doc/unsorted/howtos.txt index 0c5b482d9..81b38e54d 100644 --- a/doc/unsorted/howtos.txt +++ b/doc/unsorted/howtos.txt @@ -12,7 +12,7 @@ Here are several howtos that describe different aspects of Bcfg2 deployment * AnnotatedExamples - a description of basic Bcfg2 specification operations * BuildingDebianPackages - How to build debian packages * :ref:`appendix-guides-gentoo` - Issues specific to running Bcfg2 on Gentoo -* :ref:`server-plugins-probes-index` - How to use Probes to gather information from a client machine. +* :ref:`server-plugins-probes` - How to use Probes to gather information from a client machine. * :ref:`client-tools-actions` - How to use Actions -* :ref:`unsorted-dynamic_groups` - Using dynamic groups +* :ref:`server-plugins-probes-dynamic-groups` - Using dynamic groups * :ref:`client-modes-paranoid` - How to run an update in paranoid mode diff --git a/doc/unsorted/vim_snippet.txt b/doc/unsorted/vim_snippet.txt index e4fda7eca..4598b5c1d 100644 --- a/doc/unsorted/vim_snippet.txt +++ b/doc/unsorted/vim_snippet.txt @@ -30,9 +30,9 @@ that allow quick composition of bundles and base files. <Group name='${1:groupname}'> ${2} </Group> - # ConfigFile - snippet <Config - <ConfigFile name='${1:filename}'/> + # Path + snippet <Path + <Path name='${1:filename}'/> # Service snippet <Service <Service name='${1:svcname}'/> diff --git a/doc/unsorted/writing_specification.txt b/doc/unsorted/writing_specification.txt index e7763cee1..f9dd30a14 100644 --- a/doc/unsorted/writing_specification.txt +++ b/doc/unsorted/writing_specification.txt @@ -132,15 +132,7 @@ The following is an annotated copy of a bundle: .. code-block:: xml <Bundle> - <Path name='/etc/ssh/ssh_host_dsa_key'/> - <Path name='/etc/ssh/ssh_host_rsa_key'/> - <Path name='/etc/ssh/ssh_host_dsa_key.pub'/> - <Path name='/etc/ssh/ssh_host_rsa_key.pub'/> - <Path name='/etc/ssh/ssh_host_key'/> - <Path name='/etc/ssh/ssh_host_key.pub'/> - <Path name='/etc/ssh/sshd_config'/> - <Path name='/etc/ssh/ssh_config'/> - <Path name='/etc/ssh/ssh_known_hosts'/> + <Path glob='/etc/ssh/*'/> <Group name='rpm'> <Package name='openssh'/> <Package name='openssh-askpass'/> @@ -171,23 +163,7 @@ can be used in bundles) +----------------+-------------------------------+ | Group | Entry | +================+===============================+ -| all | /etc/ssh/ssh_host_dsa_key | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_host_rsa_key | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_host_dsa_key.pub | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_host_rsa_key.pub | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_host_key | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_host_key.pub | -+----------------+-------------------------------+ -| all | /etc/ssh/sshd_config | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_config | -+----------------+-------------------------------+ -| all | /etc/ssh/ssh_known_hosts | +| all | /etc/ssh/* | +----------------+-------------------------------+ | rpm | Package openssh | +----------------+-------------------------------+ diff --git a/examples/bcfg2.conf b/examples/bcfg2.conf index cac424576..1f0984f0e 100644 --- a/examples/bcfg2.conf +++ b/examples/bcfg2.conf @@ -1,5 +1,4 @@ [communication] -protocol = xmlrpc/ssl password = foobat # certificate = /etc/bcfg2.key # key = /etc/bcfg2.key diff --git a/man/bcfg2-admin.8 b/man/bcfg2-admin.8 index 06cbeec0b..adc335f9b 100644 --- a/man/bcfg2-admin.8 +++ b/man/bcfg2-admin.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-ADMIN" "8" "March 18, 2013" "1.3" "Bcfg2" +.TH "BCFG2-ADMIN" "8" "April 06, 2014" "1.3" "Bcfg2" .SH NAME bcfg2-admin \- Perform repository administration tasks . @@ -78,10 +78,6 @@ Use \(aqpassword\(aq for client communication. .B backup Create an archive of the entire Bcfg2 repository. .TP -.B bundle \fIaction\fP -Display details about the available bundles (See BUNDLE OPTIONS -below). -.TP .B client \fIaction\fP \fIclient\fP [attribute=value] Add, edit, or remove clients entries in metadata (See CLIENT OPTIONS below). @@ -91,9 +87,15 @@ Compare two client configurations. Can be used to verify consistent behavior between releases. Determine differences between files or directories (See COMPARE OPTIONS below). .TP +.B dbshell +Call the Django \(aqdbshell\(aq command on the configured database. +.TP .B init Initialize a new repository (interactive). .TP +.B initreports +Initialize the Reporting database. +.TP .B minestruct \fIclient\fP [\-f xml\-file] [\-g groups] Build structure entries based on client statistics extra entries (See MINESTRUCT OPTIONS below). @@ -105,12 +107,21 @@ Query server for performance data. Install configuration information into repo based on client bad entries (See PULL OPTIONS below). .TP -.B reports [init|load_stats|purge|scrub|update] -Interact with the dynamic reporting system (See REPORTS OPTIONS -below). +.B purgereports +Purge historic and expired data from the Reporting database +.TP +.B reportssqlall +Call the Django \(aqshell\(aq command on the Reporting database. .TP -.B snapshots [init|dump|query|reports] -Interact with the Snapshots database (See SNAPSHOTS OPTIONS below). +.B reportsstats +Print Reporting database statistics. +.TP +.B scrubreports +Scrub the Reporting database for duplicate reasons and orphaned +entries. +.TP +.B shell +Call the Django \(aqshell\(aq command on the configured database. .TP .B syncdb Sync the Django ORM with the configured database. @@ -118,6 +129,12 @@ Sync the Django ORM with the configured database. .B tidy Remove unused files from repository. .TP +.B updatereports +Apply database schema updates to the Reporting database. +.TP +.B validatedb +Call the Django \(aqvalidate\(aq command on the configured database. +.TP .B viz [\-H] [\-b] [\-k] [\-o png\-file] Create a graphviz diagram of client, group and bundle information (See VIZ OPTIONS below). @@ -125,23 +142,6 @@ Create a graphviz diagram of client, group and bundle information .B xcmd Provides a XML\-RPC Command Interface to the bcfg2\-server. .UNINDENT -.SS BUNDLE OPTIONS -.INDENT 0.0 -.TP -.B mode -One of the following. -.INDENT 7.0 -.TP -.B \fIlist\-xml\fP -List all available xml bundles -.TP -.B \fIlist\-genshi\fP -List all available genshi bundles -.TP -.B \fIshow\fP -Interactive dialog to get details about the available bundles -.UNINDENT -.UNINDENT .SS CLIENT OPTIONS .INDENT 0.0 .TP @@ -170,11 +170,24 @@ are \(aqprofile\(aq, \(aquuid\(aq, \(aqpassword\(aq, \(aqlocation\(aq, \(aqsecur .SS COMPARE OPTIONS .INDENT 0.0 .TP +.B \-d \fIN\fP, \-\-diff\-lines \fIN\fP +Show only N lines of a diff +.UNINDENT +.INDENT 0.0 +.TP +.B \-c, \-\-color +Show colors even if not ryn from a TTY +.TP +.B \-q, \-\-quiet +Only show that entries differ, not how they differ +.UNINDENT +.INDENT 0.0 +.TP .B old -Specify the location of the old configuration file. +Specify the location of the old configuration(s). .TP .B new -Specify the location of the new configuration file. +Specify the location of the new configuration(s). .UNINDENT .SS MINESTRUCT OPTIONS .INDENT 0.0 @@ -200,51 +213,30 @@ Specify the type of the entry to pull. .B entry name Specify the name of the entry to pull. .UNINDENT -.SS REPORTS OPTIONS -.INDENT 0.0 -.TP -.B load_stats [\-s] [\-c] [\-03] -Load statistics data. -.TP -.B purge [\-\-client [n]] [\-\-days [n]] [\-\-expired] -Purge historic and expired data. -.TP -.B scrub -Scrub the database for duplicate reasons and orphaned entries. -.TP -.B update -Apply any updates to the reporting database. -.UNINDENT -.SS SNAPSHOTS OPTIONS -.INDENT 0.0 -.TP -.B init -Initialize the snapshots database. -.TP -.B query -Query the snapshots database. -.TP -.B dump -Dump some of the contents of the snapshots database. -.TP -.B reports [\-a] [\-b] [\-e] [\-\-date=MM\-DD\-YYYY] -Generate reports for clients in the snapshots database. -.UNINDENT .SS VIZ OPTIONS .INDENT 0.0 .TP -.B \-H +.B \-H, \-\-includehosts Include hosts in diagram. .TP -.B \-b +.B \-b, \-\-includebundles Include bundles in diagram. +.UNINDENT +.INDENT 0.0 .TP -.BI \-o \ <outfile> +.B \-o \fIoutfile\fP, \-\-outfile \fIoutfile\fP Write to outfile file instead of stdout. +.UNINDENT +.INDENT 0.0 .TP -.B \-k +.B \-k, \-\-includekey Add a shape/color key. .UNINDENT +.INDENT 0.0 +.TP +.B \-c \fIhostname\fP, \-\-only\-client \fIhostname\fP +Only show groups and bundles for the named client +.UNINDENT .SH SEE ALSO .sp \fIbcfg2\-info(8)\fP, \fIbcfg2\-server(8)\fP diff --git a/man/bcfg2-server.8 b/man/bcfg2-server.8 index dcec03252..60fe58a30 100644 --- a/man/bcfg2-server.8 +++ b/man/bcfg2-server.8 @@ -34,7 +34,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .sp \fBbcfg2\-server\fP [\-d] [\-v] [\-C \fIconfigfile\fP] [\-D \fIpidfile\fP] [\-E \fIencoding\fP] [\-Q \fIrepo path\fP] [\-S \fIserver url\fP] [\-o \fIlogfile\fP] [\-x -\fIpassword\fP] [\-\-ssl\-key=\fIssl key\fP] +\fIpassword\fP] [\-\-ssl\-key=\fIssl key\fP] [\-\-no\-fam\-blocking] .SH DESCRIPTION .sp \fBbcfg2\-server\fP is the daemon component of Bcfg2 which serves @@ -70,9 +70,12 @@ Print usage information. .TP .BI \-\-ssl\-key\fB= key Specify the path to the SSL key. +.TP +.BI \-\-no\-fam\-blocking +Synonym for fam_blocking = False in bcfg2.conf .UNINDENT .SH SEE ALSO .sp -\fIbcfg2(1)\fP, \fIbcfg2\-lint(8)\fP +\fIbcfg2(1)\fP, \fIbcfg2\-lint(8)\fP, \fIbcfg2.conf(5)\fP .\" Generated by docutils manpage writer. . diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 91ebc0020..a8366721a 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2.CONF" "5" "July 19, 2013" "1.3" "Bcfg2" +.TH "BCFG2.CONF" "5" "April 06, 2014" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -69,7 +69,6 @@ valid: .ft C inotify gamin -fam pseudo .ft P .fi @@ -77,11 +76,10 @@ pseudo .UNINDENT .TP .B fam_blocking -. Whether the server should block at startup until the file monitor backend has processed all events. This can cause a slower startup, but ensure that all files are recognized before the first client -is handled. +is handled. Defaults to True. .TP .B ignore_files A comma\-separated list of globs that should be ignored by the file @@ -119,24 +117,22 @@ available plugins are: .sp .nf .ft C -Account -Base +ACL Bundler Bzr Cfg Cvs Darcs -DBStats Decisions +Defaults Deps -Editor FileProbes Fossil Git +GroupLogic GroupPatterns Guppy Hg -Hostbase Ldap Metadata NagiosGen @@ -151,13 +147,9 @@ Reporting Rules SEModules ServiceCompat -Snapshots SSHbase -Statistics Svn -TCheetah TemplateHelper -TGenshi Trigger .ft P .fi @@ -210,28 +202,9 @@ The umask to set for the server. Default is \fI0077\fP. .sp This section has a listing of all the plugins currently provided with Bcfg2. -.SS Account Plugin -.sp -The account plugin manages authentication data, including the following. -.INDENT 0.0 -.IP \(bu 2 -\fB/etc/passwd\fP -.IP \(bu 2 -\fB/etc/group\fP -.IP \(bu 2 -\fB/etc/security/limits.conf\fP -.IP \(bu 2 -\fB/etc/sudoers\fP -.IP \(bu 2 -\fB/root/.ssh/authorized_keys\fP -.UNINDENT -.SS Base Plugin +.SS ACL Plugin .sp -The Base plugin is a structure plugin that provides the ability -to add lists of unrelated entries into client configuration entry -inventories. Base works much like Bundler in its file format. This -structure plugin is good for the pile of independent configs needed for -most actual systems. +The ACL plugin controls which hosts can make which XML\-RPC calls. .SS Bundler Plugin .sp The Bundler plugin is used to describe groups of inter\-dependent @@ -251,19 +224,16 @@ The Cfg plugin provides a repository to describe configuration file contents for clients. In its simplest form, the Cfg repository is just a directory tree modeled off of the directory tree on your client machines. -.SS Cvs Plugin (experimental) +.SS Cvs Plugin .sp The Cvs plugin allows you to track changes to your Bcfg2 repository using a Concurrent version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -.SS Darcs Plugin (experimental) +.SS Darcs Plugin .sp The Darcs plugin allows you to track changes to your Bcfg2 repository using a Darcs version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -.SS DBStats Plugin -.sp -Direct to database statistics plugin. .SS Decisions Plugin .sp The Decisions plugin has support for a centralized set of per\-entry @@ -281,11 +251,6 @@ and only populate attributes that are not yet set. .sp The Deps plugin allows you to make a series of assertions like "Package X requires Package Y (and optionally also Package Z etc.)" -.SS Editor Plugin -.sp -The Editor plugin attempts to allow you to partially manage -configuration for a file. Its use is not recommended and not well -documented. .SS FileProbes Plugin .sp The FileProbes plugin allows you to probe a client for a file, which is @@ -302,6 +267,10 @@ get revision information out of your repository for reporting purposes. The Git plugin allows you to track changes to your Bcfg2 repository using a Git version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. +.SS GroupLogic Plugin +.sp +The GroupLogic plugin lets you flexibly assign group membership with a +Genshi template. .SS GroupPatterns Plugin .sp The GroupPatterns plugin is a connector that can assign clients group @@ -310,14 +279,11 @@ membership based on patterns in client hostnames. .sp The Guppy plugin is used to trace memory leaks within the bcfg2\-server process using Guppy. -.SS Hg Plugin (experimental) +.SS Hg Plugin .sp The Hg plugin allows you to track changes to your Bcfg2 repository using a Mercurial version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -.SS Hostbase Plugin -.sp -The Hostbase plugin is an IP management system built on top of Bcfg2. .SS Ldap Plugin .sp The Ldap plugin makes it possible to fetch data from an LDAP directory, @@ -330,7 +296,7 @@ metadata. .sp The NagiosGen plugin dynamically generates Nagios configuration files based on Bcfg2 data. -.SS Ohai Plugin (experimental) +.SS Ohai Plugin .sp The Ohai plugin is used to detect information about the client operating system. The data is reported back to the server using JSON. @@ -370,10 +336,10 @@ The Reporting plugin enables the collection of data for use with Bcfg2\(aqs dynamic reporting system. .SS Rules Plugin .sp -The Rules plugin provides literal configuration entries that resolve the -abstract configuration entries normally found in the Bundler and Base -plugins. The literal entries in Rules are suitable for consumption by -the appropriate client drivers. +The Rules plugin provides literal configuration entries that resolve +the abstract configuration entries normally found in Bundler. The +literal entries in Rules are suitable for consumption by the +appropriate client drivers. .SS SEModules Plugin .sp The SEModules plugin provides a way to distribute SELinux modules via @@ -381,37 +347,16 @@ Bcfg2. .SS ServiceCompat Plugin .sp The ServiceCompat plugin converts service entries for older clients. -.SS Snapshots Plugin -.sp -The Snapshots plugin stores various aspects of a client’s state when the -client checks in to the server. .SS SSHbase Plugin .sp The SSHbase generator plugin manages ssh host keys (both v1 and v2) for hosts. It also manages the ssh_known_hosts file. It can integrate host keys from other management domains and similarly export its keys. -.SS SSLCA Plugin -.sp -The SSLCA plugin is designed to handle creation of SSL privatekeys and -certificates on request. -.SS Statistics -.sp -The Statistics plugin is deprecated (see Reporting). .SS Svn Plugin .sp The Svn plugin allows you to track changes to your Bcfg2 repository using a Subversion backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -.SS TCheetah Plugin -.sp -The TCheetah plugin allows you to use the cheetah templating system to -create files. It also allows you to include the results of probes -executed on the client in the created files. -.SS TGenshi Plugin -.sp -The TGenshi plugin allows you to use the Genshi templating system to -create files. It also allows you to include the results of probes -executed on the client in the created files. .SS Trigger Plugin .sp The Trigger plugin provides a method for calling external scripts when @@ -503,7 +448,7 @@ the password clients need to use to communicate. On a client, sets the password to use to connect to the server. .TP .B protocol -Communication protocol to use. Defaults to xmlrpc/ssl. +Communication protocol to use. Defaults to xmlrpc/tlsv1. .TP .B retries A client\-only option. Number of times to retry network @@ -692,28 +637,11 @@ copies will be kept. .UNINDENT .UNINDENT .UNINDENT -.SH SNAPSHOTS OPTIONS +.SH SSL CA OPTIONS .sp -Specified in the \fB[snapshots]\fP section. These options control the -server snapshots functionality. -.INDENT 0.0 -.INDENT 3.5 -.INDENT 0.0 -.TP -.B driver -sqlite -.TP -.B database -The name of the database to use for statistics data. -.sp -e.g.: \fB$REPOSITORY_DIR/etc/bcfg2.sqlite\fP -.UNINDENT -.UNINDENT -.UNINDENT -.SH SSLCA OPTIONS -.sp -These options are necessary to configure the SSLCA plugin and can be -found in the \fB[sslca_default]\fP section of the configuration file. +These options are necessary to configure the SSL CA feature of the Cfg +plugin and can be found in the \fB[sslca_default]\fP section of the +configuration file. .INDENT 0.0 .INDENT 3.5 .INDENT 0.0 @@ -745,7 +673,7 @@ control the database connection of the server. .INDENT 0.0 .TP .B engine -The database engine used by the statistics module. One of the +The database engine used by server plugins. One of the following: .INDENT 7.0 .INDENT 3.5 @@ -762,9 +690,9 @@ ado_mssql .UNINDENT .TP .B name -The name of the database to use for statistics data. If +The name of the database to use for server data. If \(aqdatabase_engine\(aq is set to \(aqsqlite3\(aq this is a file path to -the sqlite file and defaults to \fB$REPOSITORY_DIR/etc/brpt.sqlite\fP. +the sqlite file and defaults to \fB$REPOSITORY_DIR/etc/bcfg2.sqlite\fP. .TP .B user User for database connections. Not used for sqlite3. @@ -782,6 +710,49 @@ Port for database connections. Not used for sqlite3. Various options for the database connection. The value is expected as multiple key=value pairs, separated with commas. The concrete value depends on the database engine. +.TP +.B reporting_engine +The database engine used by the Reporting plugin. One of the +following: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C + postgresql + mysql + sqlite3 + ado_mssql + +If reporting_engine is not specified, the Reporting plugin uses +the same database as the other server plugins. +.ft P +.fi +.UNINDENT +.UNINDENT +.TP +.B reporting_name +The name of the database to use for reporting data. If +\(aqdatabase_engine\(aq is set to \(aqsqlite3\(aq this is a file path to +the sqlite file and defaults to +\fB$REPOSITORY_DIR/etc/reporting.sqlite\fP. +.TP +.B reporting_user +User for reporting database connections. Not used for sqlite3. +.TP +.B reporting_password +Password for reporting database connections. Not used for sqlite3. +.TP +.B reporting_host +Host for reporting database connections. Not used for sqlite3. +.TP +.B reporting_port +Port for reporting database connections. Not used for sqlite3. +.TP +.B reporting_options +Various options for the reporting database connection. The value +is expected as multiple key=value pairs, separated with commas. +The concrete value depends on the database engine. .UNINDENT .UNINDENT .UNINDENT diff --git a/misc/bcfg2-selinux.spec b/misc/bcfg2-selinux.spec index d694783b5..e3701bbdd 100644 --- a/misc/bcfg2-selinux.spec +++ b/misc/bcfg2-selinux.spec @@ -12,11 +12,11 @@ # # Don't forget to change the Release: tag below to something like 0.1 #%%global _rc 1 -#%%global _pre 2 +%global _pre pre1 %global _pre_rc %{?_pre:.pre%{_pre}}%{?_rc:.rc%{_rc}} Name: bcfg2-selinux -Version: 1.3.3 +Version: 1.4.0 Release: 1%{?_pre_rc}%{?dist} Summary: Bcfg2 Client and Server SELinux policy @@ -32,8 +32,8 @@ Conflicts: selinux-policy = 3.11.1 License: BSD URL: http://bcfg2.org -Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}.tar.gz -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}%{?_pre_rc}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}%{?_pre_rc}-%{release}-root-%(%{__id_u} -n) BuildArch: noarch BuildRequires: checkpolicy, selinux-policy-devel, hardlink diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 276e72478..357afed4e 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -19,11 +19,17 @@ # # Don't forget to change the Release: tag below to something like 0.1 #%%global _rc 1 -#%%global _pre 2 +%global _pre pre1 %global _nightly 1 %global _date %(date +%Y%m%d) %global _pre_rc %{?_pre:pre%{_pre}}%{?_rc:rc%{_rc}} +# cherrypy 3.3 actually doesn't exist yet, but 3.2 has bugs that +# prevent it from working: +# https://bitbucket.org/cherrypy/cherrypy/issue/1154/assertionerror-in-recv-when-ssl-is-enabled +%global build_cherry_py 0 + + Name: bcfg2 Version: 1.4.0 Release: 0.1.%{?_nightly:nightly.%{_date}}%{?_pre_rc}%{?dist} @@ -42,7 +48,7 @@ Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}%{?_pre_rc}.t Source1: http://www.w3.org/2001/XMLSchema.xsd %if %{?rhel}%{!?rhel:10} <= 5 || 0%{?suse_version} # EL5 and OpenSUSE require the BuildRoot tag -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildRoot: %{_tmppath}/%{name}-%{version}%{?_pre_rc}-%{release}-root-%(%{__id_u} -n) %endif BuildArch: noarch @@ -56,7 +62,9 @@ BuildRequires: python-Genshi BuildRequires: python-gamin BuildRequires: python-pyinotify BuildRequires: python-python-daemon +%if %{build_cherry_py} BuildRequires: python-CherryPy >= 3 +%endif %else # ! suse_version BuildRequires: python-daemon BuildRequires: python-inotify @@ -73,18 +81,30 @@ BuildRequires: python-ssl BuildRequires: python-nose BuildRequires: mock BuildRequires: m2crypto +# EPEL uses the properly-named python-django starting with EPEL7 +%if 0%{?rhel} && 0%{?rhel} > 6 +BuildRequires: python-django +%else BuildRequires: Django +%endif BuildRequires: python-genshi BuildRequires: python-cheetah -BuildRequires: pylibacl BuildRequires: libselinux-python +BuildRequires: pylibacl BuildRequires: python-pep8 +BuildRequires: pylint +%if %{build_cherry_py} BuildRequires: python-cherrypy >= 3 +%endif BuildRequires: python-mock -BuildRequires: pylint %endif # rhel > 5 %endif # vendor != redhat || rhel defined %endif # ! suse_version +%if 0%{?fedora} && 0%{?fedora} >= 16 || 0%{?rhel} && 0%{?rhel} >= 7 +# Pick up _unitdir macro +BuildRequires: systemd +%endif + %if 0%{?mandriva_version} # mandriva seems to behave differently than other distros and needs @@ -113,7 +133,6 @@ BuildRequires: python-docutils BuildRequires: systemd-units %endif -Requires: python-lxml %if 0%{?rhel} && 0%{?rhel} < 6 Requires: python-ssl %endif @@ -228,6 +247,7 @@ deployment strategies. This package includes the Bcfg2 server software. +%if %{build_cherry_py} %package server-cherrypy Summary: Bcfg2 Server - CherryPy backend %if 0%{?suse_version} @@ -238,10 +258,8 @@ Group: System Environment/Daemons Requires: bcfg2 = %{version}-%{release} Requires: bcfg2-server = %{version}-%{release} -# cherrypy 3.3 actually doesn't exist yet, but 3.2 has bugs that -# prevent it from working: -# https://bitbucket.org/cherrypy/cherrypy/issue/1154/assertionerror-in-recv-when-ssl-is-enabled -Requires: python-cherrypy > 3.3 +# https://bitbucket.org/cherrypy/cherrypy/issue/1068/file-upload-crashes-when-using-https +Requires: python-cherrypy >= 3.2.6 %description server-cherrypy Bcfg2 helps system administrators produce a consistent, reproducible, @@ -270,6 +288,8 @@ Bcfg2 can enable the construction of complex change management and deployment strategies. This package includes the Bcfg2 CherryPy server backend. +%endif # build_cherry_py + %package web Summary: Bcfg2 Web Reporting Interface @@ -282,9 +302,15 @@ Requires: python-django >= 1.2 Requires: python-django-south >= 0.7 %else Group: System Tools +# EPEL uses the properly-named python-django starting with EPEL7 +%if 0%{?rhel} && 0%{?rhel} > 6 +Requires: python-django +%else Requires: Django >= 1.2 Requires: Django-south >= 0.7 %endif +Requires: bcfg2-server +%endif %if "%{_vendor}" == "redhat" Requires: mod_wsgi %global apache_conf %{_sysconfdir}/httpd @@ -413,8 +439,6 @@ awk ' # Fixup some paths %{__perl} -pi -e 's@/etc/default@%{_sysconfdir}/sysconfig@g' tools/bcfg2-cron -%{__perl} -pi -e 's@/usr/lib/bcfg2@%{_libexecdir}@g' debian/bcfg2.cron.daily -%{__perl} -pi -e 's@/usr/lib/bcfg2@%{_libexecdir}@g' debian/bcfg2.cron.hourly # Get rid of extraneous shebangs for f in `find src/lib -name \*.py` @@ -453,7 +477,7 @@ install -d %{buildroot}/var/adm/fillup-templates mv %{buildroot}%{_bindir}/bcfg2* %{buildroot}%{_sbindir} -%if 0%{?fedora} < 16 +%if 0%{?fedora} && 0%{?fedora} < 16 || 0%{?rhel} && 0%{?rhel} < 7 # Install SysV init scripts for everyone but new Fedoras install -m 755 redhat/scripts/bcfg2.init \ %{buildroot}%{_initrddir}/bcfg2 @@ -491,10 +515,17 @@ install -p -m 644 redhat/systemd/%{name}.service \ install -p -m 644 redhat/systemd/%{name}-server.service \ %{buildroot}%{_unitdir}/%{name}-server.service +%if 0%{?rhel} != 5 # Webserver install -d %{buildroot}%{apache_conf}/conf.d install -p -m 644 misc/apache/bcfg2.conf \ %{buildroot}%{apache_conf}/conf.d/wsgi_bcfg2.conf +%else +# remove web server files not in EL5 packages +rm -r %{buildroot}%{_datadir}/bcfg2/reports.wsgi \ + %{buildroot}%{_datadir}/bcfg2/site_media +%endif + # mandriva cannot handle %ghost without the file existing, # so let's touch a bunch of empty config files @@ -528,11 +559,7 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %if 0%{?suse_version} %fillup_and_insserv -f bcfg2 %else - %if 0%{?fedora} >= 16 - /bin/systemctl daemon-reload >/dev/null 2>&1 || : - %else - /sbin/chkconfig --add bcfg2 - %endif + /sbin/chkconfig --add bcfg2 %endif fi %endif @@ -546,11 +573,7 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %if 0%{?suse_version} %fillup_and_insserv -f bcfg2-server %else - %if 0%{?fedora} >= 16 - /bin/systemctl daemon-reload >/dev/null 2>&1 || : - %else - /sbin/chkconfig --add bcfg2-server - %endif + /sbin/chkconfig --add bcfg2-server %endif fi %endif @@ -564,13 +587,8 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %if 0%{?suse_version} %stop_on_removal bcfg2 %else - %if 0%{?fedora} >= 16 - /bin/systemctl --no-reload disable bcfg2.service > /dev/null 2>&1 || : - /bin/systemctl stop bcfg2.service > /dev/null 2>&1 || : - %else /sbin/service bcfg2 stop &>/dev/null || : /sbin/chkconfig --del bcfg2 - %endif %endif fi %endif @@ -585,13 +603,8 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %stop_on_removal bcfg2-server %stop_on_removal bcfg2-report-collector %else - %if 0%{?fedora} >= 16 - /bin/systemctl --no-reload disable bcfg2-server.service > /dev/null 2>&1 || : - /bin/systemctl stop bcfg2-server.service > /dev/null 2>&1 || : - %else /sbin/service bcfg2-server stop &>/dev/null || : /sbin/chkconfig --del bcfg2-server - %endif %endif fi %endif @@ -608,11 +621,7 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %if 0%{?suse_version} %insserv_cleanup %else - %if 0%{?fedora} >= 16 - /bin/systemctl try-restart bcfg2.service >/dev/null 2>&1 || : - %else /sbin/service bcfg2 condrestart &>/dev/null || : - %endif %endif fi %endif @@ -621,16 +630,9 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %if 0%{?fedora} >= 18 %systemd_postun bcfg2-server.service %else - %if 0%{?fedora} >= 16 - /bin/systemctl daemon-reload >/dev/null 2>&1 || : - %endif if [ $1 -ge 1 ] ; then # Package upgrade, not uninstall - %if 0%{?fedora} >= 16 - /bin/systemctl try-restart bcfg2-server.service >/dev/null 2>&1 || : - %else /sbin/service bcfg2-server condrestart &>/dev/null || : - %endif fi %if 0%{?suse_version} if [ $1 -eq 0 ]; then @@ -665,7 +667,7 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %{_mandir}/man5/bcfg2.conf.5* %ghost %attr(600,root,root) %config(noreplace,missingok) %{_sysconfdir}/bcfg2.cert %ghost %attr(0600,root,root) %config(noreplace,missingok) %{_sysconfdir}/bcfg2.conf -%if 0%{?fedora} >= 16 +%if 0%{?fedora} >= 16 || 0%{?rhel} >= 7 %config(noreplace) %{_unitdir}/%{name}.service %else %{_initrddir}/bcfg2 @@ -699,7 +701,7 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %defattr(-,root,root,-) %endif %ghost %attr(600,root,root) %config(noreplace) %{_sysconfdir}/bcfg2.key -%if 0%{?fedora} >= 16 +%if 0%{?fedora} >= 16 || 0%{?rhel} >= 7 %config(noreplace) %{_unitdir}/%{name}-server.service %else %{_initrddir}/bcfg2-server @@ -712,7 +714,7 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %{python_sitelib}/Bcfg2/Server %{python_sitelib}/Bcfg2/Reporting %{python_sitelib}/Bcfg2/manage.py* -%exclude %{python_sitelib}/Bcfg2/Server/CherrypyCore.py +%exclude %{python_sitelib}/Bcfg2/Server/CherryPyCore.py* %dir %{_datadir}/bcfg2 %{_datadir}/bcfg2/schemas @@ -727,19 +729,24 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %doc tools/* +%if %{build_cherry_py} %files server-cherrypy %if 0%{?rhel} == 5 || 0%{?suse_version} %defattr(-,root,root,-) %endif -%{python_sitelib}/Bcfg2/Server/CherrypyCore.py +%{python_sitelib}/Bcfg2/Server/CherryPyCore.py +%endif +# bcfg2-web package is disabled on EL5, which lacks Django +%if 0%{?rhel} != 5 %files web -%if 0%{?rhel} == 5 || 0%{?suse_version} +%if 0%{?suse_version} %defattr(-,root,root,-) %endif %{_datadir}/bcfg2/reports.wsgi %{_datadir}/bcfg2/site_media %config(noreplace) %{apache_conf}/conf.d/wsgi_bcfg2.conf +%endif %files doc %if 0%{?rhel} == 5 || 0%{?suse_version} @@ -755,6 +762,34 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ %changelog +* Wed Apr 23 2014 Jonathan S. Billings <jsbillin@umich.edu> - 1.3.4-2 +- Fixed RPM scriptlet logic for el6 vs. Fedora init commands + +* Sun Apr 6 2014 John Morris <john@zultron.com> - 1.3.4-1 +- New upstream release + +* Wed Feb 26 2014 John Morris <john@zultron.com> - 1.3.3-5 +- EL7: Re-add deps and re-enable %%check script; bz #1058427 + +* Sat Feb 1 2014 John Morris <john@zultron.com> - 1.3.3-4 +- Disable bcfg2-web package on EL5; bz #1058427 +- Disable %%check on EL7; missing EPEL deps +- BR: systemd to pick up _unitdir macro + +* Mon Jan 27 2014 Sol Jerome <sol.jerome@gmail.com> - 1.3.3-4 +- Fix BuildRequires for EPEL7's Django +- Remove unnecessary client-side lxml dependency +- Add Django dependency for bcfg2-web (the web package *does* require + Django for the database) +- Fix OS detection for RHEL7 initscripts + +* Sun Dec 15 2013 John Morris <john@zultron.com> - 1.3.3-3 +- Remove unneeded Django dep in 'web' package, bz #1043229 + +* Sun Nov 24 2013 John Morris <john@zultron.com> - 1.3.3-2 +- Fix CherryPyCore.py exclude glob to include compiled files +- Disable server-cherrypy package build to make Fedora buildsys happy + * Thu Nov 07 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.3-1 - New upstream release @@ -776,23 +811,32 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ - Changes to %%post* scripts - Rearrange %%files sections -* Mon Jul 01 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.2-1 -- New upstream release - -* Thu Mar 21 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.1-1 -- New upstream release - -* Fri Mar 15 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0 -- New upstream release - -* Tue Jan 29 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0rc2 -- New upstream release - -* Wed Jan 09 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0rc1 -- New upstream release - -* Tue Oct 30 2012 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0pre2 -- New upstream release +* Wed Jul 3 2013 John Morris <john@zultron.com> - 1.3.2-1 +- Update to new upstream version 1.3.2 +- Move settings.py into server package (fixes bug reported on bcfg2-dev ML) +- Use init scripts from redhat/scripts directory +- Fix EL5/EL6 sphinx docs +- Require python-inotify instead of gamin-python; recommended by upstream +- Remove obsolete bcfg2-py27-auth.patch, accepted upstream +- Add %%check script + - Hack test suite to use local copies of XMLSchema.xsd and xml.xsd + - Many new BRs to support %%check script + - Disable %%check script on EL5, where there is no python-mock package +- Cleanups to _pre/_rc macros +- Mark EL5 relics +- Other minor formatting + +* Mon Apr 08 2013 Fabian Affolter <mail@fabian-affolter.ch> - 1.3.1-1 +- Updated to new upstream version 1.3.1 + +* Mon Mar 18 2013 Fabian Affolter <mail@fabian-affolter.ch> - 1.3.0-1 +- Updated to new upstream version 1.3.0 + +* Wed Feb 13 2013 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 1.3.0-0.2.pre2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_19_Mass_Rebuild + +* Wed Oct 31 2012 Fabian Affolter <mail@fabian-affolter.ch> - 1.3.0-0.1.pre2 +- Updated to new upstream version 1.3.0 pre2 * Wed Oct 17 2012 Chris St. Pierre <chris.a.st.pierre@gmail.com> 1.3.0-0.2pre1 - Split bcfg2-selinux into its own specfile @@ -800,12 +844,28 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ * Fri Sep 14 2012 Chris St. Pierre <chris.a.st.pierre@gmail.com> 1.3.0-0.1pre1 - Added -selinux subpackage -* Fri Aug 31 2012 Sol Jerome <sol.jerome@gmail.com> 1.3.0-0.0pre1 -- New upstream release +* Mon Aug 27 2012 Václav PavlÃn <vpavlin@redhat.com> - 1.2.3-3 +- Scriptlets replaced with new systemd macros (#850043) * Wed Aug 15 2012 Chris St. Pierre <chris.a.st.pierre@gmail.com> 1.2.3-0.1 - Added tools/ as doc for bcfg2-server subpackage +* Wed Jul 18 2012 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 1.2.3-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_18_Mass_Rebuild + +* Sat Jul 07 2012 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.3-1 +- Fix CVE-2012-3366 +- Updated to new upstream version 1.2.3 + +* Tue May 01 2012 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.2-2 +- python-nose is needed by bcfg2-test + +* Fri Apr 06 2012 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.2-1 +- Updated to new upstream version 1.2.2 + +* Sun Feb 26 2012 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.1-2 +- Fixed systemd files + * Sat Feb 18 2012 Christopher 'm4z' Holm <686f6c6d@googlemail.com> 1.2.1 - Added Fedora and Mandriva compatibilty (for Open Build Service). - Added missing dependency redhat-lsb. @@ -814,15 +874,212 @@ sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ - Added openSUSE compatibility. - Various changes to satisfy rpmlint. +* Tue Feb 07 2012 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.1-1 +- Added examples package +- Updated to new upstream version 1.2.1 + +* Mon Jan 02 2012 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.0-6 +- Added support for systemd +- Example subpackage + +* Wed Sep 07 2011 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.0-5 +- Updated to new upstreadm version 1.2.0 + +* Wed Sep 07 2011 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.0-4.1.rc1 +- Updated to new upstreadm version 1.2.0rc1 + +* Wed Jun 22 2011 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.0-3.1.pre3 +- Updated to new upstreadm version 1.2.0pre3 + +* Wed May 04 2011 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.0-2.1.pre2 +- Added bcfg2-lint stuff +- Pooled file section entries to reduce future maintainance +- Removed Patch + +* Wed May 04 2011 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.0-1.1.pre2 +- Updated to new upstream version 1.2.0pre2 + +* Sun Mar 20 2011 Fabian Affolter <mail@fabian-affolter.ch> - 1.2.0-1.1.pre1 +- Added doc subpackage +- Updated to new upstream version 1.2.0pre1 + +* Mon Feb 07 2011 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 1.1.1-2.1 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_15_Mass_Rebuild + * Thu Jan 27 2011 Chris St. Pierre <chris.a.st.pierre@gmail.com> 1.2.0pre1-0.0 - Added -doc sub-package +* Thu Nov 18 2010 Fabian Affolter <mail@fabian-affolter.ch> - 1.1.1-2 +- Added new man page +- Updated doc section (ChangeLog is gone) + +* Thu Nov 18 2010 Fabian Affolter <mail@fabian-affolter.ch> - 1.1.1-1 +- Updated to new upstream version 1.1.1 + +* Fri Nov 5 2010 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.1.0-3 +- Add patch from Gordon Messmer to fix authentication on F14+ (Python 2.7) + +* Mon Sep 27 2010 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.1.0-2 +- Update to final version + +* Wed Sep 15 2010 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.1.0-1.3.rc5 +- Update to 1.1.0rc5: + +* Tue Aug 31 2010 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.1.0-1.2.rc4 +- Add new YUMng driver + +* Wed Jul 21 2010 David Malcolm <dmalcolm@redhat.com> - 1.1.0-1.1.rc4.1 +- Rebuilt for https://fedoraproject.org/wiki/Features/Python_2.7/MassRebuild + +* Tue Jul 20 2010 Fabian Affolter <mail@fabian-affolter.ch> - 1.1.0-1.1.rc4 +- Added patch to fix indention + +* Tue Jul 20 2010 Fabian Affolter <mail@fabian-affolter.ch> - 1.1.0-0.1.rc4 +- Updated to new upstream release candidate RC4 + * Mon Jun 21 2010 Fabian Affolter <fabian@bernewireless.net> - 1.1.0rc3-0.1 - Changed source0 in order that it works with spectool +* Sat Jun 19 2010 Fabian Affolter <mail@fabian-affolter.ch> - 1.1.0-0.1.rc3 +- Updated to new upstream release candidate RC3 + +* Sun May 02 2010 Fabian Affolter <mail@fabian-affolter.ch> - 1.1.0-0.2.rc1 +- Changed define to global +- Added graphviz for the server package + +* Wed Apr 28 2010 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.1.0-0.1.rc1 +- Update to 1.1.0rc1 + +* Tue Apr 13 2010 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.0.1-1 +- Update to final version + +* Fri Nov 6 2009 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.0.0-2 +- Fixup the bcfg2-server init script + +* Fri Nov 6 2009 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.0.0-1 +- Update to 1.0.0 final + +* Wed Nov 4 2009 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.0.0-0.5.rc2 +- Only require python-ssl on EPEL + +* Sat Oct 31 2009 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.0.0-0.4.rc2 +- Update to 1.0.0rc2 + +* Mon Oct 26 2009 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.0.0-0.3.rc1 +- Update to 1.0rc1 + +* Fri Oct 16 2009 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.0-0.2.pre5 +- Add python-ssl requirement + +* Tue Aug 11 2009 Jeffrey C. Ollie <jeff@ocjtech.us> - 1.0-0.1.pre5 +- Update to 1.0pre5 + +* Fri Jul 24 2009 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 0.9.6-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_12_Mass_Rebuild + +* Mon Feb 23 2009 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 0.9.6-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_11_Mass_Rebuild + +* Sat Nov 29 2008 Ignacio Vazquez-Abrams <ivazqueznet+rpm@gmail.com> - 0.9.6-2 +- Rebuild for Python 2.6 + +* Tue Nov 18 2008 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.6-1 +- Update to 0.9.6 final. + +* Tue Oct 14 2008 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.6-0.8.pre3 +- Update to 0.9.6pre3 + +* Sat Aug 9 2008 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.6-0.2.pre2 +- Update to 0.9.6pre2 + +* Wed May 28 2008 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.6-0.1.pre1 +- Update to 0.9.6pre1 + +* Fri Feb 15 2008 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.5.7-1 +- Update to 0.9.5.7. + +* Fri Feb 15 2008 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.5.7-1 +- Update to 0.9.5.7. + +* Fri Jan 11 2008 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.5.5-1 +- Update to 0.9.5.5 +- More egg-info entries. + +* Wed Jan 9 2008 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.5.4-1 +- Update to 0.9.5.4. + +* Tue Jan 8 2008 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.5.3-1 +- Update to 0.9.5.3 +- Package egg-info files. + +* Mon Nov 12 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.5.2-1 +- Update to 0.9.5.2 + +* Mon Nov 12 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.5-2 +- Fix oops. + +* Mon Nov 12 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.5-1 +- Update to 0.9.5 final. + +* Mon Nov 05 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.5-0.5.pre7 +- Commit new patches to CVS. + +* Mon Nov 05 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.5-0.4.pre7 +- Update to 0.9.5pre7 + +* Wed Jun 27 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.4-4 +- Oops, apply right patch + +* Wed Jun 27 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.4-3 +- Add patch to fix YUMng problem + +* Mon Jun 25 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.4-2 +- Bump revision and rebuild + +* Mon Jun 25 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.4-1 +- Update to 0.9.4 final + +* Thu Jun 21 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.4-0.1.pre4 +- Update to 0.9.4pre4 + +* Thu Jun 14 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.4-0.1.pre3 +- Update to 0.9.4pre3 + +* Tue Jun 12 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.4-0.1.pre2 +- Update to 0.9.4pre2 + +* Tue May 22 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.3-2 +- Drop requires on pyOpenSSL +- Add requires on redhat-lsb +- (Fixes #240871) + +* Mon Apr 30 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.3-1 +- Update to 0.9.3 + +* Tue Mar 20 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.2-4 +- Server needs pyOpenSSL + +* Wed Feb 28 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.2-3 +- Don't forget %%dir + +* Wed Feb 28 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.2-2 +- Fix #230478 + +* Mon Feb 19 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.2-1 +- Update to 0.9.2 + +* Thu Feb 8 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.9.1-1.d +- Update to 0.9.1d + * Fri Feb 2 2007 Mike Brady <mike.brady@devnull.net.nz> 0.9.1 - Removed use of _libdir due to Red Hat x86_64 issue. +* Tue Jan 9 2007 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.8.7.3-2 +- Merge client back into base package. + +* Wed Dec 27 2006 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.8.7.3-1 +- Update to 0.8.7.3 + * Fri Dec 22 2006 Jeffrey C. Ollie <jeff@ocjtech.us> - 0.8.7.1-5 - Server needs client library files too so put them in main package diff --git a/osx/Makefile b/osx/Makefile index 7714fa4d8..174eb53c8 100644 --- a/osx/Makefile +++ b/osx/Makefile @@ -28,9 +28,9 @@ SITELIBDIR = /Library/Python/${PYVERSION}/site-packages # an Info.plist file for packagemaker to look at for package creation # and substitute the version strings. Major/Minor versions can only be # integers (e.g. "1" and "00" for bcfg2 version 1.0.0. -BCFGVER = 1.3.3 +BCFGVER = 1.4.0pre1 MAJOR = 1 -MINOR = 33 +MINOR = 40 default: clean client diff --git a/osx/bcfg2.conf b/osx/bcfg2.conf index cac424576..1f0984f0e 100644 --- a/osx/bcfg2.conf +++ b/osx/bcfg2.conf @@ -1,5 +1,4 @@ [communication] -protocol = xmlrpc/ssl password = foobat # certificate = /etc/bcfg2.key # key = /etc/bcfg2.key diff --git a/osx/macports/Portfile b/osx/macports/Portfile index 83c7f4075..05ce40bf0 100644 --- a/osx/macports/Portfile +++ b/osx/macports/Portfile @@ -5,7 +5,7 @@ PortSystem 1.0 PortGroup python26 1.0 name bcfg2 -version 1.3.3 +version 1.4.0pre1 categories sysutils python maintainers gmail.com:sol.jerome license BSD diff --git a/redhat/scripts/bcfg2-server.init b/redhat/scripts/bcfg2-server.init index c4412d1c3..f14938a16 100755 --- a/redhat/scripts/bcfg2-server.init +++ b/redhat/scripts/bcfg2-server.init @@ -27,7 +27,7 @@ PATH=/sbin:/bin:/usr/bin:/usr/sbin test -f /etc/sysconfig/$prog && . /etc/sysconfig/$prog if [ "$BCFG2_SERVER_ENABLED" -eq 0 ] ; then - failure $"bcfg2-server is disabled - see /etc/sysconfig/bcfg2-server" + failure $"bcfg2-server is disabled - see /etc/sysconfig/$prog" echo exit 0 fi @@ -49,7 +49,13 @@ start () { stop () { echo -n $"Stopping $prog: " - killproc ${prog} && success || failure + # we do NOT want to specify the pidfile to killproc; if we do, and + # it has to kill -9 the server, it only kills the master and the + # child processes stay running (if the multiprocessing core is in + # use). By not specifying a pidfile, it looks in the process + # table for all bcfg2-server processes, and kill -9's them all if + # necessary. + killproc -d 30 ${prog} && success || failure RETVAL=$? echo rm -f /var/lock/subsys/$prog diff --git a/redhat/systemd/bcfg2.service b/redhat/systemd/bcfg2.service index 572391fd0..245c80cce 100644 --- a/redhat/systemd/bcfg2.service +++ b/redhat/systemd/bcfg2.service @@ -3,11 +3,12 @@ Description=Bcfg2 configuration client After=syslog.target network.target [Service] -Type=forking +Type=oneshot StandardOutput=syslog StandardError=syslog EnvironmentFile=-/etc/sysconfig/bcfg2 -ExecStart=/usr/sbin/bcfg2 $OPTIONS +ExecStart=/usr/sbin/bcfg2 $BCFG2_OPTIONS +RemainAfterExit=yes [Install] WantedBy=multi-user.target diff --git a/reports/reports.wsgi b/reports/reports.wsgi index c2a2be1ce..7113b3308 100644 --- a/reports/reports.wsgi +++ b/reports/reports.wsgi @@ -2,9 +2,20 @@ import os import Bcfg2.Options import Bcfg2.DBSettings -Bcfg2.Options.get_parser().parse() +config_parsed = False import django.core.handlers.wsgi + def application(environ, start_response): - return django.core.handlers.wsgi.WSGIHandler()(environ, start_response) + global config_parsed + + # with wsgi, the environment isn't present in os.environ, but + # is passwd to the application function + if 'BCFG2_CONFIG_FILE' in environ: + os.environ['BCFG2_CONFIG_FILE'] = environ['BCFG2_CONFIG_FILE'] + if not config_parsed: + Bcfg2.Options.get_parser().parse() + config_parsed = True + + return django.core.handlers.wsgi.WSGIHandler()(environ, start_response) diff --git a/reports/site_media/bcfg2_base.css b/reports/site_media/bcfg2_base.css index cd5acc6c1..edec22d7c 100644 --- a/reports/site_media/bcfg2_base.css +++ b/reports/site_media/bcfg2_base.css @@ -159,6 +159,9 @@ ul.menu-level2 { .modified-lineitem { background: #FFEC8B; } +.stale-lineitem { + background: #CCCCCC; +} table.grid-view { border: solid 1px #98DBCC; diff --git a/schemas/acl.xsd b/schemas/acl.xsd index 0c3e3ecdd..ac678b6c1 100644 --- a/schemas/acl.xsd +++ b/schemas/acl.xsd @@ -3,7 +3,7 @@ <xsd:annotation> <xsd:documentation> acl config schema for bcfg2 - Matt Schwager + Matt Schwager </xsd:documentation> </xsd:annotation> diff --git a/schemas/augeas.xsd b/schemas/augeas.xsd new file mode 100644 index 000000000..df27f91cc --- /dev/null +++ b/schemas/augeas.xsd @@ -0,0 +1,229 @@ +<?xml version="1.0" encoding="utf-8"?> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> + + <xsd:annotation> + <xsd:documentation> + Augeas commands + </xsd:documentation> + </xsd:annotation> + + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + + <xsd:complexType name="AugeasRemoveCommand"> + <xsd:annotation> + <xsd:documentation> + Implementation of the Augeas ``rm`` command. + </xsd:documentation> + </xsd:annotation> + <xsd:attribute type="xsd:string" name="path" use="required"> + <xsd:annotation> + <xsd:documentation> + Delete nodes (and all children) matching the given Augeas + path expression. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name="AugeasMoveCommand"> + <xsd:annotation> + <xsd:documentation> + Implementation of the Augeas ``mv`` command. + </xsd:documentation> + </xsd:annotation> + <xsd:attribute type="xsd:string" name="source" use="required"> + <xsd:annotation> + <xsd:documentation> + Move the node matching this path expression. ``source`` + must match exactly one node. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="destination" use="required"> + <xsd:annotation> + <xsd:documentation> + Move the node to this location. ``destination`` must match + either zero or one nodes. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name="AugeasSetCommand"> + <xsd:annotation> + <xsd:documentation> + Implementation of the Augeas ``set`` command. + </xsd:documentation> + </xsd:annotation> + <xsd:attribute type="xsd:string" name="path" use="required"> + <xsd:annotation> + <xsd:documentation> + Path to set the value for. If the path does not exist, it + and all of its ancestors will be created. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="value" use="required"> + <xsd:annotation> + <xsd:documentation> + Value to set. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name="AugeasClearCommand"> + <xsd:annotation> + <xsd:documentation> + Implementation of the Augeas ``clear`` command. + </xsd:documentation> + </xsd:annotation> + <xsd:attribute type="xsd:string" name="path" use="required"> + <xsd:annotation> + <xsd:documentation> + Path whose value will be set to ``NULL``. If the path does + not exist, it and all of its ancestors will be created. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name="AugeasSetMultiCommand"> + <xsd:annotation> + <xsd:documentation> + Set multiple node values at once. + </xsd:documentation> + </xsd:annotation> + <xsd:attribute type="xsd:string" name="base" use="required"> + <xsd:annotation> + <xsd:documentation> + The base path. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="sub" use="required"> + <xsd:annotation> + <xsd:documentation> + ``sub`` will be used as an expression relative to each node + that matches the :xml:attribute:`AugeasSetMultiCommand:base` + expression. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="value" use="required"> + <xsd:annotation> + <xsd:documentation> + The value to set on all nodes that match + :xml:attribute:`AugeasSetMultiCommand:sub` relative to each + node matching :xml:attribute:`AugeasSetMultiCommand:base`. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:simpleType name="AugeasWhenEnum"> + <xsd:restriction base="xsd:string"> + <xsd:enumeration value="before"/> + <xsd:enumeration value="after"/> + </xsd:restriction> + </xsd:simpleType> + + <xsd:complexType name="AugeasInsertCommand"> + <xsd:annotation> + <xsd:documentation> + Implementation of the Augeas ``ins`` command. + </xsd:documentation> + </xsd:annotation> + <xsd:attribute type="xsd:string" name="path" use="required"> + <xsd:annotation> + <xsd:documentation> + The path to a node that will be the sibling of the new node. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="label" use="required"> + <xsd:annotation> + <xsd:documentation> + The label of the new node to be created. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="AugeasWhenEnum" name="where" default="before"> + <xsd:annotation> + <xsd:documentation> + Where to create the node: ``before`` or ``after`` the + sibling given in :xml:attribute:`AugeasInsertCommand:path`. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:group name="augeasCommands"> + <xsd:annotation> + <xsd:documentation> + All available Augeas commands. + </xsd:documentation> + </xsd:annotation> + <xsd:choice> + <xsd:element name="Initial" type="xsd:string"> + <xsd:annotation> + <xsd:documentation> + Specify initial content for a file, which will be created + before Augeas commands are applied if a file doesn't + exist. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name="Remove" type="AugeasRemoveCommand"> + <xsd:annotation> + <xsd:documentation> + Implementation of the Augeas ``rm`` command. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name="Move" type="AugeasMoveCommand"> + <xsd:annotation> + <xsd:documentation> + Implementation of the Augeas ``mv`` command. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name="Set" type="AugeasSetCommand"> + <xsd:annotation> + <xsd:documentation> + Implementation of the Augeas ``set`` command. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name="Clear" type="AugeasClearCommand"> + <xsd:annotation> + <xsd:documentation> + Implementation of the Augeas ``clear`` command. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name="SetMulti" type="AugeasSetMultiCommand"> + <xsd:annotation> + <xsd:documentation> + Set multiple node values at once. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name="Insert" type="AugeasInsertCommand"> + <xsd:annotation> + <xsd:documentation> + Implementation of the Augeas ``ins`` command. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + </xsd:choice> + </xsd:group> +</xsd:schema> diff --git a/schemas/packages.xsd b/schemas/packages.xsd index e57280527..ae7b0430a 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -17,6 +17,7 @@ <xsd:enumeration value="yum"/> <xsd:enumeration value="apt"/> <xsd:enumeration value="pac"/> + <xsd:enumeration value="pkgng"/> </xsd:restriction> </xsd:simpleType> diff --git a/schemas/pathentry.xsd b/schemas/pathentry.xsd index e5d2ef6af..44c86f9bc 100644 --- a/schemas/pathentry.xsd +++ b/schemas/pathentry.xsd @@ -12,7 +12,34 @@ schemaLocation="genshi.xsd"/> <xsd:complexType name='PathEntry'> - <xsd:attribute type='xsd:string' name='name' use='required'/> + <xsd:annotation> + <xsd:documentation> + Abstract description of a path to be installed. This can + either be a single explicit path (e.g., ``<Path + name="/etc/foo.conf"/>``) or a glob that matches a set of + paths (e.g., ``<Path glob="/etc/foo/*"/>``). Path + globbing may not work for some dynamically handled Path + entries, for instance :ref:`Packages client configs + <generating-client-configs>`. + </xsd:documentation> + </xsd:annotation> + <xsd:attribute type='xsd:string' name='name'> + <xsd:annotation> + <xsd:documentation> + Install the single named path. Either ``name`` or + :xml:attribute:`PathEntry:glob` must be specified. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="glob"> + <xsd:annotation> + <xsd:documentation> + Install all Cfg entries matching the given glob. Either + ``glob`` or :xml:attribute:`PathEntry:name` must be + specified. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attribute type='xsd:string' name='altsrc' use='optional'/> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/pkgtype.xsd b/schemas/pkgtype.xsd index c76c52824..7ad7606b2 100644 --- a/schemas/pkgtype.xsd +++ b/schemas/pkgtype.xsd @@ -54,6 +54,15 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute name="recommended" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Whether also the recommended packages should be installed. + This is currently only used with the :ref:`APT + <client-tools-apt>` driver. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/types.xsd b/schemas/types.xsd index c91a66110..0a55f6355 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -9,6 +9,7 @@ </xsd:annotation> <xsd:include schemaLocation="selinux.xsd"/> + <xsd:include schemaLocation="augeas.xsd"/> <xsd:import namespace="http://genshi.edgewall.org/" schemaLocation="genshi.xsd"/> @@ -41,6 +42,7 @@ <xsd:simpleType name='PathTypeEnum'> <xsd:restriction base='xsd:string'> + <xsd:enumeration value='augeas' /> <xsd:enumeration value='device' /> <xsd:enumeration value='directory' /> <xsd:enumeration value='file' /> @@ -191,6 +193,7 @@ <xsd:restriction base="xsd:string"> <xsd:enumeration value="user"/> <xsd:enumeration value="group"/> + <xsd:enumeration value="other"/> </xsd:restriction> </xsd:simpleType> @@ -260,6 +263,8 @@ <xsd:choice minOccurs='0' maxOccurs='unbounded'> <xsd:element name='ACL' type='ACLType'/> + <xsd:group ref="augeasCommands"/> + <xsd:group ref="py:genshiElements"/> </xsd:choice> <xsd:attribute type="PathTypeEnum" name="type"> <xsd:annotation> @@ -395,6 +400,14 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="lens"> + <xsd:annotation> + <xsd:documentation> + The Augeas lens to use when editing files in a non-standard + (according to Augeas) location. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -474,9 +487,9 @@ <xsd:annotation> <xsd:documentation> This field is typically used to record general information - about the account or its user(s) such as their real name - and phone number. If this is not set, the GECOS will be - the same as the username. + about the account or its user(s) such as their real name + and phone number. If this is not set, the GECOS will be + the same as the username. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/solaris-ips/MANIFEST.bcfg2-server.header b/solaris-ips/MANIFEST.bcfg2-server.header index 59929fcfa..5a837b5c7 100644 --- a/solaris-ips/MANIFEST.bcfg2-server.header +++ b/solaris-ips/MANIFEST.bcfg2-server.header @@ -1,4 +1,4 @@ license ../../LICENSE license=simplified_bsd set name=description value="Configuration management server" set name=pkg.summary value="Configuration management server" -set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@1.3.3" +set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@1.4.0pre1" diff --git a/solaris-ips/MANIFEST.bcfg2.header b/solaris-ips/MANIFEST.bcfg2.header index 5f48a60a1..5fd5c0c5d 100644 --- a/solaris-ips/MANIFEST.bcfg2.header +++ b/solaris-ips/MANIFEST.bcfg2.header @@ -1,5 +1,5 @@ license ../../LICENSE license=simplified_bsd set name=description value="Configuration management client" set name=pkg.summary value="Configuration management client" -set name=pkg.fmri value="pkg://bcfg2/bcfg2@1.3.3" +set name=pkg.fmri value="pkg://bcfg2/bcfg2@1.4.0pre1" file usr/bin/bcfg2 group=bin mode=0755 owner=root path=usr/bin/bcfg2 diff --git a/solaris-ips/Makefile b/solaris-ips/Makefile index 71523f48e..63a1ac61e 100644 --- a/solaris-ips/Makefile +++ b/solaris-ips/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/gmake -VERS=1.3.3-1 +VERS=1.4.0pre1-1 PYVERSION := $(shell python -c "import sys; print sys.version[0:3]") default: clean package diff --git a/solaris-ips/pkginfo.bcfg2 b/solaris-ips/pkginfo.bcfg2 index 00483f961..236009c8f 100644 --- a/solaris-ips/pkginfo.bcfg2 +++ b/solaris-ips/pkginfo.bcfg2 @@ -1,7 +1,7 @@ PKG="SCbcfg2" NAME="bcfg2" ARCH="sparc" -VERSION="1.3.3" +VERSION="1.4.0pre1" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris-ips/pkginfo.bcfg2-server b/solaris-ips/pkginfo.bcfg2-server index ecc5e72c1..af1ab36fc 100644 --- a/solaris-ips/pkginfo.bcfg2-server +++ b/solaris-ips/pkginfo.bcfg2-server @@ -1,7 +1,7 @@ PKG="SCbcfg2-server" NAME="bcfg2-server" ARCH="sparc" -VERSION="1.3.3" +VERSION="1.4.0pre1" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris/Makefile b/solaris/Makefile index 3b367ef71..cdf61d8f7 100644 --- a/solaris/Makefile +++ b/solaris/Makefile @@ -1,7 +1,7 @@ #!/usr/sfw/bin/gmake PYTHON="/usr/local/bin/python" -VERS=1.3.3-1 +VERS=1.4.0pre1-1 PYVERSION := $(shell $(PYTHON) -c "import sys; print sys.version[0:3]") default: clean package diff --git a/solaris/pkginfo.bcfg2 b/solaris/pkginfo.bcfg2 index 00483f961..236009c8f 100644 --- a/solaris/pkginfo.bcfg2 +++ b/solaris/pkginfo.bcfg2 @@ -1,7 +1,7 @@ PKG="SCbcfg2" NAME="bcfg2" ARCH="sparc" -VERSION="1.3.3" +VERSION="1.4.0pre1" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris/pkginfo.bcfg2-server b/solaris/pkginfo.bcfg2-server index ecc5e72c1..af1ab36fc 100644 --- a/solaris/pkginfo.bcfg2-server +++ b/solaris/pkginfo.bcfg2-server @@ -1,7 +1,7 @@ PKG="SCbcfg2-server" NAME="bcfg2-server" ARCH="sparc" -VERSION="1.3.3" +VERSION="1.4.0pre1" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py index a464d6a40..679b4c52b 100644 --- a/src/lib/Bcfg2/Client/Proxy.py +++ b/src/lib/Bcfg2/Client/Proxy.py @@ -12,13 +12,9 @@ from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus # M2Crypto instead. try: import ssl - SSL_LIB = 'py26_ssl' SSL_ERROR = ssl.SSLError except ImportError: - from M2Crypto import SSL - import M2Crypto.SSL.Checker - SSL_LIB = 'm2crypto' - SSL_ERROR = SSL.SSLError + raise Exception("No SSL module support") version = sys.version_info[:2] @@ -123,7 +119,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): """ def __init__(self, host, port=None, strict=None, timeout=90, key=None, - cert=None, ca=None, scns=None, protocol='xmlrpc/ssl'): + cert=None, ca=None, scns=None, protocol='xmlrpc/tlsv1'): """Initializes the `httplib.HTTPConnection` object and stores security parameters @@ -148,15 +144,15 @@ class SSLHTTPConnection(httplib.HTTPConnection): specify the same file as `cert` if using a file that contains both. See http://docs.python.org/library/ssl.html#ssl-certificates - for details. Required if using xmlrpc/ssl with client - certificate authentication. + for details. Required if using client certificate + authentication. cert : string, optional The file system path to the local endpoint's SSL certificate. May specify the same file as `cert` if using a file that contains both. See http://docs.python.org/library/ssl.html#ssl-certificates - for details. Required if using xmlrpc/ssl with client - certificate authentication. + for details. Required if using client certificate + authentication. ca : string, optional The file system path to a set of concatenated certificate authority certs, which are used to validate certificates @@ -187,15 +183,6 @@ class SSLHTTPConnection(httplib.HTTPConnection): self.timeout = timeout def connect(self): - """Initiates a connection using previously set attributes.""" - if SSL_LIB == 'py26_ssl': - self._connect_py26ssl() - elif SSL_LIB == 'm2crypto': - self._connect_m2crypto() - else: - raise Exception("No SSL module support") - - def _connect_py26ssl(self): """Initiates a connection using the ssl module.""" # check for IPv6 hostip = socket.getaddrinfo(self.host, @@ -242,60 +229,11 @@ class SSLHTTPConnection(httplib.HTTPConnection): raise CertificateError(scn) self.sock.closeSocket = True - def _connect_m2crypto(self): - """Initiates a connection using the M2Crypto module.""" - - if self.protocol == 'xmlrpc/ssl': - ctx = SSL.Context('sslv23') - elif self.protocol == 'xmlrpc/tlsv1': - ctx = SSL.Context('tlsv1') - else: - self.logger.error("Unknown protocol %s" % (self.protocol)) - raise Exception("unknown protocol %s" % self.protocol) - - if self.ca: - # Use the certificate authority to validate the cert - # presented by the server - ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, - depth=9) - if ctx.load_verify_locations(self.ca) != 1: - raise Exception('No CA certs') - else: - self.logger.warning("No ca is specified. Cannot authenticate the " - "server with SSL.") - - if self.cert and self.key: - # A cert/key is defined, use them to support client - # authentication to the server - ctx.load_cert(self.cert, self.key) - elif self.cert: - self.logger.warning("SSL cert specfied, but no key. Cannot " - "authenticate this client with SSL.") - elif self.key: - self.logger.warning("SSL key specfied, but no cert. Cannot " - "authenticate this client with SSL.") - - self.sock = SSL.Connection(ctx) - if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', self.host): - # host is ip address - try: - hostname = socket.gethostbyaddr(self.host)[0] - except: - # fall back to ip address - hostname = self.host - else: - hostname = self.host - try: - self.sock.connect((hostname, self.port)) - # automatically checks cert matches host - except M2Crypto.SSL.Checker.WrongHost: - wr = sys.exc_info()[1] - raise CertificateError(wr) - class XMLRPCTransport(xmlrpclib.Transport): def __init__(self, key=None, cert=None, ca=None, - scns=None, use_datetime=0, timeout=90): + scns=None, use_datetime=0, timeout=90, + protocol='xmlrpc/tlsv1'): if hasattr(xmlrpclib.Transport, '__init__'): xmlrpclib.Transport.__init__(self, use_datetime) self.key = key @@ -303,6 +241,7 @@ class XMLRPCTransport(xmlrpclib.Transport): self.ca = ca self.scns = scns self.timeout = timeout + self.protocol = protocol def make_connection(self, host): host, self._extra_headers = self.get_host_info(host)[0:2] @@ -311,7 +250,8 @@ class XMLRPCTransport(xmlrpclib.Transport): cert=self.cert, ca=self.ca, scns=self.scns, - timeout=self.timeout) + timeout=self.timeout, + protocol=self.protocol) def request(self, host, handler, request_body, verbose=0): """Send request to server and return response.""" @@ -354,9 +294,15 @@ class ComponentProxy(xmlrpclib.ServerProxy): """Constructs proxies to components. """ options = [ - Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_key, - Bcfg2.Options.Common.ssl_cert, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_ca, Bcfg2.Options.Common.password, Bcfg2.Options.Common.client_timeout, + Bcfg2.Options.Common.protocol, + Bcfg2.Options.PathOption( + '--ssl-key', cf=('communication', 'key'), dest="key", + help='Path to SSL key'), + Bcfg2.Options.PathOption( + cf=('communication', 'certificate'), dest="cert", + help='Path to SSL certificate'), Bcfg2.Options.Option( "-u", "--user", default="root", cf=('communication', 'user'), help='The user to provide for authentication'), @@ -386,10 +332,12 @@ class ComponentProxy(xmlrpclib.ServerProxy): path) else: url = Bcfg2.Options.setup.server - ssl_trans = XMLRPCTransport(Bcfg2.Options.setup.key, - Bcfg2.Options.setup.cert, - Bcfg2.Options.setup.ca, - Bcfg2.Options.setup.ssl_cns, - Bcfg2.Options.setup.client_timeout) + ssl_trans = XMLRPCTransport( + key=Bcfg2.Options.setup.key, + cert=Bcfg2.Options.setup.cert, + ca=Bcfg2.Options.setup.ca, + scns=Bcfg2.Options.setup.ssl_cns, + timeout=Bcfg2.Options.setup.client_timeout, + protocol=Bcfg2.Options.setup.protocol) xmlrpclib.ServerProxy.__init__(self, url, allow_none=True, transport=ssl_trans) diff --git a/src/lib/Bcfg2/Client/Tools/APK.py b/src/lib/Bcfg2/Client/Tools/APK.py index 457197c28..7313f6fcc 100644 --- a/src/lib/Bcfg2/Client/Tools/APK.py +++ b/src/lib/Bcfg2/Client/Tools/APK.py @@ -25,7 +25,7 @@ class APK(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): """Verify Package status for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % entry.attrib['name']) return False @@ -33,7 +33,7 @@ class APK(Bcfg2.Client.Tools.PkgTool): if entry.attrib['name'] in self.installed: if entry.attrib['version'] in \ ['auto', self.installed[entry.attrib['name']]]: - #FIXME: Does APK have any sort of verification mechanism? + # FIXME: Does APK have any sort of verification mechanism? return True else: self.logger.info(" pkg %s at version %s, not %s" % diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index 5549b1717..dedc50d89 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -36,7 +36,7 @@ class Action(Bcfg2.Client.Tools.Tool): shell = True shell_string = '(in shell) ' - if not Bcfg2.Options.setup.dryrun: + if not Bcfg2.Options.setup.dry_run: if Bcfg2.Options.setup.interactive: prompt = ('Run Action %s%s, %s: (y/N): ' % (shell_string, entry.get('name'), diff --git a/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py index 2ab64f86d..24bc4cf36 100644 --- a/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py @@ -1,27 +1,143 @@ """FreeBSD Init Support for Bcfg2.""" -__revision__ = '$Rev$' - -# TODO -# - hardcoded path to ports rc.d -# - doesn't know about /etc/rc.d/ import os +import re +import Bcfg2.Options import Bcfg2.Client.Tools class FreeBSDInit(Bcfg2.Client.Tools.SvcTool): """FreeBSD service support for Bcfg2.""" name = 'FreeBSDInit' + __execs__ = ['/usr/sbin/service', '/usr/sbin/sysrc'] __handles__ = [('Service', 'freebsd')] __req__ = {'Service': ['name', 'status']} + rcvar_re = re.compile(r'^(?P<var>[a-z_]+_enable)="[A-Z]+"$') - def __init__(self, config): - Bcfg2.Client.Tools.SvcTool.__init__(self, config) - if os.uname()[0] != 'FreeBSD': - raise Bcfg2.Client.Tools.ToolInstantiationError + def get_svc_command(self, service, action): + return '/usr/sbin/service %s %s' % (service.get('name'), action) - def VerifyService(self, entry, _): + def verify_bootstatus(self, entry, bootstatus): + """Verify bootstatus for entry.""" + cmd = self.get_svc_command(entry, 'enabled') + current_bootstatus = bool(self.cmd.run(cmd)) + + if bootstatus == 'off': + if current_bootstatus: + entry.set('current_bootstatus', 'on') + return False + return True + elif not current_bootstatus: + entry.set('current_bootstatus', 'off') + return False return True - def get_svc_command(self, service, action): - return "/usr/local/etc/rc.d/%s %s" % (service.get('name'), action) + def check_service(self, entry): + # use 'onestatus' to enable status reporting for disabled services + cmd = self.get_svc_command(entry, 'onestatus') + return bool(self.cmd.run(cmd)) + + def stop_service(self, service): + # use 'onestop' to enable stopping of disabled services + self.logger.debug('Stopping service %s' % service.get('name')) + return self.cmd.run(self.get_svc_command(service, 'onestop')) + + + def VerifyService(self, entry, _): + """Verify Service status for entry.""" + entry.set('target_status', entry.get('status')) # for reporting + bootstatus = self.get_bootstatus(entry) + if bootstatus is None: + return True + current_bootstatus = self.verify_bootstatus(entry, bootstatus) + + if entry.get('status') == 'ignore': + # 'ignore' should verify + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True + + if svcstatus: + entry.set('current_status', 'on') + else: + entry.set('current_status', 'off') + + return current_bootstatus and current_svcstatus + + def InstallService(self, entry): + """Install Service entry.""" + self.logger.info("Installing Service %s" % (entry.get('name'))) + bootstatus = self.get_bootstatus(entry) + + # check if service exists + all_services_cmd = '/usr/sbin/service -l' + all_services = self.cmd.run(all_services_cmd).stdout.splitlines() + if entry.get('name') not in all_services: + self.logger.debug("Service %s does not exist" % entry.get('name')) + return False + + # get rcvar for service + vars = set() + rcvar_cmd = self.get_svc_command(entry, 'rcvar') + for line in self.cmd.run(rcvar_cmd).stdout.splitlines(): + match = self.rcvar_re.match(line) + if match: + vars.add(match.group('var')) + + if bootstatus is not None: + bootcmdrv = True + sysrcstatus = None + if bootstatus == 'on': + sysrcstatus = 'YES' + elif bootstatus == 'off': + sysrcstatus = 'NO' + if sysrcstatus is not None: + for var in vars: + if not self.cmd.run('/usr/sbin/sysrc %s="%s"' % (var, sysrcstatus)): + bootcmdrv = False + break + + if Bcfg2.Options.setup.service_mode == 'disabled': + # 'disabled' means we don't attempt to modify running svcs + return bootcmdrv + buildmode = Bcfg2.Options.setup.service_mode == 'build' + if (entry.get('status') == 'on' and not buildmode) and \ + entry.get('current_status') == 'off': + svccmdrv = self.start_service(entry) + elif (entry.get('status') == 'off' or buildmode) and \ + entry.get('current_status') == 'on': + svccmdrv = self.stop_service(entry) + else: + svccmdrv = True # ignore status attribute + return bootcmdrv and svccmdrv + else: + # when bootstatus is 'None', status == 'ignore' + return True + + def FindExtra(self): + """Find Extra FreeBSD Service entries.""" + specified = [entry.get('name') for entry in self.getSupportedEntries()] + extra = set() + for path in self.cmd.run("/usr/sbin/service -e").stdout.splitlines(): + name = os.path.basename(path) + if name not in specified: + extra.add(name) + return [Bcfg2.Client.XML.Element('Service', name=name, type='freebsd') + for name in list(extra)] + + def Remove(self, _): + """Remove extra service entries.""" + # Extra service removal is nonsensical + # Extra services need to be reflected in the config + return diff --git a/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py index 31925fa3c..22cf802cf 100644 --- a/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py @@ -29,7 +29,7 @@ class FreeBSDPackage(Bcfg2.Client.Tools.PkgTool): self.installed[name] = version def VerifyPackage(self, entry, _): - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % entry.attrib['name']) return False diff --git a/src/lib/Bcfg2/Client/Tools/IPS.py b/src/lib/Bcfg2/Client/Tools/IPS.py index c998ff083..0f82b1bc1 100644 --- a/src/lib/Bcfg2/Client/Tools/IPS.py +++ b/src/lib/Bcfg2/Client/Tools/IPS.py @@ -37,7 +37,7 @@ class IPS(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): """Verify package for entry.""" pname = entry.get('name') - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (pname)) return False if pname not in self.installed: diff --git a/src/lib/Bcfg2/Client/Tools/MacPorts.py b/src/lib/Bcfg2/Client/Tools/MacPorts.py index 265171a5a..1e9847c42 100644 --- a/src/lib/Bcfg2/Client/Tools/MacPorts.py +++ b/src/lib/Bcfg2/Client/Tools/MacPorts.py @@ -31,16 +31,16 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): """Verify Package status for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % entry.attrib['name']) return False if entry.attrib['name'] in self.installed: if (self.installed[entry.attrib['name']] == entry.attrib['version'] - or entry.attrib['version'] == 'any'): - #FIXME: We should be able to check this once - # http://trac.macports.org/ticket/15709 is implemented + or entry.attrib['version'] == 'any'): + # FIXME: We should be able to check this once + # http://trac.macports.org/ticket/15709 is implemented return True else: self.logger.info(" %s: Wrong version installed. " diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py new file mode 100644 index 000000000..fc4e16904 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py @@ -0,0 +1,296 @@ +""" Augeas driver """ + +import sys +import Bcfg2.Client.XML +from augeas import Augeas +from Bcfg2.Client.Tools.POSIX.base import POSIXTool +from Bcfg2.Client.Tools.POSIX.File import POSIXFile + + +class AugeasCommand(object): + """ Base class for all Augeas command objects """ + + def __init__(self, command, augeas_obj, logger): + self._augeas = augeas_obj + self.command = command + self.entry = self.command.getparent() + self.logger = logger + + def get_path(self, attr="path"): + """ Get a fully qualified path from the name of the parent entry and + the path given in this command tag. + + @param attr: The attribute to get the relative path from + @type attr: string + @returns: string - the fully qualified Augeas path + + """ + return "/files/%s/%s" % (self.entry.get("name").strip("/"), + self.command.get(attr).lstrip("/")) + + def _exists(self, path): + """ Return True if a path exists in Augeas, False otherwise. + + Note that a False return can mean many things: A file that + doesn't exist, a node within the file that doesn't exist, no + lens to parse the file, etc. """ + return len(self._augeas.match(path)) > 1 + + def _verify_exists(self, path=None): + """ Verify that the given path exists, with friendly debug + logging. + + @param path: The path to verify existence of. Defaults to the + result of + :func:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand.getpath`. + @type path: string + @returns: bool - Whether or not the path exists + """ + if path is None: + path = self.get_path() + self.logger.debug("Augeas: Verifying that '%s' exists" % path) + return self._exists(path) + + def _verify_not_exists(self, path=None): + """ Verify that the given path does not exist, with friendly + debug logging. + + @param path: The path to verify existence of. Defaults to the + result of + :func:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand.getpath`. + @type path: string + @returns: bool - Whether or not the path does not exist. + (I.e., True if it does not exist, False if it does + exist.) + """ + if path is None: + path = self.get_path() + self.logger.debug("Augeas: Verifying that '%s' does not exist" % path) + return not self._exists(path) + + def _verify_set(self, expected, path=None): + """ Verify that the given path is set to the given value, with + friendly debug logging. + + @param expected: The expected value of the node. + @param path: The path to verify existence of. Defaults to the + result of + :func:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand.getpath`. + @type path: string + @returns: bool - Whether or not the path matches the expected value. + + """ + if path is None: + path = self.get_path() + self.logger.debug("Augeas: Verifying '%s' == '%s'" % (path, expected)) + actual = self._augeas.get(path) + if actual == expected: + return True + else: + self.logger.debug("Augeas: '%s' failed verification: '%s' != '%s'" + % (path, actual, expected)) + return False + + def __str__(self): + return Bcfg2.Client.XML.tostring(self.command) + + def verify(self): + """ Verify that the command has been applied. """ + raise NotImplementedError + + def install(self): + """ Run the command. """ + raise NotImplementedError + + +class Remove(AugeasCommand): + """ Augeas ``rm`` command """ + def verify(self): + return self._verify_not_exists() + + def install(self): + self.logger.debug("Augeas: Removing %s" % self.get_path()) + return self._augeas.remove(self.get_path()) + + +class Move(AugeasCommand): + """ Augeas ``move`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.source = self.get_path("source") + self.dest = self.get_path("destination") + + def verify(self): + return (self._verify_not_exists(self.source), + self._verify_exists(self.dest)) + + def install(self): + self.logger.debug("Augeas: Moving %s to %s" % (self.source, self.dest)) + return self._augeas.move(self.source, self.dest) + + +class Set(AugeasCommand): + """ Augeas ``set`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.value = self.command.get("value") + + def verify(self): + return self._verify_set(self.value) + + def install(self): + self.logger.debug("Augeas: Setting %s to %s" % (self.get_path(), + self.value)) + return self._augeas.set(self.get_path(), self.value) + + +class Clear(Set): + """ Augeas ``clear`` command """ + def __init__(self, command, augeas_obj, logger): + Set.__init__(self, command, augeas_obj, logger) + self.value = None + + +class SetMulti(AugeasCommand): + """ Augeas ``setm`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.sub = self.command.get("sub") + self.value = self.command.get("value") + self.base = self.get_path("base") + + def verify(self): + return all(self._verify_set(self.value, + path="%s/%s" % (path, self.sub)) + for path in self._augeas.match(self.base)) + + def install(self): + return self._augeas.setm(self.base, self.sub, self.value) + + +class Insert(AugeasCommand): + """ Augeas ``ins`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.label = self.command.get("label") + self.where = self.command.get("where", "before") + self.before = self.where == "before" + + def verify(self): + return self._verify_exists("%s/../%s" % (self.get_path(), self.label)) + + def install(self): + self.logger.debug("Augeas: Inserting new %s %s %s" % + (self.label, self.where, self.get_path())) + return self._augeas.insert(self.get_path(), self.label, self.before) + + +class POSIXAugeas(POSIXTool): + """ Handle <Path type='augeas'...> entries. See + :ref:`client-tools-augeas`. """ + __req__ = ['name', 'mode', 'owner', 'group'] + + def __init__(self, config): + POSIXTool.__init__(self, config) + self._augeas = dict() + # file tool for setting initial values of files that don't + # exist + self.filetool = POSIXFile(config) + + def get_augeas(self, entry): + """ Get an augeas object for the given entry. """ + if entry.get("name") not in self._augeas: + aug = Augeas() + if entry.get("lens"): + self.logger.debug("Augeas: Adding %s to include path for %s" % + (entry.get("name"), entry.get("lens"))) + incl = "/augeas/load/%s/incl" % entry.get("lens") + ilen = len(aug.match(incl)) + if ilen == 0: + self.logger.error("Augeas: Lens %s does not exist" % + entry.get("lens")) + else: + aug.set("%s[%s]" % (incl, ilen + 1), entry.get("name")) + aug.load() + self._augeas[entry.get("name")] = aug + return self._augeas[entry.get("name")] + + def fully_specified(self, entry): + return len(entry.getchildren()) != 0 + + def get_commands(self, entry): + """ Get a list of commands to verify or install. + + @param entry: The entry to get commands from. + @type entry: lxml.etree._Element + @param unverified: Only get commands that failed verification. + @type unverified: bool + @returns: list of + :class:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand` + objects representing the commands. + """ + rv = [] + for cmd in entry.iterchildren(): + if cmd.tag == "Initial": + continue + if cmd.tag in globals(): + rv.append(globals()[cmd.tag](cmd, self.get_augeas(entry), + self.logger)) + else: + err = "Augeas: Unknown command %s in %s" % (cmd.tag, + entry.get("name")) + self.logger.error(err) + entry.set('qtext', "\n".join([entry.get('qtext', ''), err])) + return rv + + def verify(self, entry, modlist): + rv = True + for cmd in self.get_commands(entry): + try: + if not cmd.verify(): + err = "Augeas: Command has not been applied to %s: %s" % \ + (entry.get("name"), cmd) + self.logger.debug(err) + entry.set('qtext', "\n".join([entry.get('qtext', ''), + err])) + rv = False + cmd.command.set("verified", "false") + else: + cmd.command.set("verified", "true") + except: # pylint: disable=W0702 + err = "Augeas: Unexpected error verifying %s: %s: %s" % \ + (entry.get("name"), cmd, sys.exc_info()[1]) + self.logger.error(err) + entry.set('qtext', "\n".join([entry.get('qtext', ''), err])) + rv = False + cmd.command.set("verified", "false") + return POSIXTool.verify(self, entry, modlist) and rv + + def install(self, entry): + rv = True + if entry.get("current_exists", "true") == "false": + initial = entry.find("Initial") + if initial is not None: + self.logger.debug("Augeas: Setting initial data for %s" % + entry.get("name")) + file_entry = Bcfg2.Client.XML.Element("Path", + **dict(entry.attrib)) + file_entry.text = initial.text + self.filetool.install(file_entry) + # re-parse the file + self.get_augeas(entry).load() + for cmd in self.get_commands(entry): + try: + cmd.install() + except: # pylint: disable=W0702 + self.logger.error( + "Failure running Augeas command on %s: %s: %s" % + (entry.get("name"), cmd, sys.exc_info()[1])) + rv = False + try: + self.get_augeas(entry).save() + except: # pylint: disable=W0702 + self.logger.error("Failure saving Augeas changes to %s: %s" % + (entry.get("name"), sys.exc_info()[1])) + rv = False + return POSIXTool.install(self, entry) and rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index d7a70e202..0452ea258 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -3,7 +3,6 @@ import os import sys import stat -import time import difflib import tempfile import Bcfg2.Options @@ -189,12 +188,11 @@ class POSIXFile(POSIXTool): prompt.append('Binary file, no printable diff') attrs['current_bfile'] = b64encode(content) else: + diff = self._diff(content, self._get_data(entry)[0], + filename=entry.get("name")) if interactive: - diff = self._diff(content, self._get_data(entry)[0], - difflib.unified_diff, - filename=entry.get("name")) if diff: - udiff = '\n'.join(l.rstrip('\n') for l in diff) + udiff = '\n'.join(diff) if hasattr(udiff, "decode"): udiff = udiff.decode(Bcfg2.Options.setup.encoding) try: @@ -209,8 +207,6 @@ class POSIXFile(POSIXTool): prompt.append("Diff took too long to compute, no " "printable diff") if not sensitive: - diff = self._diff(content, self._get_data(entry)[0], - difflib.ndiff, filename=entry.get("name")) if diff: attrs["current_bdiff"] = b64encode("\n".join(diff)) else: @@ -221,28 +217,12 @@ class POSIXFile(POSIXTool): for attr, val in attrs.items(): entry.set(attr, val) - def _diff(self, content1, content2, difffunc, filename=None): - """ Return a diff of the two strings, as produced by difffunc. - warns after 5 seconds and times out after 30 seconds. """ - rv = [] - start = time.time() - longtime = False - for diffline in difffunc(content1.split('\n'), - content2.split('\n')): - now = time.time() - rv.append(diffline) - if now - start > 5 and not longtime: - if filename: - self.logger.info("POSIX: Diff of %s taking a long time" % - filename) - else: - self.logger.info("POSIX: Diff taking a long time") - longtime = True - elif now - start > 30: - if filename: - self.logger.error("POSIX: Diff of %s took too long; " - "giving up" % filename) - else: - self.logger.error("POSIX: Diff took too long; giving up") - return False - return rv + def _diff(self, content1, content2, filename=None): + """ Return a unified diff of the two strings """ + + fromfile = "%s (on disk)" % filename if filename else "" + tofile = "%s (from bcfg2)" % filename if filename else "" + return difflib.unified_diff(content1.split('\n'), + content2.split('\n'), + fromfile=fromfile, + tofile=tofile) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py index 13b45a759..c27c7559d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py @@ -58,8 +58,11 @@ class POSIX(Bcfg2.Client.Tools.Tool): mname = submodule[1].rsplit('.', 1)[-1] if mname == 'base': continue - module = getattr(__import__(submodule[1]).Client.Tools.POSIX, - mname) + try: + module = getattr(__import__(submodule[1]).Client.Tools.POSIX, + mname) + except ImportError: + continue hdlr = getattr(module, "POSIX" + mname) if POSIXTool in hdlr.__mro__: # figure out what entry type this handler handles diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 712620206..8895eaae1 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -217,18 +217,13 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): acl.delete_entry(aclentry) if os.path.isdir(path): defacl = posix1e.ACL(filedef=path) - if not defacl.valid(): - # when a default ACL is queried on a directory that - # has no default ACL entries at all, you get an empty - # ACL, which is not valid. in this circumstance, we - # just copy the access ACL to get a base valid ACL - # that we can add things to. - defacl = posix1e.ACL(acl=acl) - else: - for aclentry in defacl: - if aclentry.tag_type in [posix1e.ACL_USER, - posix1e.ACL_GROUP]: - defacl.delete_entry(aclentry) + for aclentry in defacl: + if aclentry.tag_type in [posix1e.ACL_USER, + posix1e.ACL_USER_OBJ, + posix1e.ACL_GROUP, + posix1e.ACL_GROUP_OBJ, + posix1e.ACL_OTHER]: + defacl.delete_entry(aclentry) else: defacl = None @@ -254,10 +249,16 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): try: if scope == posix1e.ACL_USER: scopename = "user" - aclentry.qualifier = self._norm_uid(qualifier) + if qualifier: + aclentry.qualifier = self._norm_uid(qualifier) + else: + aclentry.tag_type = posix1e.ACL_USER_OBJ elif scope == posix1e.ACL_GROUP: scopename = "group" - aclentry.qualifier = self._norm_gid(qualifier) + if qualifier: + aclentry.qualifier = self._norm_gid(qualifier) + else: + aclentry.tag_type = posix1e.ACL_GROUP_OBJ except (OSError, KeyError): err = sys.exc_info()[1] self.logger.error("POSIX: Could not resolve %s %s: %s" % @@ -358,7 +359,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): try: # single octal digit rv = int(perms) - if rv > 0 and rv < 8: + if rv >= 0 and rv < 8: return rv else: self.logger.error("POSIX: Permissions digit out of range in " @@ -388,13 +389,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Get a string representation of the given ACL. aclkey must be a tuple of (<acl type>, <acl scope>, <qualifier>) """ atype, scope, qualifier = aclkey + if not qualifier: + qualifier = '' acl_str = [] if atype == 'default': acl_str.append(atype) - if scope == posix1e.ACL_USER: + if scope == posix1e.ACL_USER or scope == posix1e.ACL_USER_OBJ: acl_str.append("user") - elif scope == posix1e.ACL_GROUP: + elif scope == posix1e.ACL_GROUP or scope == posix1e.ACL_GROUP_OBJ: acl_str.append("group") + elif scope == posix1e.ACL_OTHER: + acl_str.append("other") acl_str.append(qualifier) acl_str.append(self._acl_perm2string(perms)) return ":".join(acl_str) @@ -414,7 +419,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Get data on the existing state of <path> -- e.g., whether or not it exists, owner, group, permissions, etc. """ try: - ondisk = os.stat(path) + ondisk = os.lstat(path) except OSError: self.logger.debug("POSIX: %s does not exist" % path) return (False, None, None, None, None, None) @@ -451,7 +456,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if HAS_SELINUX: try: - secontext = selinux.getfilecon(path)[1].split(":")[2] + secontext = selinux.lgetfilecon(path)[1].split(":")[2] except (OSError, KeyError): err = sys.exc_info()[1] self.logger.debug("POSIX: Could not get current SELinux " @@ -460,7 +465,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: secontext = None - if HAS_ACLS: + if HAS_ACLS and not stat.S_ISLNK(ondisk[stat.ST_MODE]): acls = self._list_file_acls(path) else: acls = None @@ -562,9 +567,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): wanted = dict() for acl in entry.findall("ACL"): if acl.get("scope") == "user": - scope = posix1e.ACL_USER + if acl.get("user"): + scope = posix1e.ACL_USER + else: + scope = posix1e.ACL_USER_OBJ elif acl.get("scope") == "group": - scope = posix1e.ACL_GROUP + if acl.get("group"): + scope = posix1e.ACL_GROUP + else: + scope = posix1e.ACL_GROUP_OBJ + elif acl.get("scope") == "other": + scope = posix1e.ACL_OTHER else: self.logger.error("POSIX: Unknown ACL scope %s" % acl.get("scope")) @@ -573,7 +586,10 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): self.logger.error("POSIX: No permissions set for ACL: %s" % Bcfg2.Client.XML.tostring(acl)) continue - wanted[(acl.get("type"), scope, acl.get(acl.get("scope")))] = \ + qual = acl.get(acl.get("scope")) + if not qual: + qual = '' + wanted[(acl.get("type"), scope, qual)] = \ self._norm_acl_perms(acl.get('perms')) return wanted @@ -587,11 +603,12 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Given an ACL object, process it appropriately and add it to the return value """ try: + qual = '' if acl.tag_type == posix1e.ACL_USER: qual = pwd.getpwuid(acl.qualifier)[0] elif acl.tag_type == posix1e.ACL_GROUP: qual = grp.getgrgid(acl.qualifier)[0] - else: + elif atype == "access" or acl.tag_type == posix1e.ACL_MASK: return except (OSError, KeyError): err = sys.exc_info()[1] @@ -621,9 +638,38 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): _process_acl(acl, "default") return existing - def _verify_acls(self, entry, path=None): + def _verify_acls(self, entry, path=None): # pylint: disable=R0912 """ verify POSIX ACLs on the given entry. return True if all ACLS are correct, false otherwise """ + def _verify_acl(aclkey, perms): + """ Given ACL data, process it appropriately and add it to + missing or wrong lists if appropriate """ + if aclkey not in existing: + missing.append(self._acl2string(aclkey, perms)) + elif existing[aclkey] != perms: + wrong.append((self._acl2string(aclkey, perms), + self._acl2string(aclkey, existing[aclkey]))) + if path == entry.get("name"): + atype, scope, qual = aclkey + aclentry = Bcfg2.Client.XML.Element("ACL", type=atype, + perms=str(perms)) + if (scope == posix1e.ACL_USER or + scope == posix1e.ACL_USER_OBJ): + aclentry.set("scope", "user") + elif (scope == posix1e.ACL_GROUP or + scope == posix1e.ACL_GROUP_OBJ): + aclentry.set("scope", "group") + elif scope == posix1e.ACL_OTHER: + aclentry.set("scope", "other") + else: + self.logger.debug("POSIX: Unknown ACL scope %s on %s" % + (scope, path)) + return + + if scope != posix1e.ACL_OTHER: + aclentry.set(aclentry.get("scope"), qual) + entry.append(aclentry) + if not HAS_ACLS: if entry.findall("ACL"): self.logger.debug("POSIX: ACLs listed for %s but no pylibacl " @@ -644,25 +690,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): extra = [] wrong = [] for aclkey, perms in wanted.items(): - if aclkey not in existing: - missing.append(self._acl2string(aclkey, perms)) - elif existing[aclkey] != perms: - wrong.append((self._acl2string(aclkey, perms), - self._acl2string(aclkey, existing[aclkey]))) - if path == entry.get("name"): - atype, scope, qual = aclkey - aclentry = Bcfg2.Client.XML.Element("ACL", type=atype, - perms=str(perms)) - if scope == posix1e.ACL_USER: - aclentry.set("scope", "user") - elif scope == posix1e.ACL_GROUP: - aclentry.set("scope", "group") - else: - self.logger.debug("POSIX: Unknown ACL scope %s on %s" % - (scope, path)) - continue - aclentry.set(aclentry.get("scope"), qual) - entry.append(aclentry) + _verify_acl(aclkey, perms) for aclkey, perms in existing.items(): if aclkey not in wanted: diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 58a3bbdfc..a7fcb6709 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -79,7 +79,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): defined, and the uid/gid is in that whitelist; or b) no whitelist is defined, and the uid/gid is not in the blacklist. """ - if self._whitelist[tag] is None: + if not self._whitelist[tag]: return eid not in self._blacklist[tag] else: return eid in self._whitelist[tag] @@ -160,7 +160,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group")] + if entry.get("name") in g[3] and g[0] != entry.get("group") + and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): """ Verify a POSIXUser entry """ diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index 2ab9b7403..b82b905e7 100644 --- a/src/lib/Bcfg2/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py @@ -19,7 +19,6 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): for pkg in self.cmd.run("/usr/bin/pacman -Q").stdout.splitlines(): pkgname = pkg.split(' ')[0].strip() version = pkg.split(' ')[1].strip() - #self.logger.info(" pkgname: %s, version: %s" % (pkgname, version)) self.installed[pkgname] = version def VerifyPackage(self, entry, _): @@ -28,7 +27,7 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): self.logger.info("VerifyPackage: %s : %s" % (entry.get('name'), entry.get('version'))) - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % entry.attrib['name']) return False @@ -38,8 +37,8 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): return True elif self.installed[entry.attrib['name']] == \ entry.attrib['version']: - #FIXME: need to figure out if pacman - # allows you to verify packages + # FIXME: need to figure out if pacman + # allows you to verify packages return True else: entry.set('current_version', self.installed[entry.get('name')]) diff --git a/src/lib/Bcfg2/Client/Tools/Pkgng.py b/src/lib/Bcfg2/Client/Tools/Pkgng.py new file mode 100644 index 000000000..cd70d662d --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/Pkgng.py @@ -0,0 +1,226 @@ +"""This is the Bcfg2 support for pkg.""" + +import os +import Bcfg2.Options +import Bcfg2.Client.Tools + + +class Pkgng(Bcfg2.Client.Tools.Tool): + """Support for pkgng packages on FreeBSD.""" + + options = Bcfg2.Client.Tools.Tool.options + [ + Bcfg2.Options.PathOption( + cf=('Pkgng', 'path'), + default='/usr/sbin/pkg', dest='pkg_path', + help='Pkgng tool path')] + + name = 'Pkgng' + __execs__ = [] + __handles__ = [('Package', 'pkgng'), ('Path', 'ignore')] + __req__ = {'Package': ['name', 'version'], 'Path': ['type']} + + def __init__(self, config): + Bcfg2.Client.Tools.Tool.__init__(self, config) + + self.pkg = Bcfg2.Options.setup.pkg_path + self.__execs__ = [self.pkg] + + self.pkgcmd = self.pkg + ' install -fy' + if not Bcfg2.Options.setup.debug: + self.pkgcmd += ' -q' + self.pkgcmd += ' %s' + + self.ignores = [entry.get('name') for struct in config + for entry in struct + if entry.tag == 'Path' and + entry.get('type') == 'ignore'] + + self.__important__ = self.__important__ + \ + [entry.get('name') for struct in config + for entry in struct + if (entry.tag == 'Path' and + entry.get('name').startswith('/etc/pkg/'))] + self.nonexistent = [entry.get('name') for struct in config + for entry in struct if entry.tag == 'Path' + and entry.get('type') == 'nonexistent'] + self.actions = {} + self.pkg_cache = {} + + try: + self._load_pkg_cache() + except OSError: + raise Bcfg2.Client.Tools.ToolInstantiationError + + def _load_pkg_cache(self): + """Cache the version of all currently installed packages.""" + self.pkg_cache = {} + output = self.cmd.run([self.pkg, 'query', '-a', '%n %v']).stdout + for line in output.splitlines(): + parts = line.split(' ') + name = ' '.join(parts[:-1]) + self.pkg_cache[name] = parts[-1] + + def FindExtra(self): + """Find extra packages.""" + packages = [entry.get('name') for entry in self.getSupportedEntries()] + extras = [(name, value) for (name, value) in self.pkg_cache.items() + if name not in packages] + return [Bcfg2.Client.XML.Element('Package', name=name, + type='pkgng', version=version) + for (name, version) in extras] + + def VerifyChecksums(self, entry, modlist): + """Verify the checksum of the files, owned by a package.""" + output = self.cmd.run([self.pkg, 'check', '-s', + entry.get('name')]).stdout.splitlines() + files = [] + for item in output: + if "checksum mismatch" in item: + files.append(item.split()[-1]) + elif "No such file or directory" in item: + continue + else: + self.logger.error("Got Unsupported pattern %s " + "from pkg check" % item) + + files = list(set(files) - set(self.ignores)) + # We check if there is file in the checksum to do + if files: + # if files are found there we try to be sure our modlist is sane + # with erroneous symlinks + modlist = [os.path.realpath(filename) for filename in modlist] + bad = [filename for filename in files if filename not in modlist] + if bad: + self.logger.debug("It is suggested that you either manage " + "these files, revert the changes, or ignore " + "false failures:") + self.logger.info("Package %s failed validation. Bad files " + "are:" % entry.get('name')) + self.logger.info(bad) + entry.set('qtext', + "Reinstall Package %s-%s to fix failing files? " + "(y/N) " % (entry.get('name'), entry.get('version'))) + return False + return True + + def _get_candidate_versions(self, name): + """ + Get versions of the specified package name available for + installation from the configured remote repositories. + """ + output = self.cmd.run([self.pkg, 'search', '-Qversion', '-q', + '-Sname', '-e', name]).stdout.splitlines() + versions = [] + for line in output: + versions.append(line) + + if len(versions) == 0: + return None + + return sorted(versions) + + def VerifyPackage(self, entry, modlist, checksums=True): + """Verify package for entry.""" + if 'version' not in entry.attrib: + self.logger.info("Cannot verify unversioned package %s" % + (entry.attrib['name'])) + return False + + pkgname = entry.get('name') + if pkgname not in self.pkg_cache: + self.logger.info("Package %s not installed" % (entry.get('name'))) + entry.set('current_exists', 'false') + return False + + installed_version = self.pkg_cache[pkgname] + candidate_versions = self._get_candidate_versions(pkgname) + if candidate_versions is not None: + candidate_version = candidate_versions[0] + else: + self.logger.error("Package %s is installed but no candidate" + "version was found." % (entry.get('name'))) + return False + + if entry.get('version').startswith('auto'): + desired_version = candidate_version + entry.set('version', "auto: %s" % desired_version) + elif entry.get('version').startswith('any'): + desired_version = installed_version + entry.set('version', "any: %s" % desired_version) + else: + desired_version = entry.get('version') + + if desired_version != installed_version: + entry.set('current_version', installed_version) + entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % + (entry.get('name'), entry.get('current_version'), + desired_version)) + return False + else: + # version matches + if (not Bcfg2.Options.setup.quick and + entry.get('verify', 'true') == 'true' + and checksums): + pkgsums = self.VerifyChecksums(entry, modlist) + return pkgsums + return True + + def Remove(self, packages): + """Deal with extra configuration detected.""" + pkgnames = " ".join([pkg.get('name') for pkg in packages]) + if len(packages) > 0: + self.logger.info('Removing packages:') + self.logger.info(pkgnames) + self.cmd.run([self.pkg, 'delete', '-y', pkgnames]) + self._load_pkg_cache() + self.modified += packages + self.extra = self.FindExtra() + + def Install(self, packages): + ipkgs = [] + bad_pkgs = [] + for pkg in packages: + versions = self._get_candidate_versions(pkg.get('name')) + if versions is None: + self.logger.error("pkg has no information about package %s" % + (pkg.get('name'))) + continue + + if pkg.get('version').startswith('auto') or \ + pkg.get('version').startswith('any'): + ipkgs.append("%s-%s" % (pkg.get('name'), versions[0])) + continue + + if pkg.get('version') in versions: + ipkgs.append("%s-%s" % (pkg.get('name'), pkg.get('version'))) + continue + else: + self.logger.error("Package %s: desired version %s not in %s" % + (pkg.get('name'), pkg.get('version'), + versions)) + bad_pkgs.append(pkg.get('name')) + + if bad_pkgs: + self.logger.error("Cannot find correct versions of packages:") + self.logger.error(bad_pkgs) + if not ipkgs: + return + if not self.cmd.run(self.pkgcmd % (" ".join(ipkgs))): + self.logger.error("pkg command failed") + self._load_pkg_cache() + self.extra = self.FindExtra() + mark = [] + states = dict() + for package in packages: + states[package] = self.VerifyPackage(package, [], checksums=False) + if states[package]: + self.modified.append(package) + if package.get('origin') == 'Packages': + mark.append(package.get('name')) + if mark: + self.cmd.run([self.pkg, 'set', '-A1', '-y'] + mark) + return states + + def VerifyPath(self, _entry, _): + """Do nothing here since we only verify Path type=ignore.""" + return True diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py index a61ede820..5c092f46b 100644 --- a/src/lib/Bcfg2/Client/Tools/Portage.py +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -50,7 +50,7 @@ class Portage(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, modlist): """Verify package for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (entry.get('name'))) return False @@ -68,11 +68,11 @@ class Portage(Bcfg2.Client.Tools.PkgTool): if ('verify' not in entry.attrib or entry.get('verify').lower() == 'true'): - # Check the package if: - # - Not running in quick mode - # - No verify option is specified in the literal configuration - # OR - # - Verify option is specified and is true + # Check the package if: + # - Not running in quick mode + # - No verify option is specified in the literal configuration + # OR + # - Verify option is specified and is true self.logger.debug('Running equery check on %s' % entry.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/SMF.py b/src/lib/Bcfg2/Client/Tools/SMF.py index 8b23a4a37..1a580d8a5 100644 --- a/src/lib/Bcfg2/Client/Tools/SMF.py +++ b/src/lib/Bcfg2/Client/Tools/SMF.py @@ -25,7 +25,7 @@ class SMF(Bcfg2.Client.Tools.SvcTool): def GetFMRI(self, entry): """Perform FMRI resolution for service.""" - if not 'FMRI' in entry.attrib: + if 'FMRI' not in entry.attrib: rv = self.cmd.run(["/usr/bin/svcs", "-H", "-o", "FMRI", entry.get('name')]) if rv.success: diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 20a172d3d..027d91c71 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,8 +13,6 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} - conflicts = ['Chkconfig'] - def get_svc_command(self, service, action): return "/bin/systemctl %s %s.service" % (action, service.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 4e8ac76a4..449503b55 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -165,12 +165,13 @@ class VCS(Bcfg2.Client.Tools.Tool): def Verifysvn(self, entry, _): """Verify svn repositories""" + # pylint: disable=E1101 headrev = pysvn.Revision(pysvn.opt_revision_kind.head) + # pylint: enable=E1101 client = pysvn.Client() try: cur_rev = str(client.info(entry.get('name')).revision.number) - server = client.info2(entry.get('sourceurl'), - headrev, + server = client.info2(entry.get('sourceurl'), headrev, recurse=False) if server: server_rev = str(server[0][1].rev.number) diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 0b38044d4..86048cb0b 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -632,34 +632,38 @@ class YUM(Bcfg2.Client.Tools.PkgTool): package_fail = True stat['version_fail'] = True # Just chose the first pkg for the error message + current_pkg = all_pkg_objs[0] if virt_pkg: provides = \ - [p for p in all_pkg_objs[0].provides + [p for p in current_pkg.provides if p[0] == entry.get("name")][0] - entry.set('current_version', "%s:%s-%s" % provides[2]) + current_evr = provides[2] self.logger.info( " %s: Wrong version installed. " "Want %s, but %s provides %s" % (entry.get("name"), nevra2string(nevra), - nevra2string(all_pkg_objs[0]), + nevra2string(current_pkg), yum.misc.prco_tuple_to_string(provides))) else: - entry.set('current_version', "%s:%s-%s.%s" % - (all_pkg_objs[0].epoch, - all_pkg_objs[0].version, - all_pkg_objs[0].release, - all_pkg_objs[0].arch)) + current_evr = (current_pkg.epoch, + current_pkg.version, + current_pkg.release) self.logger.info(" %s: Wrong version installed. " "Want %s, but have %s" % (entry.get("name"), nevra2string(nevra), - nevra2string(all_pkg_objs[0]))) - entry.set('version', "%s:%s-%s.%s" % - (nevra.get('epoch', 'any'), - nevra.get('version', 'any'), - nevra.get('release', 'any'), - nevra.get('arch', 'any'))) + nevra2string(current_pkg))) + wanted_evr = (nevra.get('epoch', 'any'), + nevra.get('version', 'any'), + nevra.get('release', 'any')) + entry.set('current_version', "%s:%s-%s" % current_evr) + entry.set('version', "%s:%s-%s" % wanted_evr) + if yum.compareEVR(current_evr, wanted_evr) == 1: + entry.set("package_fail_action", "downgrade") + else: + entry.set("package_fail_action", "update") + qtext_versions.append("U(%s)" % str(all_pkg_objs[0])) continue @@ -910,7 +914,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): cleanup() - def Install(self, packages): # pylint: disable=R0912,R0914 + def Install(self, packages): # pylint: disable=R0912,R0914,R0915 """ Try and fix everything that Yum.VerifyPackages() found wrong for each Package Entry. This can result in individual RPMs being installed (for the first time), deleted, downgraded @@ -932,6 +936,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): install_pkgs = [] gpg_keys = [] upgrade_pkgs = [] + downgrade_pkgs = [] reinstall_pkgs = [] def queue_pkg(pkg, inst, queue): @@ -971,11 +976,14 @@ class YUM(Bcfg2.Client.Tools.PkgTool): continue status = self.instance_status[inst] if (not status.get('installed', False) and - Bcfg2.Options.setup.yum_install_missing): + Bcfg2.Options.setup.yum_install_missing): queue_pkg(pkg, inst, install_pkgs) elif (status.get('version_fail', False) and Bcfg2.Options.setup.yum_fix_version): - queue_pkg(pkg, inst, upgrade_pkgs) + if pkg.get("package_fail_action") == "downgrade": + queue_pkg(pkg, inst, downgrade_pkgs) + else: + queue_pkg(pkg, inst, upgrade_pkgs) elif (status.get('verify_fail', False) and Bcfg2.Options.setup.yum_reinstall_broken): queue_pkg(pkg, inst, reinstall_pkgs) @@ -1039,6 +1047,19 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.logger.error("Error upgrading package %s: %s" % (pkg_arg, yume)) + if len(downgrade_pkgs) > 0: + self.logger.info("Attempting to downgrade packages") + + for inst in downgrade_pkgs: + pkg_arg = self.instance_status[inst].get('pkg').get('name') + self.logger.debug("Downgrading %s" % pkg_arg) + try: + self.yumbase.downgrade(**build_yname(pkg_arg, inst)) + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] + self.logger.error("Error downgrading package %s: %s" % + (pkg_arg, yume)) + if len(reinstall_pkgs) > 0: self.logger.info("Attempting to reinstall packages") for inst in reinstall_pkgs: diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 91d4ac5c6..4ba06abae 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -5,9 +5,29 @@ # pylint: disable=E0611,W0611,W0613,C0103 try: - from lxml.etree import Element, SubElement, XML, tostring + from lxml.etree import Element, SubElement, tostring, XMLParser from lxml.etree import XMLSyntaxError as ParseError + from lxml.etree import XML as _XML + from Bcfg2.Compat import wraps driver = 'lxml' + + # libxml2 2.9.0+ doesn't parse 10M+ documents by default: + # https://mail.gnome.org/archives/commits-list/2012-August/msg00645.html + try: + _parser = XMLParser(huge_tree=True) + except TypeError: + _parser = XMLParser() + + @wraps(_XML) + def XML(val, **kwargs): + """ unicode strings w/encoding declaration are not supported in + recent lxml.etree, so we try to read XML, and if it fails we try + encoding the string. """ + kwargs.setdefault('parser', _parser) + try: + return _XML(val, **kwargs) + except ValueError: + return _XML(val.encode(), **kwargs) except ImportError: # lxml not available from xml.parsers.expat import ExpatError as ParseError diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 2461c1316..073aa7694 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -69,8 +69,8 @@ def prompt(msg): except UnicodeEncodeError: ans = input(msg.encode('utf-8')) return ans in ['y', 'Y'] - except EOFError: - # handle ^C on rhel-based platforms + except (EOFError, KeyboardInterrupt): + # handle ^C raise SystemExit(1) except: print("Error while reading input: %s" % sys.exc_info()[1]) @@ -113,10 +113,10 @@ class Client(object): help='Force removal of additional configuration items')), Bcfg2.Options.ExclusiveOptionGroup( Bcfg2.Options.PathOption( - '-f', '--file', type=argparse.FileType('r'), + '-f', '--file', type=argparse.FileType('rb'), help='Configure from a file rather than querying the server'), Bcfg2.Options.PathOption( - '-c', '--cache', type=argparse.FileType('w'), + '-c', '--cache', type=argparse.FileType('wb'), help='Store the configuration in a file')), Bcfg2.Options.BooleanOption( '--exit-on-probe-failure', default=True, @@ -144,7 +144,10 @@ class Client(object): Bcfg2.Options.BooleanOption( "-e", "--show-extra", help='Enable extra entry output'), Bcfg2.Options.BooleanOption( - "-k", "--kevlar", help='Run in bulletproof mode')] + "-k", "--kevlar", help='Run in bulletproof mode'), + Bcfg2.Options.BooleanOption( + "-i", "--only-important", + help='Only configure the important entries')] def __init__(self): self.config = None @@ -403,7 +406,7 @@ class Client(object): self.config = newconfig if not Bcfg2.Options.setup.no_lock: - #check lock here + # check lock here try: lockfile = open(Bcfg2.Options.setup.lockfile, 'w') if locked(lockfile.fileno()): @@ -559,11 +562,13 @@ class Client(object): if x not in b_to_rem] # take care of important entries first - if not Bcfg2.Options.setup.dry_run: + if (not Bcfg2.Options.setup.dry_run or + Bcfg2.Options.setup.only_important): + important_installs = set() for parent in self.config.findall(".//Path/.."): name = parent.get("name") - if (name and (name in Bcfg2.Options.setup.only_bundles or - name not in Bcfg2.Options.setup.except_bundles)): + if not name or (name in Bcfg2.Options.setup.except_bundles and + name not in Bcfg2.Options.setup.only_bundles): continue for cfile in parent.findall("./Path"): if (cfile.get('name') not in self.__important__ or @@ -574,6 +579,9 @@ class Client(object): if t.handlesEntry(cfile) and t.canVerify(cfile)] if not tools: continue + if Bcfg2.Options.setup.dry_run: + important_installs.add(cfile) + continue if (Bcfg2.Options.setup.interactive and not self.promptFilter("Install %s: %s? (y/N):", [cfile])): @@ -589,6 +597,11 @@ class Client(object): cfile.set('qtext', '') if tools[0].VerifyPath(cfile, []): self.whitelist.remove(cfile) + if Bcfg2.Options.setup.dry_run and len(important_installs) > 0: + self.logger.info("In dryrun mode: " + "suppressing entry installation for:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in important_installs]) def Inventory(self): """ @@ -845,11 +858,13 @@ class Client(object): self.times['inventory'] = time.time() self.CondDisplayState('initial') self.InstallImportant() - self.Decide() - self.Install() - self.times['install'] = time.time() - self.Remove() - self.times['remove'] = time.time() + if not Bcfg2.Options.setup.only_important: + self.Decide() + self.Install() + self.times['install'] = time.time() + self.Remove() + self.times['remove'] = time.time() + if self.modified: self.ReInventory() self.times['reinventory'] = time.time() diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 24835a3e8..12dba7fba 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -26,8 +26,8 @@ settings = dict( # pylint: disable=C0103 DEBUG=False, ALLOWED_HOSTS=['*'], MEDIA_URL='/site_media/', - MANAGERS=(('Root', 'root')), - ADMINS=(('Root', 'root')), + MANAGERS=(('Root', 'root'),), + ADMINS=(('Root', 'root'),), # Language code for this installation. All choices can be found # here: # http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes @@ -63,7 +63,8 @@ settings = dict( # pylint: disable=C0103 'django.core.context_processors.debug', 'django.core.context_processors.i18n', 'django.core.context_processors.media', - 'django.core.context_processors.request')) + 'django.core.context_processors.request'), + DATABASE_ROUTERS=['Bcfg2.DBSettings.PerApplicationRouter']) if HAS_SOUTH: settings['INSTALLED_APPS'] += ('south', 'Bcfg2.Reporting') @@ -95,6 +96,18 @@ def finalize_django_config(opts=None, silent=False): OPTIONS=opts.db_opts, SCHEMA=opts.db_schema)) + if hasattr(opts, "reporting_db_engine") and \ + opts.reporting_db_engine is not None: + settings['DATABASES']['Reporting'] = dict( + ENGINE="django.db.backends.%s" % opts.reporting_db_engine, + NAME=opts.reporting_db_name, + USER=opts.reporting_db_user, + PASSWORD=opts.reporting_db_password, + HOST=opts.reporting_db_host, + PORT=opts.reporting_db_port, + OPTIONS=opts.reporting_db_opts, + SCHEMA=opts.reporting_db_schema) + settings['TIME_ZONE'] = opts.timezone settings['TEMPLATE_DEBUG'] = settings['DEBUG'] = \ @@ -112,6 +125,9 @@ def finalize_django_config(opts=None, silent=False): logger = logging.getLogger() logger.debug("Finalizing Django settings: %s" % settings) + module = sys.modules[__name__] + for name, value in settings.items(): + setattr(module, name, value) try: django.conf.settings.configure(**settings) except RuntimeError: @@ -120,6 +136,66 @@ def finalize_django_config(opts=None, silent=False): sys.exc_info()[1]) +def sync_databases(**kwargs): + """ Synchronize all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Syncing database %s" % (database)) + django.core.management.call_command("syncdb", database=database, + **kwargs) + + +def migrate_databases(**kwargs): + """ Do South migrations on all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Migrating database %s" % (database)) + django.core.management.call_command("migrate", database=database, + **kwargs) + + +def get_db_label(application): + """ Get the name of the database for a given Django "application". The + rule is that if a database with the same name as the application exists, + use it. Otherwise use the default. Returns a string suitible for use as a + key in the Django database settings dict """ + if application in settings['DATABASES']: + return application + + return 'default' + + +class PerApplicationRouter(object): + """ Django database router for redirecting different applications to their + own database """ + + def _db_per_app(self, model, **_): + """ If a database with the same name as the application exists, use it. + Otherwise use the default """ + return get_db_label(model._meta.app_label) # pylint: disable=W0212 + + def db_for_read(self, model, **hints): + """ Called when Django wants to find out what database to read from """ + return self._db_per_app(model, **hints) + + def db_for_write(self, model, **hints): + """ Called when Django wants to find out what database to write to """ + return self._db_per_app(model, **hints) + + def allow_relation(self, obj1, obj2, **_): + """ Called when Django wants to determine what relations to allow. Only + allow relations within an app """ + # pylint: disable=W0212 + return obj1._meta.app_label == obj2._meta.app_label + # pylint: enable=W0212 + + def allow_syncdb(self, *_): + """ Called when Django wants to determine which models to sync to a + given database. Take the cowards way out and sync all models to all + databases to allow for easy migrations. """ + return True + + class _OptionContainer(object): """ Container for options loaded at import-time to configure databases """ @@ -131,6 +207,7 @@ class _OptionContainer(object): default="/etc/bcfg2-web.conf", action=Bcfg2.Options.ConfigFileAction, help='Web interface configuration file'), + # default database options Bcfg2.Options.Option( cf=('database', 'engine'), default='sqlite3', help='Database engine', dest='db_engine'), @@ -148,11 +225,40 @@ class _OptionContainer(object): cf=('database', 'port'), help='Database port', dest='db_port'), Bcfg2.Options.Option( cf=('database', 'schema'), help='Database schema', - dest='db_schema'), + dest='db_schema', default='public'), Bcfg2.Options.Option( cf=('database', 'options'), help='Database options', dest='db_opts', type=Bcfg2.Options.Types.comma_dict, default=dict()), + # reporting database options + Bcfg2.Options.Option( + cf=('database', 'reporting_engine'), + help='Reporting database engine', dest='reporting_db_engine'), + Bcfg2.Options.Option( + cf=('database', 'reporting_name'), + default='<repository>/etc/reporting.sqlite', + help="Reporting database name", dest="reporting_db_name"), + Bcfg2.Options.Option( + cf=('database', 'reporting_user'), + help='Reporting database username', dest='reporting_db_user'), + Bcfg2.Options.Option( + cf=('database', 'reporting_password'), + help='Reporting database password', dest='reporting_db_password'), + Bcfg2.Options.Option( + cf=('database', 'reporting_host'), + help='Reporting database host', dest='reporting_db_host'), + Bcfg2.Options.Option( + cf=('database', 'reporting_port'), + help='Reporting database port', dest='reporting_db_port'), + Bcfg2.Options.Option( + cf=('database', 'reporting_schema'), + help='Reporting database schema', dest='reporting_db_schema', + default='public'), + Bcfg2.Options.Option( + cf=('database', 'reporting_options'), + help='Reporting database options', dest='reporting_db_opts', + type=Bcfg2.Options.Types.comma_dict, default=dict()), + # Django options Bcfg2.Options.Option( cf=('reporting', 'timezone'), help='Django timezone'), Bcfg2.Options.BooleanOption( diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index 0f7995e0f..11eaeebd1 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -21,7 +21,7 @@ class TermiosFormatter(logging.Formatter): def __init__(self, fmt=None, datefmt=None): logging.Formatter.__init__(self, fmt, datefmt) - if sys.stdout.isatty(): + if hasattr(sys.stdout, 'isatty') and sys.stdout.isatty(): # now get termios info try: self.width = struct.unpack('hhhh', diff --git a/src/lib/Bcfg2/Options/Actions.py b/src/lib/Bcfg2/Options/Actions.py index 8b97f1da8..8b941f2bb 100644 --- a/src/lib/Bcfg2/Options/Actions.py +++ b/src/lib/Bcfg2/Options/Actions.py @@ -7,7 +7,27 @@ from Bcfg2.Options.Parser import get_parser __all__ = ["ConfigFileAction", "ComponentAction", "PluginsAction"] -class ComponentAction(argparse.Action): +class FinalizableAction(argparse.Action): + """ A FinalizableAction requires some additional action to be taken + when storing the value, and as a result must be finalized if the + default value is used.""" + + def __init__(self, *args, **kwargs): + argparse.Action.__init__(self, *args, **kwargs) + self._final = False + + def finalize(self, parser, namespace): + """ Finalize a default value by calling the action callable. """ + if not self._final: + self.__call__(parser, namespace, getattr(namespace, self.dest, + self.default)) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + self._final = True + + +class ComponentAction(FinalizableAction): """ ComponentAction automatically imports classes and modules based on the value of the option, and automatically collects options from the loaded classes and modules. It cannot be used by @@ -84,8 +104,7 @@ class ComponentAction(argparse.Action): if self.mapping: if 'choices' not in kwargs: kwargs['choices'] = self.mapping.keys() - self._final = False - argparse.Action.__init__(self, *args, **kwargs) + FinalizableAction.__init__(self, *args, **kwargs) def _import(self, module, name): """ Import the given name from the given module, handling @@ -123,17 +142,10 @@ class ComponentAction(argparse.Action): break if cls: get_parser().add_component(cls) - else: + elif not self.fail_silently: print("Could not load component %s" % name) return cls - def finalize(self, parser, namespace): - """ Finalize a default value by loading the components given - in it. This lets a default be specified with a list of - strings instead of a list of classes. """ - if not self._final: - self.__call__(parser, namespace, self.default) - def __call__(self, parser, namespace, values, option_string=None): if values is None: result = None @@ -146,18 +158,19 @@ class ComponentAction(argparse.Action): result.append(cls) else: result = self._load_component(values) - self._final = True - setattr(namespace, self.dest, result) + FinalizableAction.__call__(self, parser, namespace, result, + option_string=option_string) -class ConfigFileAction(argparse.Action): +class ConfigFileAction(FinalizableAction): """ ConfigFileAction automatically loads and parses a supplementary config file (e.g., ``bcfg2-web.conf`` or ``bcfg2-lint.conf``). """ def __call__(self, parser, namespace, values, option_string=None): - get_parser().add_config_file(self.dest, values) - setattr(namespace, self.dest, values) + parser.add_config_file(self.dest, values, reparse=False) + FinalizableAction.__call__(self, parser, namespace, values, + option_string=option_string) class PluginsAction(ComponentAction): diff --git a/src/lib/Bcfg2/Options/Common.py b/src/lib/Bcfg2/Options/Common.py index 9ba08eb87..620a7604c 100644 --- a/src/lib/Bcfg2/Options/Common.py +++ b/src/lib/Bcfg2/Options/Common.py @@ -94,7 +94,7 @@ class Common(object): #: Log to syslog syslog = BooleanOption( - cf=('logging', 'syslog'), help="Log to syslog") + cf=('logging', 'syslog'), help="Log to syslog", default=True) #: Server location location = Option( @@ -107,20 +107,16 @@ class Common(object): '-x', '--password', cf=('communication', 'password'), metavar='<password>', help="Communication Password") - #: Path to SSL key - ssl_key = PathOption( - '--ssl-key', cf=('communication', 'key'), dest="key", - help='Path to SSL key', default="/etc/pki/tls/private/bcfg2.key") - - #: Path to SSL certificate - ssl_cert = PathOption( - cf=('communication', 'certificate'), dest="cert", - help='Path to SSL certificate', default="/etc/pki/tls/certs/bcfg2.crt") - #: Path to SSL CA certificate ssl_ca = PathOption( cf=('communication', 'ca'), help='Path to SSL CA Cert') + #: Communication protocol + protocol = Option( + cf=('communication', 'protocol'), default='xmlrpc/tlsv1', + choices=['xmlrpc/ssl', 'xmlrpc/tlsv1'], + help='Communication protocol to use.') + #: Default Path paranoid setting default_paranoid = Option( cf=('mdata', 'paranoid'), dest="default_paranoid", default='true', diff --git a/src/lib/Bcfg2/Options/Options.py b/src/lib/Bcfg2/Options/Options.py index be7e7c646..3874f810d 100644 --- a/src/lib/Bcfg2/Options/Options.py +++ b/src/lib/Bcfg2/Options/Options.py @@ -10,7 +10,18 @@ from Bcfg2.Options import Types from Bcfg2.Compat import ConfigParser -__all__ = ["Option", "BooleanOption", "PathOption", "PositionalArgument"] +__all__ = ["Option", "BooleanOption", "PathOption", "PositionalArgument", + "_debug"] + + +def _debug(msg): + """ Option parsing happens before verbose/debug have been set -- + they're options, after all -- so option parsing verbosity is + enabled by changing this to True. The verbosity here is primarily + of use to developers. """ + if os.environ.get('BCFG2_OPTIONS_DEBUG', '0') == '1': + print(msg) + #: A dict that records a mapping of argparse action name (e.g., #: "store_true") to the argparse Action class for it. See @@ -158,6 +169,10 @@ class Option(object): the appropriate default value in the appropriate format.""" for parser, action in self.actions.items(): if hasattr(action, "finalize"): + if parser: + _debug("Finalizing %s for %s" % (self, parser)) + else: + _debug("Finalizing %s" % self) action.finalize(parser, namespace) def from_config(self, cfp): @@ -181,23 +196,25 @@ class Option(object): exclude.update(o.cf[1] for o in parser.option_list if o.cf and o.cf[0] == self.cf[0]) - return dict([(o, cfp.get(self.cf[0], o)) - for o in fnmatch.filter(cfp.options(self.cf[0]), - self.cf[1]) - if o not in exclude]) + rv = dict([(o, cfp.get(self.cf[0], o)) + for o in fnmatch.filter(cfp.options(self.cf[0]), + self.cf[1]) + if o not in exclude]) else: - return dict() + rv = dict() else: + if self.type: + rtype = self.type + else: + rtype = lambda x: x try: - val = cfp.getboolean(*self.cf) + rv = rtype(cfp.getboolean(*self.cf)) except ValueError: - val = cfp.get(*self.cf) + rv = rtype(cfp.get(*self.cf)) except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return None - if self.type: - return self.type(val) - else: - return val + rv = None + _debug("Setting %s from config file(s): %s" % (self, rv)) + return rv def default_from_config(self, cfp): """ Set the default value of this option from the config file @@ -208,9 +225,13 @@ class Option(object): """ if self.env and self.env in os.environ: self.default = os.environ[self.env] + _debug("Setting the default of %s from environment: %s" % + (self, self.default)) else: val = self.from_config(cfp) if val is not None: + _debug("Setting the default of %s from config: %s" % + (self, val)) self.default = val def _get_default(self): @@ -250,13 +271,17 @@ class Option(object): self.parsers.append(parser) if self.args: # cli option + _debug("Adding %s to %s as a CLI option" % (self, parser)) action = parser.add_argument(*self.args, **self._kwargs) if not self._dest: self._dest = action.dest if self._default: action.default = self._default self.actions[parser] = action - # else, config file-only option + else: + # else, config file-only option + _debug("Adding %s to %s as a config file-only option" % + (self, parser)) class PathOption(Option): @@ -281,6 +306,26 @@ class PathOption(Option): Option.__init__(self, *args, **kwargs) +class _BooleanOptionAction(argparse.Action): + """ BooleanOptionAction sets a boolean value in the following ways: + - if None is passed, store the default + - if the option_string is not None, then the option was passed on the + command line, thus store the opposite of the default (this is the + argparse store_true and store_false behavior) + - if a boolean value is passed, use that + + Defined here instead of :mod:`Bcfg2.Options.Actions` because otherwise + there is a circular import Options -> Actions -> Parser -> Options """ + + def __call__(self, parser, namespace, values, option_string=None): + if values is None: + setattr(namespace, self.dest, self.default) + elif option_string is not None: + setattr(namespace, self.dest, not self.default) + else: + setattr(namespace, self.dest, bool(values)) + + class BooleanOption(Option): """ Shortcut for boolean options. The default is False, but this can easily be overridden: @@ -292,11 +337,10 @@ class BooleanOption(Option): "--dwim", default=True, help="Do What I Mean")] """ def __init__(self, *args, **kwargs): - if 'default' in kwargs and kwargs['default']: - kwargs.setdefault('action', 'store_false') - else: - kwargs.setdefault('action', 'store_true') - kwargs.setdefault('default', False) + kwargs.setdefault('action', _BooleanOptionAction) + kwargs.setdefault('nargs', 0) + kwargs.setdefault('default', False) + Option.__init__(self, *args, **kwargs) diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py index 80f966246..677a69e4c 100644 --- a/src/lib/Bcfg2/Options/Parser.py +++ b/src/lib/Bcfg2/Options/Parser.py @@ -5,7 +5,7 @@ import sys import argparse from Bcfg2.version import __version__ from Bcfg2.Compat import ConfigParser -from Bcfg2.Options import Option, PathOption, BooleanOption +from Bcfg2.Options import Option, PathOption, BooleanOption, _debug __all__ = ["setup", "OptionParserException", "Parser", "get_parser"] @@ -37,6 +37,7 @@ class Parser(argparse.ArgumentParser): #: Option for specifying the path to the Bcfg2 config file configfile = PathOption('-C', '--config', + env="BCFG2_CONFIG_FILE", help="Path to configuration file", default="/etc/bcfg2.conf") @@ -121,14 +122,16 @@ class Parser(argparse.ArgumentParser): """ Add a component (and all of its options) to the parser. """ if component not in self.components: + _debug("Adding component %s to %s" % (component, self)) self.components.append(component) if hasattr(component, "options"): self.add_options(getattr(component, "options")) - def _set_defaults(self): + def _set_defaults_from_config(self): """ Set defaults from the config file for all options that can come from the config file, but haven't yet had their default set """ + _debug("Setting defaults on all options") for opt in self.option_list: if opt not in self._defaults_set: opt.default_from_config(self._cfp) @@ -138,15 +141,15 @@ class Parser(argparse.ArgumentParser): """ populate the namespace with default values for any options that aren't already in the namespace (i.e., options without CLI arguments) """ + _debug("Parsing config file-only options") for opt in self.option_list[:]: if not opt.args and opt.dest not in self.namespace: value = opt.default if value: - for parser, action in opt.actions.items(): - if parser is None: - action(self, self.namespace, value) - else: - action(parser, parser.namespace, value) + for _, action in opt.actions.items(): + _debug("Setting config file-only option %s to %s" % + (opt, value)) + action(self, self.namespace, value) else: setattr(self.namespace, opt.dest, value) @@ -155,6 +158,7 @@ class Parser(argparse.ArgumentParser): additional post-processing step. (Mostly :class:`Bcfg2.Options.Actions.ComponentAction` subclasses.) """ + _debug("Finalizing options") for opt in self.option_list[:]: opt.finalize(self.namespace) @@ -162,20 +166,23 @@ class Parser(argparse.ArgumentParser): """ Delete all options from the namespace except for a few predefined values and config file options. """ self.parsed = False + _debug("Resetting namespace") for attr in dir(self.namespace): if (not attr.startswith("_") and attr not in ['uri', 'version', 'name'] and attr not in self._config_files): + _debug("Deleting %s" % attr) delattr(self.namespace, attr) def add_config_file(self, dest, cfile, reparse=True): """ Add a config file, which triggers a full reparse of all options. """ if dest not in self._config_files: + _debug("Adding new config file %s for %s" % (cfile, dest)) self._reset_namespace() self._cfp.read([cfile]) self._defaults_set = [] - self._set_defaults() + self._set_defaults_from_config() if reparse: self._parse_config_options() self._config_files.append(dest) @@ -188,6 +195,7 @@ class Parser(argparse.ArgumentParser): (I.e., the argument list that was initially parsed.) :type argv: list """ + _debug("Reparsing all options") self._reset_namespace() self.parse(argv or self.argv) @@ -200,15 +208,19 @@ class Parser(argparse.ArgumentParser): :func:`Bcfg2.Options.Parser.reparse`. :type argv: list """ + _debug("Parsing options") if argv is None: argv = sys.argv[1:] if self.parsed and self.argv == argv: + _debug("Returning already parsed namespace") return self.namespace self.argv = argv # phase 1: get and read config file + _debug("Option parsing phase 1: Get and read main config file") bootstrap_parser = argparse.ArgumentParser(add_help=False) self.configfile.add_to_parser(bootstrap_parser) + self.configfile.default_from_config(self._cfp) bootstrap = bootstrap_parser.parse_known_args(args=self.argv)[0] # check whether the specified bcfg2.conf exists @@ -219,6 +231,7 @@ class Parser(argparse.ArgumentParser): # phase 2: re-parse command line for early options; currently, # that's database options + _debug("Option parsing phase 2: Parse early options") if not self._early: early_opts = argparse.Namespace() early_parser = Parser(add_help=False, namespace=early_opts, @@ -232,35 +245,62 @@ class Parser(argparse.ArgumentParser): early_components.append(component) early_parser.add_component(component) early_parser.parse(self.argv) + _debug("Early parsing complete, calling hooks") for component in early_components: if hasattr(component, "component_parsed_hook"): + _debug("Calling component_parsed_hook on %s" % component) getattr(component, "component_parsed_hook")(early_opts) # phase 3: re-parse command line, loading additional # components, until all components have been loaded. On each # iteration, set defaults from config file/environment # variables + _debug("Option parsing phase 3: Main parser loop") + # _set_defaults_from_config must be called before _parse_config_options + # This is due to a tricky interaction between the two methods: + # + # (1) _set_defaults_from_config does what its name implies, it updates + # the "default" property of each Option based on the value that exists + # in the config. + # + # (2) _parse_config_options will look at each option and set it to the + # default value that is _currently_ defined. If the option does not + # exist in the namespace, it will be added. The method carefully + # avoids overwriting the value of an option that is already defined in + # the namespace. + # + # Thus, if _set_defaults_from_config has not been called yet when + # _parse_config_options is called, all config file options will get set + # to their hardcoded defaults. This process defines the options in the + # namespace and _parse_config_options will never look at them again. + self._set_defaults_from_config() self._parse_config_options() while not self.parsed: self.parsed = True - self._set_defaults() + self._set_defaults_from_config() self.parse_known_args(args=self.argv, namespace=self.namespace) self._parse_config_options() self._finalize() - self._parse_config_options() # phase 4: fix up <repository> macros + _debug("Option parsing phase 4: Fix up macros") repo = getattr(self.namespace, "repository", repository.default) for attr in dir(self.namespace): value = getattr(self.namespace, attr) - if not attr.startswith("_") and hasattr(value, "replace"): + if (not attr.startswith("_") and + hasattr(value, "replace") and + "<repository>" in value): setattr(self.namespace, attr, value.replace("<repository>", repo, 1)) + _debug("Fixing up macros in %s: %s -> %s" % + (attr, value, getattr(self.namespace, attr))) # phase 5: call post-parsing hooks + _debug("Option parsing phase 5: Call hooks") if not self._early: for component in self.components: if hasattr(component, "options_parsed_hook"): + _debug("Calling post-parsing hook on %s" % component) getattr(component, "options_parsed_hook")() return self.namespace diff --git a/src/lib/Bcfg2/Options/Types.py b/src/lib/Bcfg2/Options/Types.py index 2f0fd7d52..d11e54fba 100644 --- a/src/lib/Bcfg2/Options/Types.py +++ b/src/lib/Bcfg2/Options/Types.py @@ -50,6 +50,15 @@ def comma_dict(value): return result +def anchored_regex_list(value): + """ Split an option string on whitespace and compile each element as + an anchored regex """ + try: + return [re.compile('^' + x + '$') for x in re.split(r'\s+', value)] + except re.error: + raise ValueError("Not a list of regexes", value) + + def octal(value): """ Given an octal string, get an integer representation. """ return int(value, 8) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 6c1dfdccb..12c9cdaa8 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -6,6 +6,7 @@ import time import threading # pylint: disable=E0611 +from lockfile import LockFailed, LockTimeout try: from lockfile.pidlockfile import PIDLockFile from lockfile import Error as PIDFileError @@ -63,6 +64,8 @@ class ReportingCollector(object): bcfg2-admin""" self.terminate = None self.context = None + self.children = [] + self.cleanup_threshold = 25 if Bcfg2.Options.setup.debug: level = logging.DEBUG @@ -106,12 +109,24 @@ class ReportingCollector(object): self.terminate = threading.Event() atexit.register(self.shutdown) self.context = daemon.DaemonContext(detach_process=True) + iter = 0 if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") try: self.context.pidfile = PIDLockFile(Bcfg2.Options.setup.daemon) self.context.open() + except LockFailed: + self.logger.error("Failed to daemonize: %s" % + sys.exc_info()[1]) + self.shutdown() + return + except LockTimeout: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + self.setup['daemon']) + self.shutdown() + return except PIDFileError: self.logger.error("Error writing pid file: %s" % sys.exc_info()[1]) @@ -128,6 +143,13 @@ class ReportingCollector(object): continue store_thread = ReportingStoreThread(interaction, self.storage) store_thread.start() + self.children.append(store_thread) + + iter += 1 + if iter >= self.cleanup_threshold: + self.reap_children() + iter = 0 + except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() @@ -147,3 +169,16 @@ class ReportingCollector(object): pass if self.storage: self.storage.shutdown() + + def reap_children(self): + """Join any non-live threads""" + newlist = [] + + self.logger.debug("Starting reap_children") + for child in self.children: + if child.isAlive(): + newlist.append(child) + else: + child.join() + self.logger.debug("Joined child thread %s" % child.getName()) + self.children = newlist diff --git a/src/lib/Bcfg2/Reporting/Compat.py b/src/lib/Bcfg2/Reporting/Compat.py index 57261970d..9113fdb91 100644 --- a/src/lib/Bcfg2/Reporting/Compat.py +++ b/src/lib/Bcfg2/Reporting/Compat.py @@ -10,9 +10,7 @@ if VERSION[0] == 1 and VERSION[1] < 6: try: # Django < 1.6 - from django.conf.urls import defaults - django_urls = defaults + from django.conf.urls.defaults import url, patterns except ImportError: # Django > 1.6 - from django.conf import urls - django_urls = urls + from django.conf.urls import url, patterns diff --git a/src/lib/Bcfg2/Reporting/Reports.py b/src/lib/Bcfg2/Reporting/Reports.py index 35c09a7e1..219d74584 100755 --- a/src/lib/Bcfg2/Reporting/Reports.py +++ b/src/lib/Bcfg2/Reporting/Reports.py @@ -43,6 +43,8 @@ def print_fields(fields, client, fmt, extra=None): fdata.append(client.current_interaction.extra_count) elif field == 'bad': fdata.append((client.current_interaction.bad_count)) + elif field == 'stale': + fdata.append(client.current_interaction.isstale()) else: try: fdata.append(getattr(client, field)) diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index c223c3c73..406216861 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -4,6 +4,7 @@ The base for the original DjangoORM (DBStats) from lxml import etree from datetime import datetime +import traceback from time import strptime import Bcfg2.Options import Bcfg2.DBSettings @@ -14,6 +15,7 @@ from django.core import management from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db.models import FieldDoesNotExist from django.core.cache import cache +from django import db #Used by GetCurrentEntry import difflib @@ -368,7 +370,12 @@ class DjangoORM(StorageBase): self._import_interaction(interaction) except: self.logger.error("Failed to import interaction: %s" % - sys.exc_info()[1]) + traceback.format_exc().splitlines()[-1]) + finally: + self.logger.debug("%s: Closing database connection" % + self.__class__.__name__) + db.close_connection() + def validate(self): """Validate backend storage. Should be called once when loaded""" @@ -380,9 +387,9 @@ class DjangoORM(StorageBase): vrb = 1 else: vrb = 0 - management.call_command("syncdb", verbosity=vrb, interactive=False) - management.call_command("migrate", verbosity=vrb, - interactive=False) + Bcfg2.DBSettings.sync_databases(verbosity=vrb, interactive=False) + Bcfg2.DBSettings.migrate_databases(verbosity=vrb, + interactive=False) except: msg = "Failed to update database schema: %s" % sys.exc_info()[1] self.logger.error(msg) diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 0598e4d33..2d96990b1 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -3,7 +3,7 @@ import sys from django.core.exceptions import ImproperlyConfigured try: - from django.db import models, backend, connection + from django.db import models, backend, connections except ImproperlyConfigured: e = sys.exc_info()[1] print("Reports: unable to import django models: %s" % e) @@ -12,6 +12,7 @@ except ImproperlyConfigured: from django.core.cache import cache from datetime import datetime, timedelta from Bcfg2.Compat import cPickle +from Bcfg2.DBSettings import get_db_label TYPE_GOOD = 0 @@ -61,7 +62,8 @@ def _quote(value): global _our_backend if not _our_backend: try: - _our_backend = backend.DatabaseOperations(connection) + _our_backend = backend.DatabaseOperations( + connections[get_db_label('Reporting')]) except TypeError: _our_backend = backend.DatabaseOperations() return _our_backend.quote_name(value) @@ -91,8 +93,8 @@ class InteractionManager(models.Manager): maxdate -- datetime object. Most recent date to pull. (default None) """ - from django.db import connection - cursor = connection.cursor() + from django.db import connections + cursor = connections[get_db_label('Reporting')].cursor() cfilter = "expiration is null" sql = 'select ri.id, x.client_id from ' + \ diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 7edf3a949..8b197231c 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5 <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.3</span> + <span>Bcfg2 Version 1.4.0pre1</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index 33c78a5f0..6a314bd88 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -32,7 +32,7 @@ This is needed for Django versions less than 1.5 <td class='right_column_narrow'>{{ entry.bad_count }}</td> <td class='right_column_narrow'>{{ entry.modified_count }}</td> <td class='right_column_narrow'>{{ entry.extra_count }}</td> - <td class='right_column'><span {% if entry.timestamp|isstale:entry_max %}class='dirty-lineitem'{% endif %}>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</span></td> + <td class='right_column'><span {% if entry.isstale %}class='dirty-lineitem'{% endif %}>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</span></td> <td class='right_column_wide'> {% if entry.server %} <a href='{% add_url_filter server=entry.server %}'>{{ entry.server }}</a> diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 489682f30..4a93e77e0 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -189,19 +189,6 @@ def build_metric_list(mdict): @register.filter -def isstale(timestamp, entry_max=None): - """ - Check for a stale timestamp - - Compares two timestamps and returns True if the - difference is greater then 24 hours. - """ - if not entry_max: - entry_max = datetime.now() - return entry_max - timestamp > timedelta(hours=24) - - -@register.filter def sort_interactions_by_name(value): """ Sort an interaction list by client name @@ -318,7 +305,11 @@ def determine_client_state(entry): dirty. If the client is reporting dirty, this will figure out just _how_ dirty and adjust the color accordingly. """ + if entry.isstale(): + return "stale-lineitem" if entry.state == 'clean': + if entry.extra_count > 0: + return "extra-lineitem" return "clean-lineitem" bad_percentage = 100 * (float(entry.bad_count) / entry.total_count) diff --git a/src/lib/Bcfg2/Reporting/urls.py b/src/lib/Bcfg2/Reporting/urls.py index a9e5690be..3a40cb932 100644 --- a/src/lib/Bcfg2/Reporting/urls.py +++ b/src/lib/Bcfg2/Reporting/urls.py @@ -1,4 +1,4 @@ -from Bcfg2.Reporting.Compat.django_urls import * +from Bcfg2.Reporting.Compat import url, patterns # django compat imports from django.core.urlresolvers import reverse, NoReverseMatch from django.http import HttpResponsePermanentRedirect from Bcfg2.Reporting.utils import filteredUrls, paginatedUrls, timeviewUrls diff --git a/src/lib/Bcfg2/Reporting/utils.py b/src/lib/Bcfg2/Reporting/utils.py index d9b8213b1..0d394fcd8 100755 --- a/src/lib/Bcfg2/Reporting/utils.py +++ b/src/lib/Bcfg2/Reporting/utils.py @@ -1,5 +1,4 @@ """Helper functions for reports""" -from Bcfg2.Reporting.Compat.django_urls import * import re """List of filters provided by filteredUrls""" diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index c7c2a503f..0b8ed65cc 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -13,7 +13,7 @@ from django.http import \ from django.shortcuts import render_to_response, get_object_or_404 from django.core.urlresolvers import \ resolve, reverse, Resolver404, NoReverseMatch -from django.db import connection, DatabaseError +from django.db import DatabaseError from django.db.models import Q, Count from Bcfg2.Reporting.models import * diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 207106596..0807fb2b0 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -173,15 +173,33 @@ class Backup(AdminCmd): class Client(_ServerAdminCmd): - """ Create, delete, or list client entries """ + """ Create, modify, delete, or list client entries """ + __plugin_whitelist__ = ["Metadata"] options = _ServerAdminCmd.options + [ Bcfg2.Options.PositionalArgument( "mode", - choices=["add", "del", "list"]), - Bcfg2.Options.PositionalArgument("hostname", nargs='?')] - - __plugin_whitelist__ = ["Metadata"] + choices=["add", "del", "delete", "remove", "rm", "up", "update", + "list"]), + Bcfg2.Options.PositionalArgument("hostname", nargs='?'), + Bcfg2.Options.PositionalArgument("attributes", metavar="KEY=VALUE", + nargs='*')] + + valid_attribs = ['profile', 'uuid', 'password', 'floating', 'secure', + 'address', 'auth'] + + def get_attribs(self, setup): + """ Get attributes for adding or updating a client from the command + line """ + attr_d = {} + for i in setup.attributes: + attr, val = i.split('=', 1) + if attr not in self.valid_attribs: + print("Attribute %s unknown. Valid attributes: %s" % + (attr, self.valid_attribs)) + raise SystemExit(1) + attr_d[attr] = val + return attr_d def run(self, setup): if setup.mode != 'list' and not setup.hostname: @@ -189,23 +207,32 @@ class Client(_ServerAdminCmd): elif setup.mode == 'list' and setup.hostname: self.logger.warning("<hostname> is not honored in list mode") - if setup.mode == 'add': - try: - self.metadata.add_client(setup.hostname) - except MetadataConsistencyError: - err = sys.exc_info()[1] - self.errExit("Error adding client %s: %s" % (setup.hostname, - err)) - elif setup.mode == 'del': + if setup.mode == 'list': + for client in self.metadata.list_clients(): + print(client) + else: + include_attribs = True + if setup.mode == 'add': + func = self.metadata.add_client + action = "adding" + elif setup.mode in ['up', 'update']: + func = self.metadata.update_client + action = "updating" + elif setup.mode in ['del', 'delete', 'rm', 'remove']: + func = self.metadata.remove_client + include_attribs = False + action = "deleting" + + if include_attribs: + args = (setup.hostname, self.get_attribs(setup)) + else: + args = (setup.hostname,) try: - self.metadata.remove_client(setup.hostname) + func(*args) except MetadataConsistencyError: err = sys.exc_info()[1] - self.errExit("Error deleting client %s: %s" % (setup.hostname, - err)) - elif setup.mode == 'list': - for client in self.metadata.list_clients(): - print(client) + self.errExit("Error %s client %s: %s" % (setup.hostname, + action, err)) class Compare(AdminCmd): @@ -885,8 +912,9 @@ if HAS_DJANGO: def run(self, setup): Bcfg2.Server.models.load_models() try: - management.call_command("syncdb", interactive=False, - verbosity=setup.verbose + setup.debug) + Bcfg2.DBSettings.sync_databases( + interactive=False, + verbosity=setup.verbose + setup.debug) except ImproperlyConfigured: err = sys.exc_info()[1] self.logger.error("Django configuration problem: %s" % err) @@ -933,10 +961,10 @@ if HAS_REPORTS: def run(self, setup): verbose = setup.verbose + setup.debug try: - management.call_command("syncdb", interactive=False, - verbosity=verbose) - management.call_command("migrate", interactive=False, - verbosity=verbose) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=verbose) + Bcfg2.DBSettings.migrate_databases(interactive=False, + verbosity=verbose) except: # pylint: disable=W0702 self.errExit("%s failed: %s" % (self.__class__.__name__.title(), diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 0023e9313..769addf55 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -113,7 +113,8 @@ class BuiltinCore(NetworkCore): keyfile=Bcfg2.Options.setup.key, certfile=Bcfg2.Options.setup.cert, register=False, - ca=Bcfg2.Options.setup.ca) + ca=Bcfg2.Options.setup.ca, + protocol=Bcfg2.Options.setup.protocol) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Server startup failed: %s" % err) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 398053374..892f2832a 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -19,14 +19,13 @@ import Bcfg2.Server.Statistics import Bcfg2.Server.FileMonitor from itertools import chain from Bcfg2.Server.Cache import Cache -from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 +from Bcfg2.Compat import xmlrpclib, wraps # pylint: disable=W0622 from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin import track_statistics try: from django.core.exceptions import ImproperlyConfigured - from django.core import management import django.conf HAS_DJANGO = True except ImportError: @@ -74,6 +73,24 @@ def sort_xml(node, key=None): node[:] = sorted_children +def close_db_connection(func): + """ Decorator that closes the Django database connection at the end of + the function. This should decorate any exposed function that + might open a database connection. """ + @wraps(func) + def inner(self, *args, **kwargs): + """ The decorated function """ + rv = func(self, *args, **kwargs) + if self._database_available: # pylint: disable=W0212 + from django import db + self.logger.debug("%s: Closing database connection" % + threading.current_thread().name) + db.close_connection() + return rv + + return inner + + class CoreInitError(Exception): """ Raised when the server core cannot be initialized. """ pass @@ -114,7 +131,8 @@ class Core(object): Bcfg2.Options.Common.repository, Bcfg2.Options.Common.filemonitor, Bcfg2.Options.BooleanOption( - cf=('server', 'fam_blocking'), default=False, + "--no-fam-blocking", cf=('server', 'fam_blocking'), + dest="fam_blocking", default=True, help='FAM blocks on startup until all events are processed'), Bcfg2.Options.BooleanOption( cf=('logging', 'performance'), dest="perflog", @@ -128,6 +146,10 @@ class Core(object): default='off', choices=['off', 'on', 'initial', 'cautious', 'aggressive'])] + #: The name of this server core. This can be overridden by core + #: implementations to provide a more specific name. + name = "Core" + def __init__(self): # pylint: disable=R0912,R0915 """ .. automethod:: _run @@ -196,6 +218,12 @@ class Core(object): self.revision = '-1' atexit.register(self.shutdown) + #: if :func:`Bcfg2.Server.Core.shutdown` is called explicitly, + #: then :mod:`atexit` calls it *again*, so it gets called + #: twice. This is potentially bad, so we use + #: :attr:`Bcfg2.Server.Core._running` as a flag to determine + #: if the core needs to be shutdown, and only do it once. + self._running = True #: Threading event to signal worker threads (e.g., #: :attr:`fam_thread`) to shutdown @@ -236,16 +264,16 @@ class Core(object): self._database_available = False if HAS_DJANGO: try: - management.call_command("syncdb", interactive=False, - verbosity=0) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=0) self._database_available = True except ImproperlyConfigured: - err = sys.exc_info()[1] - self.logger.error("Django configuration problem: %s" % err) + self.logger.error("Django configuration problem: %s" % + sys.exc_info()[1]) except: - err = sys.exc_info()[1] self.logger.error("Updating database %s failed: %s" % - (Bcfg2.Options.setup.db_name, err)) + (Bcfg2.Options.setup.db_name, + sys.exc_info()[1])) def __str__(self): return self.__class__.__name__ @@ -332,7 +360,7 @@ class Core(object): This does not start plugin threads; that is done later, in :func:`Bcfg2.Server.Core.BaseCore.run` """ for plugin in Bcfg2.Options.setup.plugins: - if not plugin in self.plugins: + if plugin not in self.plugins: self.init_plugin(plugin) # Remove blacklisted plugins @@ -403,14 +431,22 @@ class Core(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ - self.logger.info("Shutting down core...") + if not self._running: + self.logger.debug("%s: Core already shut down" % self.name) + return + self.logger.info("%s: Shutting down core..." % self.name) if not self.terminate.isSet(): self.terminate.set() - self.fam.shutdown() - self.logger.info("FAM shut down") - for plugin in list(self.plugins.values()): - plugin.shutdown() - self.logger.info("All plugins shut down") + self._running = False + self.fam.shutdown() + self.logger.info("%s: FAM shut down" % self.name) + for plugin in list(self.plugins.values()): + plugin.shutdown() + self.logger.info("%s: All plugins shut down" % self.name) + if self._database_available: + from django import db + self.logger.info("%s: Closing database connection" % self.name) + db.close_connection() @property def metadata_cache_mode(self): @@ -601,9 +637,10 @@ class Core(object): del entry.attrib['realname'] return ret except: - self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, entry.get('realname'), - entry.get('name'))) + self.logger.error( + "Failed binding entry %s:%s with altsrc %s: %s" % + (entry.tag, entry.get('realname'), entry.get('name'), + sys.exc_info()[1])) entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) @@ -1052,6 +1089,7 @@ class Core(object): @exposed @track_statistics() + @close_db_connection def DeclareVersion(self, address, version): """ Declare the client version. @@ -1074,6 +1112,7 @@ class Core(object): return True @exposed + @close_db_connection def GetProbes(self, address): """ Fetch probes for the client. @@ -1099,6 +1138,7 @@ class Core(object): (client, err)) @exposed + @close_db_connection def RecvProbeData(self, address, probedata): """ Receive probe data from clients. @@ -1146,6 +1186,7 @@ class Core(object): return True @exposed + @close_db_connection def AssertProfile(self, address, profile): """ Set profile for a client. @@ -1165,6 +1206,7 @@ class Core(object): return True @exposed + @close_db_connection def GetConfig(self, address): """ Build config for a client by calling :func:`BuildConfiguration`. @@ -1184,6 +1226,7 @@ class Core(object): self.critical_error("Metadata consistency failure for %s" % client) @exposed + @close_db_connection def RecvStats(self, address, stats): """ Act on statistics upload with :func:`process_statistics`. @@ -1199,6 +1242,7 @@ class Core(object): return True @exposed + @close_db_connection def GetDecisionList(self, address, mode): """ Get the decision list for the client with :func:`GetDecisions`. @@ -1326,8 +1370,16 @@ class NetworkCore(Core): daemonized, etc.""" options = Core.options + [ Bcfg2.Options.Common.daemon, Bcfg2.Options.Common.syslog, - Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_key, - Bcfg2.Options.Common.ssl_cert, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.protocol, + Bcfg2.Options.PathOption( + '--ssl-key', cf=('communication', 'key'), dest="key", + help='Path to SSL key', + default="/etc/pki/tls/private/bcfg2.key"), + Bcfg2.Options.PathOption( + cf=('communication', 'certificate'), dest="cert", + help='Path to SSL certificate', + default="/etc/pki/tls/certs/bcfg2.crt"), Bcfg2.Options.BooleanOption( '--listen-all', cf=('server', 'listen_all'), default=False, help="Listen on all interfaces"), diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py index f8b602d90..b60302871 100755 --- a/src/lib/Bcfg2/Server/Encryption.py +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -173,6 +173,17 @@ def ssl_encrypt(plaintext, passwd, algorithm=None, salt=None): return b64encode("Salted__" + salt + crypted) + "\n" +def is_encrypted(val): + """ Make a best guess if the value is encrypted or not. This just + checks to see if ``val`` is a base64-encoded string whose content + starts with "Salted__", so it may have (rare) false positives. It + will not have false negatives. """ + try: + return b64decode(val).startswith("Salted__") + except: # pylint: disable=W0702 + return False + + def bruteforce_decrypt(crypted, passphrases=None, algorithm=None): """ Convenience method to decrypt the given encrypted string by trying the given passphrases or all passphrases sequentially until @@ -233,6 +244,10 @@ class DecryptError(Exception): """ Exception raised when decryption fails. """ +class EncryptError(Exception): + """ Exception raised when encryption fails. """ + + class CryptoTool(object): """ Generic decryption/encryption interface base object """ @@ -319,6 +334,8 @@ class CfgEncryptor(Encryptor): Bcfg2.Options.setup.config) def encrypt(self): + if is_encrypted(self.data): + raise EncryptError("Data is alraedy encrypted") return ssl_encrypt(self.data, self.passphrase) def get_destination_filename(self, original_filename): @@ -355,7 +372,7 @@ class CfgDecryptor(Decryptor): class PropertiesCryptoMixin(object): """ Mixin to provide some common methods for Properties crypto """ - default_xpath = '//*' + default_xpath = '//*[@encrypted]' def _get_elements(self, xdata): """ Get the list of elements to encrypt or decrypt """ @@ -425,11 +442,13 @@ class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): def encrypt(self): xdata = lxml.etree.XML(self.data, parser=XMLParser) for elt in self._get_elements(xdata): + if is_encrypted(elt.text): + raise EncryptError("Element is already encrypted: %s" % + print_xml(elt)) try: pname, passphrase = self._get_element_passphrase(elt) except PassphraseError: - self.logger.error(str(sys.exc_info()[1])) - return False + raise EncryptError(str(sys.exc_info()[1])) self.logger.debug("Encrypting %s" % print_xml(elt)) elt.text = ssl_encrypt(elt.text, passphrase).strip() elt.set("encrypted", pname) @@ -441,7 +460,6 @@ class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): """ decryptor class for Properties files """ - default_xpath = '//*[@encrypted]' def decrypt(self): decrypted_any = False @@ -640,9 +658,9 @@ class CLI(object): if data is None: try: data = getattr(tool, mode)() - except DecryptError: - self.logger.error("Failed to %s %s, skipping" % (mode, - fname)) + except (EncryptError, DecryptError): + self.logger.error("Failed to %s %s, skipping: %s" % + (mode, fname, sys.exc_info()[1])) continue if Bcfg2.Options.setup.stdout: if len(Bcfg2.Options.setup.files) > 1: diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 69463ab4c..b349d20fd 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -27,11 +27,11 @@ class GaminEvent(Event): class Gamin(FileMonitor): """ File monitor backend with `Gamin - <http://people.gnome.org/~veillard/gamin/>`_ support. """ + <http://people.gnome.org/~veillard/gamin/>`_ support. **Deprecated.** """ - #: The Gamin backend is fairly decent, particularly newer - #: releases, so it has a fairly high priority. - __priority__ = 90 + #: The Gamin backend is deprecated, but better than pseudo, so it + #: has a medium priority. + __priority__ = 50 def __init__(self): FileMonitor.__init__(self) @@ -46,6 +46,9 @@ class Gamin(FileMonitor): #: The queue used to record monitors that are added before #: :func:`start` has been called and :attr:`mon` is created. self.add_q = [] + + self.logger.warning("The Gamin file monitor backend is deprecated. " + "Please switch to a supported file monitor.") __init__.__doc__ = FileMonitor.__init__.__doc__ def start(self): diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index b8eb06aa1..c4b34a469 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -212,7 +212,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): AddMonitor.__doc__ = Pseudo.AddMonitor.__doc__ def shutdown(self): - if self.notifier: + if self.started and self.notifier: self.notifier.stop() shutdown.__doc__ = Pseudo.shutdown.__doc__ diff --git a/src/lib/Bcfg2/Server/Lint/Bundler.py b/src/lib/Bcfg2/Server/Lint/Bundler.py index 0caf4d7ed..aee15cb5d 100644 --- a/src/lib/Bcfg2/Server/Lint/Bundler.py +++ b/src/lib/Bcfg2/Server/Lint/Bundler.py @@ -1,12 +1,12 @@ """ ``bcfg2-lint`` plugin for :ref:`Bundler -<server-plugins-structures-bundler-index>` """ +<server-plugins-structures-bundler>` """ from Bcfg2.Server.Lint import ServerPlugin class Bundler(ServerPlugin): """ Perform various :ref:`Bundler - <server-plugins-structures-bundler-index>` checks. """ + <server-plugins-structures-bundler>` checks. """ def Run(self): self.missing_bundles() diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index e2d1ec597..fc4506c12 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -9,6 +9,7 @@ from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ import CfgPlaintextGenerator from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML @@ -76,6 +77,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for Cheetah-templated Cfg files"), Bcfg2.Options.Option( + cf=("Comments", "jinja2_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Jinja2-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "jinja2_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Jinja2-templated Cfg files"), + Bcfg2.Options.Option( cf=("Comments", "infoxml_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for info.xml files"), @@ -235,6 +244,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): rtype = "cfg" elif isinstance(entry, CfgCheetahGenerator): rtype = "cheetah" + elif isinstance(entry, CfgJinja2Generator): + rtype = "jinja2" elif isinstance(entry, CfgInfoXML): self.check_xml(entry.infoxml.name, entry.infoxml.pnode.data, diff --git a/src/lib/Bcfg2/Server/Lint/Crypto.py b/src/lib/Bcfg2/Server/Lint/Crypto.py new file mode 100644 index 000000000..53a54031c --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Crypto.py @@ -0,0 +1,61 @@ +""" Check for data that claims to be encrypted, but is not. """ + +import os +import lxml.etree +import Bcfg2.Options +from Bcfg2.Server.Lint import ServerlessPlugin +from Bcfg2.Server.Encryption import is_encrypted + + +class Crypto(ServerlessPlugin): + """ Check for templated scripts or executables. """ + + def Run(self): + if os.path.exists(os.path.join(Bcfg2.Options.setup.repository, "Cfg")): + self.check_cfg() + if os.path.exists(os.path.join(Bcfg2.Options.setup.repository, + "Properties")): + self.check_properties() + # TODO: check all XML files + + @classmethod + def Errors(cls): + return {"unencrypted-cfg": "error", + "empty-encrypted-properties": "error", + "unencrypted-properties": "error"} + + def check_cfg(self): + """ Check for Cfg files that end in .crypt but aren't encrypted """ + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, "Cfg")): + for fname in files: + fpath = os.path.join(root, fname) + if self.HandlesFile(fpath) and fname.endswith(".crypt"): + if not is_encrypted(open(fpath).read()): + self.LintError( + "unencrypted-cfg", + "%s is a .crypt file, but it is not encrypted" % + fpath) + + def check_properties(self): + """ Check for Properties data that has an ``encrypted`` attribute but + aren't encrypted """ + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, "Properties")): + for fname in files: + fpath = os.path.join(root, fname) + if self.HandlesFile(fpath) and fname.endswith(".xml"): + xdata = lxml.etree.parse(fpath) + for elt in xdata.xpath('//*[@encrypted]'): + if not elt.text: + self.LintError( + "empty-encrypted-properties", + "Element in %s has an 'encrypted' attribute, " + "but no text content: %s" % + (fpath, self.RenderXML(elt))) + elif not is_encrypted(elt.text): + self.LintError( + "unencrypted-properties", + "Element in %s has an 'encrypted' attribute, " + "but is not encrypted: %s" % + (fpath, self.RenderXML(elt))) diff --git a/src/lib/Bcfg2/Server/Lint/Jinja2.py b/src/lib/Bcfg2/Server/Lint/Jinja2.py new file mode 100755 index 000000000..333249cc2 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Jinja2.py @@ -0,0 +1,41 @@ +""" Check Jinja2 templates for syntax errors. """ + +import sys +import Bcfg2.Server.Lint +from jinja2 import Template, TemplateSyntaxError +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator + + +class Jinja2(Bcfg2.Server.Lint.ServerPlugin): + """ Check Jinja2 templates for syntax errors. """ + + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + + @classmethod + def Errors(cls): + return {"jinja2-syntax-error": "error", + "unknown-jinja2-error": "error"} + + def check_template(self, entry): + """ Generic check for all jinja2 templates """ + try: + Template(entry.data.decode(entry.encoding)) + except TemplateSyntaxError: + err = sys.exc_info()[1] + self.LintError("jinja2-syntax-error", + "Jinja2 syntax error in %s: %s" % (entry.name, err)) + except: + err = sys.exc_info()[1] + self.LintError("unknown-jinja2-error", + "Unknown Jinja2 error in %s: %s" % (entry.name, + err)) + + def check_cfg(self): + """ Check jinja2 templates in Cfg for syntax errors. """ + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if (self.HandlesFile(entry.name) and + isinstance(entry, CfgJinja2Generator)): + self.check_template(entry) diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 5d9e229fa..ebf4c4954 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -123,12 +123,30 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"unknown-entry-type": "error", + return {"missing-elements": "error", + "unknown-entry-type": "error", "unknown-entry-tag": "error", "required-attrs-missing": "error", "required-attr-format": "error", "extra-attrs": "warning"} + def check_default_acl(self, path): + """ Check that a default ACL contains either no entries or minimum + required entries """ + defaults = 0 + if path.xpath("ACL[@type='default' and @scope='user' and @user='']"): + defaults += 1 + if path.xpath("ACL[@type='default' and @scope='group' and @group='']"): + defaults += 1 + if path.xpath("ACL[@type='default' and @scope='other']"): + defaults += 1 + if defaults > 0 and defaults < 3: + self.LintError( + "missing-elements", + "A Path must have either no default ACLs or at" + " least default:user::, default:group:: and" + " default:other::") + def check_packages(self): """ Check Packages sources for Source entries with missing attributes. """ @@ -172,7 +190,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): rules.name)) def check_bundles(self): - """ Check bundles for BoundPath entries with missing + """ Check bundles for BoundPath and BoundPackage entries with missing attrs. """ if 'Bundler' not in self.core.plugins: return @@ -183,6 +201,25 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): "//*[substring(name(), 1, 5) = 'Bound']"): self.check_entry(path, bundle.name) + # ensure that abstract Path tags have either name + # or glob specified + for path in bundle.xdata.xpath("//Path"): + if ('name' not in path.attrib and + 'glob' not in path.attrib): + self.LintError( + "required-attrs-missing", + "Path tags require either a 'name' or 'glob' " + "attribute: \n%s" % self.RenderXML(path)) + # ensure that abstract Package tags have either name + # or group specified + for package in bundle.xdata.xpath("//Package"): + if ('name' not in package.attrib and + 'group' not in package.attrib): + self.LintError( + "required-attrs-missing", + "Package tags require either a 'name' or 'group' " + "attribute: \n%s" % self.RenderXML(package)) + def check_entry(self, entry, filename): """ Generic entry check. @@ -221,6 +258,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): required_attrs['major'] = is_device_mode required_attrs['minor'] = is_device_mode + if tag == 'Path': + self.check_default_acl(entry) + if tag == 'ACL' and 'scope' in required_attrs: required_attrs[entry.get('scope')] = is_username diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py new file mode 100644 index 000000000..5a80a5884 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py @@ -0,0 +1,80 @@ +""" Check for templated scripts or executables. """ + +import os +import stat +import Bcfg2.Server.Lint +from Bcfg2.Compat import any # pylint: disable=W0622 +from Bcfg2.Server.Plugin import default_path_metadata +from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML +from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator import \ + CfgEncryptedGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator import \ + CfgEncryptedCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator import \ + CfgEncryptedJinja2Generator + + +class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin): + """ Check for templated scripts or executables. """ + templates = [CfgGenshiGenerator, CfgCheetahGenerator, CfgJinja2Generator, + CfgEncryptedGenshiGenerator, CfgEncryptedCheetahGenerator, + CfgEncryptedJinja2Generator] + extensions = [".pl", ".py", ".sh", ".rb"] + + def Run(self): + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if (self.HandlesFile(entry.name) and + any(isinstance(entry, t) for t in self.templates)): + self.check_template(entryset, entry) + + @classmethod + def Errors(cls): + return {"templated-script": "warning", + "templated-executable": "warning"} + + def check_template(self, entryset, entry): + """ Check a template to see if it's a script or an executable. """ + # first, check for a known script extension + ext = os.path.splitext(entryset.path)[1] + if ext in self.extensions: + self.LintError("templated-script", + "Templated script found: %s\n" + "File has a known script extension: %s\n" + "Template a config file for the script instead" % + (entry.name, ext)) + return + + # next, check for a shebang line + firstline = open(entry.name).readline() + if firstline.startswith("#!"): + self.LintError("templated-script", + "Templated script found: %s\n" + "File starts with a shebang: %s\n" + "Template a config file for the script instead" % + (entry.name, firstline)) + return + + # finally, check for executable permissions in info.xml + for entry in entryset.entries.values(): + if isinstance(entry, CfgInfoXML): + for pinfo in entry.infoxml.pnode.data.xpath("//FileInfo"): + try: + mode = int( + pinfo.get("mode", + default_path_metadata()['mode']), 8) + except ValueError: + # LintError will be produced by RequiredAttrs plugin + self.logger.warning("Non-octal mode: %s" % mode) + continue + if mode & stat.S_IXUSR != 0: + self.LintError( + "templated-executable", + "Templated executable found: %s\n" + "Template a config file for the executable instead" + % entry.name) + return diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py index fbd5a2893..a952da724 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -23,9 +23,11 @@ class TemplateHelper(ServerPlugin): def __init__(self, *args, **kwargs): ServerPlugin.__init__(self, *args, **kwargs) - self.reserved_keywords = dir(HelperModule("foo.py")) - self.reserved_defaults = \ - self.core.plugins['TemplateHelper'].reserved_defaults + # we instantiate a dummy helper to discover which keywords and + # defaults are reserved + dummy = HelperModule("foo.py") + self.reserved_keywords = dir(dummy) + self.reserved_defaults = dummy.reserved_defaults def Run(self): for helper in self.core.plugins['TemplateHelper'].entries.values(): diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index e38619355..0b3f1e24d 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -90,6 +90,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "xml-failed-to-parse": "error", "xml-failed-to-read": "error", "xml-failed-to-verify": "error", + "xinclude-does-not-exist": "error", "input-output-error": "error"} def check_properties(self): @@ -113,9 +114,17 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): :type filename: string :returns: lxml.etree._ElementTree - the parsed data""" try: - return lxml.etree.parse(filename) - except SyntaxError: - result = self.cmd.run(["xmllint", filename]) + xdata = lxml.etree.parse(filename) + if self.files is None: + self._expand_wildcard_xincludes(xdata) + xdata.xinclude() + return xdata + except (lxml.etree.XIncludeError, SyntaxError): + cmd = ["xmllint", "--noout"] + if self.files is None: + cmd.append("--xinclude") + cmd.append(filename) + result = self.cmd.run(cmd) self.LintError("xml-failed-to-parse", "%s fails to parse:\n%s" % (filename, result.stdout + result.stderr)) @@ -125,6 +134,33 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Failed to open file %s" % filename) return False + def _expand_wildcard_xincludes(self, xdata): + """ a lightweight version of + :func:`Bcfg2.Server.Plugin.helpers.XMLFileBacked._follow_xincludes` """ + xinclude = '%sinclude' % Bcfg2.Server.XI_NAMESPACE + for el in xdata.findall('//' + xinclude): + name = el.get("href") + if name.startswith("/"): + fpath = name + else: + fpath = os.path.join(os.path.dirname(xdata.docinfo.URL), name) + + # expand globs in xinclude, a bcfg2-specific extension + extras = glob.glob(fpath) + if not extras: + msg = "%s: %s does not exist, skipping: %s" % \ + (xdata.docinfo.URL, name, self.RenderXML(el)) + if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): + self.logger.debug(msg) + else: + self.LintError("xinclude-does-not-exist", msg) + + parent = el.getparent() + parent.remove(el) + for extra in extras: + if extra != xdata.docinfo.URL: + lxml.etree.SubElement(parent, xinclude, href=extra) + def validate(self, filename, schemafile, schema=None): """ Validate a file against the given schema. @@ -146,6 +182,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if not schema: return False datafile = self.parse(filename) + if not datafile: + return False if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py new file mode 100644 index 000000000..6383a3c99 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -0,0 +1,72 @@ +"""Ensure that all JSON files in the Bcfg2 repository are +valid. Currently, the only plugins that uses JSON are Ohai and +Properties.""" + +import os +import sys +import glob +import fnmatch +import Bcfg2.Server.Lint + +try: + import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): + import simplejson as json + + +class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): + """Ensure that all JSON files in the Bcfg2 repository are + valid. Currently, the only plugins that uses JSON are Ohai and + Properties. """ + + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + + #: A list of file globs that give the path to JSON files. The + #: globs are extended :mod:`fnmatch` globs that also support + #: ``**``, which matches any number of any characters, + #: including forward slashes. + self.globs = ["Properties/*.json", "Ohai/*.json"] + self.files = self.get_files() + + def Run(self): + for path in self.files: + self.logger.debug("Validating JSON in %s" % path) + try: + json.load(open(path)) + except ValueError: + self.LintError("json-failed-to-parse", + "%s does not contain valid JSON: %s" % + (path, sys.exc_info()[1])) + + @classmethod + def Errors(cls): + return {"json-failed-to-parse": "error"} + + def get_files(self): + """Return a list of all JSON files to validate, based on + :attr:`Bcfg2.Server.Lint.ValidateJSON.ValidateJSON.globs`. """ + if self.files is not None: + listfiles = lambda p: fnmatch.filter(self.files, + os.path.join('*', p)) + else: + listfiles = lambda p: glob.glob( + os.path.join(Bcfg2.Options.setup.repository, p)) + + rv = [] + for path in self.globs: + if '/**/' in path: + if self.files is not None: + rv.extend(listfiles(path)) + else: # self.files is None + fpath, fname = path.split('/**/') + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, + fpath)): + rv.extend([os.path.join(root, f) + for f in files if f == fname]) + else: + rv.extend(listfiles(path)) + return rv diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 8a793fd94..9b3e6ece2 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -13,6 +13,7 @@ import lxml.etree import Bcfg2.Options import Bcfg2.Server.Core import Bcfg2.Server.Plugins +from Bcfg2.Compat import walk_packages def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -297,11 +298,10 @@ class LintPluginAction(Bcfg2.Options.ComponentAction): bases = ['Bcfg2.Server.Lint'] def __call__(self, parser, namespace, values, option_string=None): - for plugin in getattr(Bcfg2.Options.setup, "plugins", []): - module = sys.modules[plugin.__module__] - if hasattr(module, "%sLint" % plugin.name): - print("Adding lint plugin %s" % plugin) - values.append(plugin) + plugins = getattr(Bcfg2.Options.setup, "plugins", []) + for lint_plugin in walk_packages(path=__path__): + if lint_plugin[1] in plugins: + values.append(lint_plugin[1]) Bcfg2.Options.ComponentAction.__call__(self, parser, namespace, values, option_string) diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 294963669..724b34d8d 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -275,6 +275,7 @@ class ChildCore(Core): @exposed def GetConfig(self, client): """ Render the configuration for a client """ + self.metadata.update_client_list() self.logger.debug("%s: Building configuration for %s" % (self.name, client)) return lxml.etree.tostring(self.BuildConfiguration(client)) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 1cb5a7b3e..559612d1e 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -18,7 +18,7 @@ from Bcfg2.Compat import CmpMixin, wraps from Bcfg2.Server.Plugin.base import Plugin from Bcfg2.Server.Plugin.interfaces import Generator, TemplateDataProvider from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ - PluginExecutionError + PluginExecutionError, PluginInitError try: import Bcfg2.Server.Encryption @@ -219,6 +219,18 @@ class DatabaseBacked(Plugin): .. private-include: _must_lock """ + def __init__(self, core): + Plugin.__init__(self, core) + use_db = getattr(Bcfg2.Options.setup, "%s_db" % self.name.lower(), + False) + if use_db and not HAS_DJANGO: + raise PluginInitError("%s is configured to use the database but " + "Django libraries are not found" % self.name) + elif use_db and not self.core.database_available: + raise PluginInitError("%s is configured to use the database but " + "the database is unavailable due to prior " + "errors" % self.name) + @property def _use_db(self): """ Whether or not this plugin is configured to use the @@ -227,11 +239,7 @@ class DatabaseBacked(Plugin): False) if use_db and HAS_DJANGO and self.core.database_available: return True - elif not use_db: - return False else: - self.logger.error("%s: use_database is true but django not found" % - self.name) return False @property @@ -267,7 +275,8 @@ class PluginDatabaseModel(object): inherit from. This is just a mixin; models must also inherit from django.db.models.Model to be valid Django models.""" - class Meta: # pylint: disable=C0111,W0232 + class Meta(object): # pylint: disable=W0232 + """ Model metadata options """ app_label = "Server" @@ -638,7 +647,13 @@ class XMLFileBacked(FileBacked): if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): self.logger.debug(msg) else: - self.logger.warning(msg) + self.logger.error(msg) + # add a FAM monitor for this path. this isn't perfect + # -- if there's an xinclude of "*.xml", we'll watch + # the literal filename "*.xml". but for non-globbing + # filenames, it works fine. + if fpath not in self.extra_monitors: + self.add_monitor(fpath) parent = el.getparent() parent.remove(el) @@ -748,9 +763,6 @@ class StructFile(XMLFileBacked): err)) if HAS_CRYPTO and self.encryption: - lax_decrypt = self.xdata.get( - "lax_decryption", - str(Bcfg2.Options.setup.lax_decryption)).lower() == "true" for el in self.xdata.xpath("//*[@encrypted]"): try: el.text = self._decrypt(el).encode('ascii', @@ -759,10 +771,14 @@ class StructFile(XMLFileBacked): self.logger.info("%s: Decrypted %s to gibberish, skipping" % (self.name, el.tag)) except Bcfg2.Server.Encryption.EVPError: + lax_decrypt = self.xdata.get( + "lax_decryption", + str(Bcfg2.Options.setup.lax_decryption)).lower() == \ + "true" msg = "Failed to decrypt %s element in %s" % (el.tag, self.name) if lax_decrypt: - self.logger.warning(msg) + self.logger.debug(msg) else: raise PluginExecutionError(msg) Index.__doc__ = XMLFileBacked.Index.__doc__ @@ -774,16 +790,11 @@ class StructFile(XMLFileBacked): passes = Bcfg2.Options.setup.passphrases try: passphrase = passes[element.get("encrypted")] - try: - return Bcfg2.Server.Encryption.ssl_decrypt(element.text, - passphrase) - except Bcfg2.Server.Encryption.EVPError: - # error is raised below - pass + return Bcfg2.Server.Encryption.ssl_decrypt(element.text, + passphrase) except KeyError: - # bruteforce_decrypt raises an EVPError with a sensible - # error message, so we just let it propagate up the stack - return Bcfg2.Server.Encryption.bruteforce_decrypt(element.text) + raise Bcfg2.Server.Encryption.EVPError("No passphrase named '%s'" % + element.get("encrypted")) raise Bcfg2.Server.Encryption.EVPError("Failed to decrypt") def _include_element(self, item, metadata, *args): @@ -818,7 +829,8 @@ class StructFile(XMLFileBacked): """ stream = self.template.generate( **get_xml_template_data(self, metadata)).filter(removecomment) - return lxml.etree.XML(stream.render('xml', strip_whitespace=False), + return lxml.etree.XML(stream.render('xml', + strip_whitespace=False).encode(), parser=Bcfg2.Server.XMLParser) def _match(self, item, metadata, *args): @@ -935,7 +947,7 @@ class InfoXML(StructFile): _include_tests = copy.copy(StructFile._include_tests) _include_tests['Path'] = lambda el, md, entry, *args: \ - entry.get("name") == el.get("name") + entry.get('realname', entry.get('name')) == el.get("name") def Match(self, metadata, entry): # pylint: disable=W0221 """ Implementation of diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 622b69c79..c45d6fa84 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -216,6 +216,10 @@ class Metadata(object): """ raise NotImplementedError + def update_client_list(self): + """ Re-read the cached list of clients """ + raise NotImplementedError + class Connector(object): """ Connector plugins augment client metadata instances with diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 8b9330c9b..41ee57b6d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -4,31 +4,30 @@ import os import re import sys import copy -import Bcfg2.Server -import Bcfg2.Server.Plugin +import fnmatch +import lxml.etree +from Bcfg2.Server.Plugin import StructFile, Plugin, Structure, \ + StructureValidator, XMLDirectoryBacked, Generator from genshi.template import TemplateError -class BundleFile(Bcfg2.Server.Plugin.StructFile): +class BundleFile(StructFile): """ Representation of a bundle XML file """ bundle_name_re = re.compile(r'^(?P<name>.*)\.(xml|genshi)$') def __init__(self, filename, should_monitor=False): - Bcfg2.Server.Plugin.StructFile.__init__(self, filename, - should_monitor=should_monitor) + StructFile.__init__(self, filename, should_monitor=should_monitor) if self.name.endswith(".genshi"): self.logger.warning("Bundler: %s: Bundle filenames ending with " ".genshi are deprecated; add the Genshi XML " "namespace to a .xml bundle instead" % self.name) - __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ def Index(self): - Bcfg2.Server.Plugin.StructFile.Index(self) + StructFile.Index(self) if self.xdata.get("name"): self.logger.warning("Bundler: %s: Explicitly specifying bundle " "names is deprecated" % self.name) - Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ @property def bundle_name(self): @@ -37,9 +36,10 @@ class BundleFile(Bcfg2.Server.Plugin.StructFile): os.path.basename(self.name)).group("name") -class Bundler(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): +class Bundler(Plugin, + Structure, + StructureValidator, + XMLDirectoryBacked): """ The bundler creates dependent clauses based on the bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' @@ -47,18 +47,30 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, patterns = re.compile(r'^.*\.(?:xml|genshi)$') def __init__(self, core): - Bcfg2.Server.Plugin.Plugin.__init__(self, core) - Bcfg2.Server.Plugin.Structure.__init__(self) - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data) + Plugin.__init__(self, core) + Structure.__init__(self) + StructureValidator.__init__(self) + XMLDirectoryBacked.__init__(self, self.data) #: Bundles by bundle name, rather than filename self.bundles = dict() def HandleEvent(self, event): - Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - + XMLDirectoryBacked.HandleEvent(self, event) self.bundles = dict([(b.bundle_name, b) for b in self.entries.values()]) + def validate_structures(self, metadata, structures): + """ Translate <Path glob='...'/> entries into <Path name='...'/> + entries """ + for struct in structures: + for pathglob in struct.xpath("//Path[@glob]"): + for plugin in self.core.plugins_by_type(Generator): + for match in fnmatch.filter(plugin.Entries['Path'].keys(), + pathglob.get("glob")): + lxml.etree.SubElement(pathglob.getparent(), + "Path", name=match) + pathglob.getparent().remove(pathglob) + def BuildStructures(self, metadata): bundleset = [] bundles = copy.copy(metadata.bundles) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index e2a2f696a..849c75f70 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -1,6 +1,7 @@ """ CfgEncryptedGenerator lets you encrypt your plaintext :ref:`server-plugins-generators-cfg` files on the server. """ +import Bcfg2.Options from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: @@ -25,7 +26,6 @@ class CfgEncryptedGenerator(CfgGenerator): CfgGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") - __init__.__doc__ = CfgGenerator.__init__.__doc__ def handle_event(self, event): CfgGenerator.handle_event(self, event) @@ -35,11 +35,13 @@ class CfgEncryptedGenerator(CfgGenerator): try: self.data = bruteforce_decrypt(self.data) except EVPError: - raise PluginExecutionError("Failed to decrypt %s" % self.name) - handle_event.__doc__ = CfgGenerator.handle_event.__doc__ + msg = "Cfg: Failed to decrypt %s" % self.name + if Bcfg2.Options.setup.lax_decryption: + self.logger.debug(msg) + else: + raise PluginExecutionError(msg) def get_data(self, entry, metadata): if self.data is None: raise PluginExecutionError("Failed to decrypt %s" % self.name) return CfgGenerator.get_data(self, entry, metadata) - get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py new file mode 100644 index 000000000..c8da84ae0 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py @@ -0,0 +1,25 @@ +""" Handle encrypted Jinja2 templates (.crypt.jinja2 or +.jinja2.crypt files)""" + +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator \ + import CfgEncryptedGenerator + + +class CfgEncryptedJinja2Generator(CfgJinja2Generator, CfgEncryptedGenerator): + """ CfgEncryptedJinja2Generator lets you encrypt your Jinja2 + :ref:`server-plugins-generators-cfg` files on the server """ + + #: handle .crypt.jinja2 or .jinja2.crypt files + __extensions__ = ['jinja2.crypt', 'crypt.jinja2'] + + #: Override low priority from parent class + __priority__ = 0 + + def handle_event(self, event): + CfgEncryptedGenerator.handle_event(self, event) + handle_event.__doc__ = CfgEncryptedGenerator.handle_event.__doc__ + + def get_data(self, entry, metadata): + return CfgJinja2Generator.get_data(self, entry, metadata) + get_data.__doc__ = CfgJinja2Generator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py new file mode 100644 index 000000000..e36ee78aa --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py @@ -0,0 +1,52 @@ +""" The CfgJinja2Generator allows you to use the `Jinja2 +<http://jinja.pocoo.org/>`_ templating system to generate +:ref:`server-plugins-generators-cfg` files. """ + +import Bcfg2.Options +from Bcfg2.Server.Plugin import PluginExecutionError, \ + DefaultTemplateDataProvider, get_template_data +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + +try: + from jinja2 import Template + HAS_JINJA2 = True +except ImportError: + HAS_JINJA2 = False + + +class DefaultJinja2DataProvider(DefaultTemplateDataProvider): + """ Template data provider for Jinja2 templates. Jinja2 and + Genshi currently differ over the value of the ``path`` variable, + which is why this is necessary. """ + + def get_template_data(self, entry, metadata, template): + rv = DefaultTemplateDataProvider.get_template_data(self, entry, + metadata, template) + rv['path'] = rv['name'] + return rv + + +class CfgJinja2Generator(CfgGenerator): + """ The CfgJinja2Generator allows you to use the `Jinja2 + <http://jinja.pocoo.org/>`_ templating system to generate + :ref:`server-plugins-generators-cfg` files. """ + + #: Handle .jinja2 files + __extensions__ = ['jinja2'] + + #: Low priority to avoid matching host- or group-specific + #: .crypt.jinja2 files + __priority__ = 50 + + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) + if not HAS_JINJA2: + raise PluginExecutionError("Jinja2 is not available") + __init__.__doc__ = CfgGenerator.__init__.__doc__ + + def get_data(self, entry, metadata): + template = Template(self.data.decode(Bcfg2.Options.setup.encoding)) + return template.render( + get_template_data(entry, metadata, self.name, + default=DefaultJinja2DataProvider())) + get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index e9698f526..8cc3f7b21 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -34,7 +34,6 @@ class CfgPrivateKeyCreator(XMLCfgCreator): pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path)) self.pubkey_creator = CfgPublicKeyCreator(pubkey_name) self.cmd = Executor() - __init__.__doc__ = XMLCfgCreator.__init__.__doc__ def _gen_keypair(self, metadata, spec=None): """ Generate a keypair according to the given client medata diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index d2b982349..5dc3d98eb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -872,8 +872,7 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, """ The Cfg plugin provides a repository to describe configuration file contents for clients. In its simplest form, the Cfg repository is just a directory tree modeled off of the directory tree on your client - machines. - """ + machines. """ __author__ = 'bcfg-dev@mcs.anl.gov' es_cls = CfgEntrySet es_child_cls = Bcfg2.Server.Plugin.SpecificData diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index 3d3ef8f8c..b30a9acea 100644 --- a/src/lib/Bcfg2/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -31,4 +31,4 @@ class Decisions(Bcfg2.Server.Plugin.Plugin, self.blacklist = DecisionFile(os.path.join(self.data, "blacklist.xml")) def GetDecisions(self, metadata, mode): - return getattr(self, mode).get_decision(metadata) + return getattr(self, mode).get_decisions(metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 78f86f28e..1d15656af 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -674,6 +674,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if attribs is None: attribs = dict() if self._use_db: + if attribs: + msg = "Metadata does not support setting client attributes " +\ + "with use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: client = MetadataClientModel.objects.get(hostname=client_name) except MetadataClientModel.DoesNotExist: @@ -681,7 +686,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, client = MetadataClientModel(hostname=client_name) # pylint: enable=E1102 client.save() - self.clients = self.list_clients() + self.update_client_list() return client else: try: @@ -734,7 +739,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, attribs, alias=True) def list_clients(self): - """ List all clients in client database """ + """ List all clients in client database. + + Making ``self.clients`` a property and reading the client list + dynamically from the database on every call to + ``self.clients`` can result in very high rates of database + reads, so we cache the ``list_clients()`` results to reduce + the database load. When the database is in use, the client + list is reread periodically with + :func:`Bcfg2.Server.Plugins.Metadata.update_client_list`. """ if self._use_db: return set([c.hostname for c in MetadataClientModel.objects.all()]) else: @@ -785,13 +798,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.warning(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) client.delete() - self.clients = self.list_clients() + self.update_client_list() else: return self._remove_xdata(self.clients_xml, "Client", client_name) def _handle_clients_xml_event(self, _): # pylint: disable=R0912 """ handle all events for clients.xml and files xincluded from clients.xml """ + # disable metadata builds during parsing. this prevents + # clients from getting bogus metadata during the brief time it + # takes to rebuild the clients.xml data + self.states['clients.xml'] = False + xdata = self.clients_xml.xdata self.clients = [] self.clientgroups = {} @@ -853,9 +871,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.clientgroups[clname].append(profile) except KeyError: self.clientgroups[clname] = [profile] + self.update_client_list() + self.cache.expire() self.states['clients.xml'] = True - if self._use_db: - self.clients = self.list_clients() def _get_condition(self, element): """ Return a predicate that returns True if a client meets @@ -883,7 +901,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _handle_groups_xml_event(self, _): # pylint: disable=R0912 """ re-read groups.xml on any event on it """ + # disable metadata builds during parsing. this prevents + # clients from getting bogus metadata during the brief time it + # takes to rebuild the groups.xml data + self.states['groups.xml'] = False + self.groups = {} + self.group_membership = dict() + self.negated_groups = dict() + self.ordered_groups = [] # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of @@ -908,10 +934,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if grp.get('default', 'false') == 'true': self.default = grp.get('name') - self.group_membership = dict() - self.negated_groups = dict() - self.ordered_groups = [] - # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is # infinitely recursive, so "all" elements really means _all_ @@ -944,6 +966,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership.setdefault(gname, []) self.group_membership[gname].append( self._aggregate_conditions(conditions)) + self.cache.expire() self.states['groups.xml'] = True def HandleEvent(self, event): @@ -1447,6 +1470,32 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return True # pylint: enable=R0911,R0912 + def update_client_list(self): + """ Re-read the client list from the database (if the database is in + use) """ + if self._use_db: + self.logger.debug("Metadata: Re-reading client list from database") + old = set(self.clients) + self.clients = self.list_clients() + + # we could do this with set.symmetric_difference(), but we + # want detailed numbers of added/removed clients for + # logging + new = set(self.clients) + added = new - old + removed = old - new + self.logger.debug("Metadata: Added %s clients: %s" % + (len(added), added)) + self.logger.debug("Metadata: Removed %s clients: %s" % + (len(removed), removed)) + + for client in added.union(removed): + self.cache.expire(client) + + def start_client_run(self, metadata): + """ Hook to reread client list if the database is in use """ + self.update_client_list() + def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index ba7baab11..c5fb46c97 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -10,7 +10,9 @@ import Bcfg2.Server.Plugin try: import json -except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): import simplejson as json PROBECODE = """#!/bin/sh diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index dba56eed2..3d5c68e3f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -69,12 +69,11 @@ class AptSource(Source): else: return ["%sPackages.gz" % self.rawurl] - def read_files(self): + def read_files(self): # pylint: disable=R0912 bdeps = dict() + brecs = dict() bprov = dict() - depfnames = ['Depends', 'Pre-Depends'] - if self.recommended: - depfnames.append('Recommends') + self.essentialpkgs = set() for fname in self.files: if not self.rawurl: barch = [x @@ -86,6 +85,7 @@ class AptSource(Source): barch = self.arches[0] if barch not in bdeps: bdeps[barch] = dict() + brecs[barch] = dict() bprov[barch] = dict() try: reader = gzip.GzipFile(fname) @@ -100,9 +100,10 @@ class AptSource(Source): pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: self.essentialpkgs.add(pkgname) - elif words[0] in depfnames: + elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): if '|' in dep: @@ -113,17 +114,24 @@ class AptSource(Source): barch, vindex) vindex += 1 - bdeps[barch][pkgname].append(dyn_dname) + + if words[0] == 'Recommends': + brecs[barch][pkgname].append(dyn_dname) + else: + bdeps[barch][pkgname].append(dyn_dname) bprov[barch][dyn_dname] = set(cdeps) else: raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() - bdeps[barch][pkgname].append(raw_dep) + if words[0] == 'Recommends': + brecs[barch][pkgname].append(raw_dep) + else: + bdeps[barch][pkgname].append(raw_dep) elif words[0] == 'Provides': for pkg in words[1].split(','): dname = pkg.rstrip().strip() if dname not in bprov[barch]: bprov[barch][dname] = set() bprov[barch][dname].add(pkgname) - self.process_files(bdeps, bprov) + self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 8b20df58a..004e27874 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -289,7 +289,7 @@ class Collection(list, Debuggable): return any(source.is_virtual_package(self.metadata, package) for source in self) - def get_deps(self, package): + def get_deps(self, package, recs=None): """ Get a list of the dependencies of the given package. The base implementation simply aggregates the results of @@ -299,9 +299,14 @@ class Collection(list, Debuggable): :type package: string :returns: list of strings, but see :ref:`pkg-objects` """ + recommended = None + if recs and package in recs: + recommended = recs[package] + for source in self: if source.is_package(self.metadata, package): - return source.get_deps(self.metadata, package) + return source.get_deps(self.metadata, package, recommended) + return [] def get_essential(self): @@ -465,7 +470,8 @@ class Collection(list, Debuggable): return list(complete.difference(initial)) @track_statistics() - def complete(self, packagelist): # pylint: disable=R0912,R0914 + def complete(self, packagelist, # pylint: disable=R0912,R0914 + recommended=None): """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the @@ -529,7 +535,7 @@ class Collection(list, Debuggable): self.debug_log("Packages: handling package requirement %s" % (current,)) packages.add(current) - deps = self.get_deps(current) + deps = self.get_deps(current, recommended) newdeps = set(deps).difference(examined) if newdeps: self.debug_log("Packages: Package %s added requirements %s" diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py new file mode 100644 index 000000000..e393cabfe --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py @@ -0,0 +1,86 @@ +""" pkgng backend for :mod:`Bcfg2.Server.Plugins.Packages` """ + +import lzma +import tarfile + +try: + import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): + import simplejson as json + +from Bcfg2.Server.Plugins.Packages.Collection import Collection +from Bcfg2.Server.Plugins.Packages.Source import Source + + +class PkgngCollection(Collection): + """ Handle collections of pkgng sources. This is a no-op object + that simply inherits from + :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`, + overrides nothing, and defers all operations to :class:`PacSource` + """ + + def __init__(self, metadata, sources, cachepath, basepath, debug=False): + # we define an __init__ that just calls the parent __init__, + # so that we can set the docstring on __init__ to something + # different from the parent __init__ -- namely, the parent + # __init__ docstring, minus everything after ``.. -----``, + # which we use to delineate the actual docs from the + # .. autoattribute hacks we have to do to get private + # attributes included in sphinx 1.0 """ + Collection.__init__(self, metadata, sources, cachepath, basepath, + debug=debug) + __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] + + +class PkgngSource(Source): + """ Handle pkgng sources """ + + #: PkgngSource sets the ``type`` on Package entries to "pkgng" + ptype = 'pkgng' + + @property + def urls(self): + """ A list of URLs to the base metadata file for each + repository described by this source. """ + if not self.rawurl: + rv = [] + for part in self.components: + for arch in self.arches: + rv.append("%s/freebsd:%s:%s/%s/packagesite.txz" % + (self.url, self.version, arch, part)) + return rv + else: + return ["%s/packagesite.txz" % self.rawurl] + + def read_files(self): + bdeps = dict() + for fname in self.files: + if not self.rawurl: + abi = [x + for x in fname.split('@') + if x.startswith('freebsd:')][0][8:] + barch = ':'.join(abi.split(':')[1:]) + else: + # RawURL entries assume that they only have one <Arch></Arch> + # element and that it is the architecture of the source. + barch = self.arches[0] + if barch not in bdeps: + bdeps[barch] = dict() + try: + tar = tarfile.open(fileobj=lzma.LZMAFile(fname)) + reader = tar.extractfile('packagesite.yaml') + except: + self.logger.error("Packages: Failed to read file %s" % fname) + raise + for line in reader.readlines(): + if not isinstance(line, str): + line = line.decode('utf-8') + pkg = json.loads(line) + pkgname = pkg['name'] + self.pkgnames.add(pkgname) + if 'deps' in pkg: + bdeps[barch][pkgname] = pkg['deps'].keys() + self.process_files(bdeps, dict()) + read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 4b6130f72..24db2963d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -246,6 +246,10 @@ class Source(Debuggable): # pylint: disable=R0902 #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` self.provides = dict() + #: A dict of ``<package name>`` -> ``<list of recommended + #: symbols>``. This will not necessarily be populated. + self.recommends = dict() + #: The file (or directory) used for this source's cache data self.cachefile = os.path.join(self.basepath, "cache-%s" % self.cachekey) @@ -310,7 +314,7 @@ class Source(Debuggable): # pylint: disable=R0902 :raises: cPickle.UnpicklingError - If the saved data is corrupt """ data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, - self.essentialpkgs) = cPickle.load(data) + self.essentialpkgs, self.recommends) = cPickle.load(data) def save_state(self): """ Save state to :attr:`cachefile`. If caching and @@ -318,7 +322,7 @@ class Source(Debuggable): # pylint: disable=R0902 does not need to be implemented. """ cache = open(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides, - self.essentialpkgs), cache, 2) + self.essentialpkgs, self.recommends), cache, 2) cache.close() @track_statistics() @@ -513,13 +517,14 @@ class Source(Debuggable): # pylint: disable=R0902 as its final step.""" pass - def process_files(self, dependencies, provides): + def process_files(self, dependencies, # pylint: disable=R0912,W0102 + provides, recommends=dict()): """ Given dicts of depends and provides generated by :func:`read_files`, this generates :attr:`deps` and :attr:`provides` and calls :func:`save_state` to save the cached data to disk. - Both arguments are dicts of dicts of lists. Keys are the + All arguments are dicts of dicts of lists. Keys are the arches of packages contained in this source; values are dicts whose keys are package names and values are lists of either dependencies for each package the symbols provided by each @@ -531,14 +536,20 @@ class Source(Debuggable): # pylint: disable=R0902 :param provides: A dict of symbols provided by packages in this repository. :type provides: dict; see above. + :param recommends: A dict of recommended dependencies + found for this source. + :type recommends: dict; see above. """ self.deps['global'] = dict() + self.recommends['global'] = dict() self.provides['global'] = dict() for barch in dependencies: self.deps[barch] = dict() + self.recommends[barch] = dict() self.provides[barch] = dict() for pkgname in self.pkgnames: pset = set() + rset = set() for barch in dependencies: if pkgname not in dependencies[barch]: dependencies[barch][pkgname] = [] @@ -548,6 +559,18 @@ class Source(Debuggable): # pylint: disable=R0902 else: for barch in dependencies: self.deps[barch][pkgname] = dependencies[barch][pkgname] + + for barch in recommends: + if pkgname not in recommends[barch]: + recommends[barch][pkgname] = [] + rset.add(tuple(recommends[barch][pkgname])) + if len(rset) == 1: + self.recommends['global'][pkgname] = rset.pop() + else: + for barch in recommends: + self.recommends[barch][pkgname] = \ + recommends[barch][pkgname] + provided = set() for bprovided in list(provides.values()): provided.update(set(bprovided)) @@ -655,17 +678,24 @@ class Source(Debuggable): # pylint: disable=R0902 """ return ['global'] + [a for a in self.arches if a in metadata.groups] - def get_deps(self, metadata, package): + def get_deps(self, metadata, package, recommended=None): """ Get a list of the dependencies of the given package. :param package: The name of the symbol :type package: string :returns: list of strings """ + recs = [] + if ((recommended is None and self.recommended) or + (recommended and recommended.lower() == 'true')): + for arch in self.get_arches(metadata): + if package in self.recommends[arch]: + recs.extend(self.recommends[arch][package]) + for arch in self.get_arches(metadata): if package in self.deps[arch]: - return self.deps[arch][package] - return [] + recs.extend(self.deps[arch][package]) + return recs def get_provides(self, metadata, package): """ Get a list of all symbols provided by the given package. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index b98d3f419..f26ded4c5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -63,6 +63,7 @@ import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from lockfile import FileLock from Bcfg2.Utils import Executor +from distutils.spawn import find_executable # pylint: disable=E0611 # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -89,7 +90,9 @@ try: import yum try: import json - except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 + except (ImportError, AttributeError): import simplejson as json HAS_YUM = True except ImportError: @@ -340,25 +343,21 @@ class YumCollection(Collection): @property def helper(self): - """ The full path to :file:`bcfg2-yum-helper`. First, we - check in the config file to see if it has been explicitly - specified; next we see if it's in $PATH (which we do by making - a call to it; I wish there was a way to do this without - forking, but apparently not); finally we check in /usr/sbin, - the default location. """ + """The full path to :file:`bcfg2-yum-helper`. First, we check in the + config file to see if it has been explicitly specified; next + we see if it's in $PATH; finally we default to /usr/sbin, the + default location. """ + # pylint: disable=W0212 if not self._helper: - # pylint: disable=W0212 self.__class__._helper = Bcfg2.Options.setup.yum_helper if not self.__class__._helper: # first see if bcfg2-yum-helper is in PATH - try: - self.debug_log("Checking for bcfg2-yum-helper in $PATH") - self.cmd.run(['bcfg2-yum-helper']) - self.__class__._helper = 'bcfg2-yum-helper' - except OSError: + self.debug_log("Checking for bcfg2-yum-helper in $PATH") + self.__class__._helper = find_executable('bcfg2-yum-helper') + if not self.__class__._helper: self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" - # pylint: enable=W0212 - return self._helper + return self.__class__._helper + # pylint: enable=W0212 @property def use_yum(self): @@ -417,6 +416,25 @@ class YumCollection(Collection): yumconf.write(open(self.cfgfile, 'w')) + def get_arch(self): + """ If 'arch' for each source is the same, return that arch, otherwise + None. + + This helps bcfg2-yum-helper when the client arch is + incompatible with the bcfg2 server's arch. + + In case multiple arches are found, punt back to the default behavior. + """ + arches = set() + for source in self: + for url_map in source.url_map: + if url_map['arch'] in self.metadata.groups: + arches.add(url_map['arch']) + if len(arches) == 1: + return arches.pop() + else: + return None + def get_config(self, raw=False): # pylint: disable=W0221 """ Get the yum configuration for this collection. @@ -839,7 +857,7 @@ class YumCollection(Collection): return new @track_statistics() - def complete(self, packagelist): + def complete(self, packagelist, recommended=None): """ Build a complete list of all packages and their dependencies. When using the Python yum libraries, this defers to the @@ -857,7 +875,7 @@ class YumCollection(Collection): resolved. """ if not self.use_yum: - return Collection.complete(self, packagelist) + return Collection.complete(self, packagelist, recommended) lock = FileLock(os.path.join(self.cachefile, "lock")) slept = 0 @@ -872,10 +890,12 @@ class YumCollection(Collection): if packagelist: try: - result = self.call_helper( - "complete", - dict(packages=list(packagelist), - groups=list(self.get_relevant_groups()))) + helper_dict = dict(packages=list(packagelist), + groups=list(self.get_relevant_groups())) + arch = self.get_arch() + if arch is not None: + helper_dict['arch'] = arch + result = self.call_helper("complete", helper_dict) except ValueError: # error reported by call_helper() return set(), packagelist diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 49f64bdf3..d11ac60fe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -101,7 +101,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, cf=("packages", "backends"), dest="packages_backends", help="Packages backends to load", type=Bcfg2.Options.Types.comma_list, - action=PackagesBackendAction, default=['Yum', 'Apt', 'Pac']), + action=PackagesBackendAction, + default=['Yum', 'Apt', 'Pac', 'Pkgng']), Bcfg2.Options.PathOption( cf=("packages", "cache"), dest="packages_cache", help="Path to the Packages cache", @@ -319,8 +320,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, structures.append(indep) @track_statistics() - def _build_packages(self, metadata, independent, structures, - collection=None): + def _build_packages(self, metadata, independent, # pylint: disable=R0914 + structures, collection=None): """ Perform dependency resolution and build the complete list of packages that need to be included in the specification by :func:`validate_structures`, based on the initial list of @@ -357,10 +358,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, initial = set() to_remove = [] groups = [] + recommended = dict() + for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.update(collection.packages_from_entry(pkg)) + + if pkg.get("recommended"): + recommended[pkg.get("name")] = pkg.get("recommended") elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -399,7 +405,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", collection.cachekey) if pkey not in pcache: - pcache[pkey] = collection.complete(base) + pcache[pkey] = collection.complete(base, recommended) packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 9f2375fcd..21d50ace6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -10,7 +10,7 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Server.Cache import Bcfg2.Server.Plugin -from Bcfg2.Compat import unicode # pylint: disable=W0622 +from Bcfg2.Compat import unicode, any # pylint: disable=W0622 import Bcfg2.Server.FileMonitor from Bcfg2.Logger import Debuggable from Bcfg2.Server.Statistics import track_statistics @@ -51,8 +51,10 @@ def load_django_models(): try: import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True @@ -431,7 +433,13 @@ class Probes(Bcfg2.Server.Plugin.Probing, options = [ Bcfg2.Options.BooleanOption( cf=('probes', 'use_database'), dest="probes_db", - help="Use database capabilities of the Probes plugin")] + help="Use database capabilities of the Probes plugin"), + Bcfg2.Options.Option( + cf=('probes', 'allowed_groups'), dest="probes_allowed_groups", + help="Whitespace-separated list of group name regexps to which " + "probes can assign a client", + default=[re.compile('.*')], + type=Bcfg2.Options.Types.anchored_regex_list)] options_parsed_hook = staticmethod(load_django_models) def __init__(self, core): @@ -480,7 +488,13 @@ class Probes(Bcfg2.Server.Plugin.Probing, for line in dlines[:]: match = self.groupline_re.match(line) if match: - groups.append(match.group("groupname")) + newgroup = match.group("groupname") + if self._group_allowed(newgroup): + groups.append(newgroup) + else: + self.logger.warning( + "Disallowed group assignment %s from %s" % + (newgroup, client.hostname)) dlines.remove(line) return (groups, ProbeData("\n".join(dlines))) @@ -489,3 +503,10 @@ class Probes(Bcfg2.Server.Plugin.Probing, def get_additional_data(self, metadata): return self.probestore.get_data(metadata.hostname) + + def _group_allowed(self, group): + """ Determine if the named group can be set as a probe group + by checking the regexes listed in the [probes] groups_allowed + setting """ + return any(r.match(group) + for r in Bcfg2.Options.setup.probes_allowed_groups) diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 87cee7029..28400f6d2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -13,8 +13,10 @@ from Bcfg2.Server.Plugin import PluginExecutionError try: import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True @@ -161,7 +163,6 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): Bcfg2.Server.Plugin.StructFile.__init__(self, name, should_monitor=should_monitor) PropertyFile.__init__(self, name) - __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ def _write(self): open(self.name, "wb").write( @@ -169,7 +170,6 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): xml_declaration=False, pretty_print=True).decode('UTF-8')) return True - _write.__doc__ = PropertyFile._write.__doc__ def validate_data(self): """ ensure that the data in this object validates against the @@ -192,7 +192,6 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): self.name) else: return True - validate_data.__doc__ = PropertyFile.validate_data.__doc__ def get_additional_data(self, metadata): if Bcfg2.Options.setup.automatch: diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 8b8ada852..282de8247 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -54,7 +54,7 @@ class Reporting(Statistics, Threaded, PullSource): self.logger.error(msg) raise PluginInitError(msg) - def start_threads(self): + # This must be loaded here for bcfg2-admin try: self.transport = Bcfg2.Options.setup.reporting_transport() except TransportError: @@ -63,6 +63,10 @@ class Reporting(Statistics, Threaded, PullSource): if self.debug_flag: self.transport.set_debug(self.debug_flag) + def start_threads(self): + """Nothing to do here""" + pass + def set_debug(self, debug): rv = Statistics.set_debug(self, debug) if self.transport is not None: diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index b2a16e52e..b752650f0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -20,8 +20,8 @@ class Svn(Bcfg2.Server.Plugin.Version): Bcfg2.Options.Option( cf=("svn", "conflict_resolution"), dest="svn_conflict_resolution", type=lambda v: v.replace("-", "_"), - choices=dir(pysvn.wc_conflict_choice), - default=pysvn.wc_conflict_choice.postpone, + choices=dir(pysvn.wc_conflict_choice), # pylint: disable=E1101 + default=pysvn.wc_conflict_choice.postpone, # pylint: disable=E1101 help="SVN conflict resolution method"), Bcfg2.Options.Option( cf=("svn", "user"), dest="svn_user", help="SVN username"), diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 5e6846a44..6ad5b5635 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -72,7 +72,7 @@ class SSLServer(SocketServer.TCPServer, object): def __init__(self, listen_all, server_address, RequestHandlerClass, keyfile=None, certfile=None, reqCert=False, ca=None, - timeout=None, protocol='xmlrpc/ssl'): + timeout=None, protocol='xmlrpc/tlsv1'): """ :param listen_all: Listen on all interfaces :type listen_all: bool @@ -333,7 +333,7 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, """ Component XMLRPCServer. """ def __init__(self, listen_all, server_address, RequestHandlerClass=None, - keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl', + keyfile=None, certfile=None, ca=None, protocol='xmlrpc/tlsv1', timeout=10, logRequests=False, register=True, allow_none=True, encoding=None): """ diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 35d4cfa0a..196d77273 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.3" +__version__ = "1.4.0pre1" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index 274bd3659..00b4e92e3 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -16,6 +16,7 @@ class BackendAction(Bcfg2.Options.ComponentAction): class CLI(object): """ bcfg2-server CLI class """ + parse_first = True options = [ Bcfg2.Options.Option( cf=('server', 'backend'), help='Server Backend', diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py new file mode 100644 index 000000000..500016c9d --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8 -*- +import os +import sys +import copy +import lxml.etree +import tempfile +from mock import Mock, MagicMock, patch +try: + from Bcfg2.Client.Tools.POSIX.Augeas import * + HAS_AUGEAS = True +except ImportError: + POSIXAugeas = None + HAS_AUGEAS = False + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from TestPOSIX.Testbase import TestPOSIXTool +from common import * + + +test_data = """<Test> + <Empty/> + <Text>content with spaces</Text> + <Attrs foo="foo" bar="bar"/> + <Children identical="false"> + <Foo/> + <Bar attr="attr"/> + </Children> + <Children identical="true"> + <Thing>one</Thing> + <Thing>two</Thing> + </Children> + <Children multi="true"> + <Thing>same</Thing> + <Thing>same</Thing> + <Thing>same</Thing> + <Thing>same</Thing> + </Children> +</Test> +""" + +test_xdata = lxml.etree.XML(test_data) + +class TestPOSIXAugeas(TestPOSIXTool): + test_obj = POSIXAugeas + + applied_commands = dict( + insert=lxml.etree.Element( + "Insert", label="Thing", + path='Test/Children[#attribute/identical = "true"]/Thing'), + set=lxml.etree.Element("Set", path="Test/Text/#text", + value="content with spaces"), + move=lxml.etree.Element( + "Move", source="Test/Foo", + destination='Test/Children[#attribute/identical = "false"]/Foo'), + remove=lxml.etree.Element("Remove", path="Test/Bar"), + clear=lxml.etree.Element("Clear", path="Test/Empty/#text"), + setm=lxml.etree.Element( + "SetMulti", sub="#text", value="same", + base='Test/Children[#attribute/multi = "true"]/Thing')) + + @skipUnless(HAS_AUGEAS, "Python Augeas libraries not found") + def setUp(self): + TestPOSIXTool.setUp(self) + fd, self.tmpfile = tempfile.mkstemp() + os.fdopen(fd, 'w').write(test_data) + + def tearDown(self): + tmpfile = getattr(self, "tmpfile", None) + if tmpfile and os.path.exists(tmpfile): + os.unlink(tmpfile) + + def test_fully_specified(self): + ptool = self.get_obj() + + entry = lxml.etree.Element("Path", name="/test", type="augeas") + self.assertFalse(ptool.fully_specified(entry)) + + lxml.etree.SubElement(entry, "Set", path="/test", value="test") + self.assertTrue(ptool.fully_specified(entry)) + + def test_install(self): + # this is tested adequately by the other tests + pass + + def test_verify(self): + # this is tested adequately by the other tests + pass + + @patch("Bcfg2.Client.Tools.POSIX.Augeas.POSIXTool.verify") + def _verify(self, commands, mock_verify): + ptool = self.get_obj() + mock_verify.return_value = True + + entry = lxml.etree.Element("Path", name=self.tmpfile, + type="augeas", lens="Xml") + entry.extend(commands) + + modlist = [] + self.assertTrue(ptool.verify(entry, modlist)) + mock_verify.assert_called_with(ptool, entry, modlist) + self.assertXMLEqual(lxml.etree.parse(self.tmpfile).getroot(), + test_xdata) + + def test_verify_insert(self): + """ Test successfully verifying an Insert command """ + self._verify([self.applied_commands['insert']]) + + def test_verify_set(self): + """ Test successfully verifying a Set command """ + self._verify([self.applied_commands['set']]) + + def test_verify_move(self): + """ Test successfully verifying a Move command """ + self._verify([self.applied_commands['move']]) + + def test_verify_remove(self): + """ Test successfully verifying a Remove command """ + self._verify([self.applied_commands['remove']]) + + def test_verify_clear(self): + """ Test successfully verifying a Clear command """ + self._verify([self.applied_commands['clear']]) + + def test_verify_set_multi(self): + """ Test successfully verifying a SetMulti command """ + self._verify([self.applied_commands['setm']]) + + def test_verify_all(self): + """ Test successfully verifying multiple commands """ + self._verify(self.applied_commands.values()) + + @patch("Bcfg2.Client.Tools.POSIX.Augeas.POSIXTool.install") + def _install(self, commands, expected, mock_install, **attrs): + ptool = self.get_obj() + mock_install.return_value = True + + entry = lxml.etree.Element("Path", name=self.tmpfile, + type="augeas", lens="Xml") + for key, val in attrs.items(): + entry.set(key, val) + entry.extend(commands) + + self.assertTrue(ptool.install(entry)) + mock_install.assert_called_with(ptool, entry) + self.assertXMLEqual(lxml.etree.parse(self.tmpfile).getroot(), + expected) + + def test_install_set_existing(self): + """ Test setting the value of an existing node """ + expected = copy.deepcopy(test_xdata) + expected.find("Text").text = "Changed content" + self._install([lxml.etree.Element("Set", path="Test/Text/#text", + value="Changed content")], + expected) + + def test_install_set_new(self): + """ Test setting the value of an new node """ + expected = copy.deepcopy(test_xdata) + newtext = lxml.etree.SubElement(expected, "NewText") + newtext.text = "new content" + self._install([lxml.etree.Element("Set", path="Test/NewText/#text", + value="new content")], + expected) + + def test_install_remove(self): + """ Test removing a node """ + expected = copy.deepcopy(test_xdata) + expected.remove(expected.find("Attrs")) + self._install( + [lxml.etree.Element("Remove", + path='Test/*[#attribute/foo = "foo"]')], + expected) + + def test_install_move(self): + """ Test moving a node """ + expected = copy.deepcopy(test_xdata) + foo = expected.xpath("//Foo")[0] + expected.append(foo) + self._install( + [lxml.etree.Element("Move", source='Test/Children/Foo', + destination='Test/Foo')], + expected) + + def test_install_clear(self): + """ Test clearing a node """ + # TODO: clearing a node doesn't seem to work with the XML lens + # + # % augtool -b + # augtool> set /augeas/load/Xml/incl[3] "/tmp/test.xml" + # augtool> load + # augtool> clear '/files/tmp/test.xml/Test/Text/#text' + # augtool> save + # error: Failed to execute command + # saving failed (run 'print /augeas//error' for details) + # augtool> print /augeas//error + # + # The error isn't useful. + pass + + def test_install_set_multi(self): + """ Test setting multiple nodes at once """ + expected = copy.deepcopy(test_xdata) + for thing in expected.xpath("Children[@identical='true']/Thing"): + thing.text = "same" + self._install( + [lxml.etree.Element( + "SetMulti", value="same", + base='Test/Children[#attribute/identical = "true"]', + sub="Thing/#text")], + expected) + + def test_install_insert(self): + """ Test inserting a node """ + expected = copy.deepcopy(test_xdata) + children = expected.xpath("Children[@identical='true']")[0] + thing = lxml.etree.Element("Thing") + thing.text = "three" + children.append(thing) + self._install( + [lxml.etree.Element( + "Insert", + path='Test/Children[#attribute/identical = "true"]/Thing[2]', + label="Thing", where="after"), + lxml.etree.Element( + "Set", + path='Test/Children[#attribute/identical = "true"]/Thing[3]/#text', + value="three")], + expected) + + def test_install_initial(self): + """ Test creating initial content and then modifying it """ + os.unlink(self.tmpfile) + expected = copy.deepcopy(test_xdata) + expected.find("Text").text = "Changed content" + initial = lxml.etree.Element("Initial") + initial.text = test_data + modify = lxml.etree.Element("Set", path="Test/Text/#text", + value="Changed content") + self._install([initial, modify], expected, current_exists="false") diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py index 31e297888..69dd562be 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py @@ -270,7 +270,6 @@ class TestPOSIXFile(TestPOSIXTool): mock_open.assert_called_with(entry.get("name")) mock_open.return_value.read.assert_any_call() ptool._diff.assert_called_with(ondisk, entry.text, - difflib.unified_diff, filename=entry.get("name")) self.assertIsNotNone(entry.get("qtext")) del entry.attrib['qtext'] @@ -280,8 +279,8 @@ class TestPOSIXFile(TestPOSIXTool): entry = reset() ptool._get_diffs(entry, content=ondisk) self.assertFalse(mock_open.called) - ptool._diff.assert_called_with(ondisk, entry.text, difflib.ndiff, - filename=entry.get("name")) + ptool._diff.assert_called_with(ondisk, entry.text, + filename=entry.get("name")) self.assertIsNone(entry.get("qtext")) self.assertEqual(entry.get("current_bdiff"), b64encode("\n".join(ptool._diff.return_value))) @@ -296,9 +295,7 @@ class TestPOSIXFile(TestPOSIXTool): mock_open.assert_called_with(entry.get("name")) mock_open.return_value.read.assert_any_call() self.assertItemsEqual(ptool._diff.call_args_list, - [call(ondisk, entry.text, difflib.unified_diff, - filename=entry.get("name")), - call(ondisk, entry.text, difflib.ndiff, + [call(ondisk, entry.text, filename=entry.get("name"))]) self.assertIsNotNone(entry.get("qtext")) self.assertTrue(entry.get("qtext").startswith("test\n")) @@ -318,9 +315,7 @@ class TestPOSIXFile(TestPOSIXTool): mock_open.assert_called_with(entry.get("name")) mock_open.return_value.read.assert_any_call() self.assertItemsEqual(ptool._diff.call_args_list, - [call(ondisk, encoded, difflib.unified_diff, - filename=entry.get("name")), - call(ondisk, encoded, difflib.ndiff, + [call(ondisk, encoded, filename=entry.get("name"))]) self.assertIsNotNone(entry.get("qtext")) self.assertEqual(entry.get("current_bdiff"), @@ -415,35 +410,23 @@ class TestPOSIXFile(TestPOSIXTool): ptool._rename_tmpfile.assert_called_with(newfile, entry) mock_install.assert_called_with(ptool, entry) - @patch("time.time") - def test_diff(self, mock_time): + @patch("difflib.unified_diff") + def test_diff(self, mock_diff): ptool = self.get_obj() + filename = "/test" content1 = "line1\nline2" content2 = "line3" - self.now = 1345640723 - - def time_rv(): - self.now += 1 - return self.now - mock_time.side_effect = time_rv - rv = ["line1", "line2", "line3"] - func = Mock() - func.return_value = rv - self.assertItemsEqual(ptool._diff(content1, content2, func), rv) - func.assert_called_with(["line1", "line2"], ["line3"]) - - func.reset_mock() - mock_time.reset_mock() - def time_rv(): - self.now += 5 - return self.now - mock_time.side_effect = time_rv - - def slow_diff(content1, content2): - for i in range(1, 10): - yield "line%s" % i - func.side_effect = slow_diff - self.assertFalse(ptool._diff(content1, content2, func), rv) - func.assert_called_with(["line1", "line2"], ["line3"]) + mock_diff.return_value = rv + self.assertItemsEqual(ptool._diff(content1, content2), rv) + mock_diff.assert_called_with(["line1", "line2"], ["line3"], + fromfile='', tofile='') + + mock_diff.reset_mock() + self.assertItemsEqual(ptool._diff(content1, content2, + filename=filename), + rv) + mock_diff.assert_called_with(["line1", "line2"], ["line3"], + fromfile='/test (on disk)', + tofile='/test (from bcfg2)') diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py index 5a752b2ac..ea4ca3f5f 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py @@ -586,16 +586,16 @@ class TestPOSIXTool(TestTool): self.assertEqual(6, ptool._norm_acl_perms("rwa")) self.assertEqual(4, ptool._norm_acl_perms("rr")) - @patch('os.stat') - def test__gather_data(self, mock_stat): + @patch('os.lstat') + def test__gather_data(self, mock_lstat): ptool = self.get_obj() path = '/test' - mock_stat.side_effect = OSError + mock_lstat.side_effect = OSError self.assertFalse(ptool._gather_data(path)[0]) - mock_stat.assert_called_with(path) + mock_lstat.assert_called_with(path) - mock_stat.reset_mock() - mock_stat.side_effect = None + mock_lstat.reset_mock() + mock_lstat.side_effect = None # create a return value stat_rv = MagicMock() def stat_getitem(key): @@ -608,7 +608,7 @@ class TestPOSIXTool(TestTool): # and ensure that they're stripped return int('060660', 8) stat_rv.__getitem__ = Mock(side_effect=stat_getitem) - mock_stat.return_value = stat_rv + mock_lstat.return_value = stat_rv # disable selinux and acls for this call -- we test them # separately so that we can skip those tests as appropriate @@ -620,7 +620,7 @@ class TestPOSIXTool(TestTool): (stat_rv, '0', '10', '0660', None, None)) Bcfg2.Client.Tools.POSIX.base.HAS_SELINUX, \ Bcfg2.Client.Tools.POSIX.base.HAS_ACLS = states - mock_stat.assert_called_with(path) + mock_lstat.assert_called_with(path) @skipUnless(HAS_SELINUX, "SELinux not found, skipping") def test__gather_data_selinux(self): @@ -628,12 +628,12 @@ class TestPOSIXTool(TestTool): context = 'system_u:object_r:root_t:s0' path = '/test' - @patch('os.stat') - @patchIf(HAS_SELINUX, "selinux.getfilecon") - def inner(mock_getfilecon, mock_stat): - mock_getfilecon.return_value = [len(context) + 1, context] - mock_stat.return_value = MagicMock() - mock_stat.return_value.__getitem__.return_value = MagicMock() + @patch('os.lstat') + @patchIf(HAS_SELINUX, "selinux.lgetfilecon") + def inner(mock_lgetfilecon, mock_lstat): + mock_lgetfilecon.return_value = [len(context) + 1, context] + mock_lstat.return_value = MagicMock() + mock_lstat.return_value.__getitem__.return_value = MagicMock() # disable acls for this call and test them separately state = (Bcfg2.Client.Tools.POSIX.base.HAS_ACLS, Bcfg2.Client.Tools.POSIX.base.HAS_SELINUX) @@ -642,30 +642,40 @@ class TestPOSIXTool(TestTool): self.assertEqual(ptool._gather_data(path)[4], 'root_t') Bcfg2.Client.Tools.POSIX.base.HAS_ACLS, \ Bcfg2.Client.Tools.POSIX.base.HAS_SELINUX = state - mock_getfilecon.assert_called_with(path) + mock_lgetfilecon.assert_called_with(path) inner() @skipUnless(HAS_ACLS, "ACLS not found, skipping") - @patch('os.stat') - def test__gather_data_acls(self, mock_stat): + @patch('os.lstat') + @patch('stat.S_ISLNK') + def test__gather_data_acls(self, mock_S_ISLNK, mock_lstat): ptool = self.get_obj() ptool._list_file_acls = Mock() acls = {("default", posix1e.ACL_USER, "testuser"): "rwx", ("access", posix1e.ACL_GROUP, "testgroup"): "rx"} ptool._list_file_acls.return_value = acls path = '/test' - mock_stat.return_value = MagicMock() - mock_stat.return_value.__getitem__.return_value = MagicMock() + mock_lstat.return_value = MagicMock() + mock_lstat.return_value.__getitem__.return_value = MagicMock() + mock_S_ISLNK.return_value = False # disable selinux for this call and test it separately state = (Bcfg2.Client.Tools.POSIX.base.HAS_ACLS, Bcfg2.Client.Tools.POSIX.base.HAS_SELINUX) Bcfg2.Client.Tools.POSIX.base.HAS_ACLS = True Bcfg2.Client.Tools.POSIX.base.HAS_SELINUX = False self.assertItemsEqual(ptool._gather_data(path)[5], acls) + ptool._list_file_acls.assert_called_with(path) + + # symlinks can't have their own ACLs, so ensure that the + # _list_file_acls call is skipped and no ACLs are returned + mock_S_ISLNK.return_value = True + ptool._list_file_acls.reset_mock() + self.assertEqual(ptool._gather_data(path)[5], None) + self.assertFalse(ptool._list_file_acls.called) + Bcfg2.Client.Tools.POSIX.base.HAS_ACLS, \ Bcfg2.Client.Tools.POSIX.base.HAS_SELINUX = state - ptool._list_file_acls.assert_called_with(path) @patchIf(HAS_SELINUX, "selinux.matchpathcon") def test_verify_metadata(self, mock_matchpathcon): diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py index 9647413b6..cc1ea6fd7 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py @@ -26,10 +26,10 @@ class TestPOSIXUsers(TestTool): def setUp(self): TestTool.setUp(self) - set_setup_default('uid_whitelist') - set_setup_default('uid_blacklist') - set_setup_default('gid_whitelist') - set_setup_default('gid_blacklist') + set_setup_default('uid_whitelist', []) + set_setup_default('uid_blacklist', []) + set_setup_default('gid_whitelist', []) + set_setup_default('gid_blacklist', []) set_setup_default('encoding', 'UTF-8') def get_obj(self, config=None): diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py index 1bf073f81..0c059b5f3 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py @@ -41,10 +41,12 @@ class TestTool(Bcfg2TestCase): @patch("%s.%s._analyze_config" % (self.test_obj.__module__, self.test_obj.__name__)) def inner(mock_analyze_config, mock_check_execs): - t = self.get_obj() + self.get_obj() mock_analyze_config.assert_called_with() mock_check_execs.assert_called_with() + inner() + def test__analyze_config(self): t = self.get_obj() t.getSupportedEntries = Mock() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index 81c4837e1..37beaa26c 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -8,6 +8,7 @@ import genshi.core from Bcfg2.Compat import reduce from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugin.helpers import * +from Bcfg2.Server.Plugin.exceptions import PluginInitError # add all parent testsuite directories to sys.path to allow (most) # relative imports in python 2.4 @@ -34,6 +35,7 @@ def tostring(el): class FakeElementTree(lxml.etree._ElementTree): xinclude = Mock() + parse = Mock class TestFunctions(Bcfg2TestCase): @@ -71,7 +73,7 @@ class TestDatabaseBacked(TestPlugin): self.assertFalse(db._use_db) setattr(Bcfg2.Options.setup, attr, True) - self.assertFalse(db._use_db) + self.assertRaises(PluginInitError, self.get_obj, core) class TestPluginDatabaseModel(Bcfg2TestCase): @@ -771,19 +773,15 @@ class TestStructFile(TestXMLFileBacked): @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping") @patchIf(HAS_CRYPTO, "Bcfg2.Server.Encryption.ssl_decrypt") - @patchIf(HAS_CRYPTO, "Bcfg2.Server.Encryption.bruteforce_decrypt") - def test_decrypt(self, mock_bruteforce, mock_ssl): + def test_decrypt(self, mock_ssl): sf = self.get_obj() def reset(): - mock_bruteforce.reset_mock() mock_ssl.reset_mock() - # test element without text contents Bcfg2.Options.setup.passphrases = dict() self.assertIsNone(sf._decrypt(lxml.etree.Element("Test"))) - self.assertFalse(mock_bruteforce.called) self.assertFalse(mock_ssl.called) # test element with a passphrase in the config file @@ -794,29 +792,17 @@ class TestStructFile(TestXMLFileBacked): mock_ssl.return_value = "decrypted with ssl" self.assertEqual(sf._decrypt(el), mock_ssl.return_value) mock_ssl.assert_called_with(el.text, "foopass") - self.assertFalse(mock_bruteforce.called) - - # test failure to decrypt element with a passphrase in the config - reset() - mock_ssl.side_effect = EVPError - self.assertRaises(EVPError, sf._decrypt, el) - mock_ssl.assert_called_with(el.text, "foopass") - self.assertFalse(mock_bruteforce.called) # test element without valid passphrase reset() el.set("encrypted", "true") - mock_bruteforce.return_value = "decrypted with bruteforce" - self.assertEqual(sf._decrypt(el), mock_bruteforce.return_value) - mock_bruteforce.assert_called_with(el.text) + self.assertRaises(EVPError, sf._decrypt, el) self.assertFalse(mock_ssl.called) - # test failure to decrypt element without valid passphrase + # test failure to decrypt element with a passphrase in the config reset() - mock_bruteforce.side_effect = EVPError + mock_ssl.side_effect = EVPError self.assertRaises(EVPError, sf._decrypt, el) - mock_bruteforce.assert_called_with(el.text) - self.assertFalse(mock_ssl.called) def test_include_element(self): sf = self.get_obj() @@ -1135,6 +1121,25 @@ class TestInfoXML(TestStructFile): self.assertTrue(inc("Path", name="/etc/bar.conf", negate="true")) self.assertTrue(inc("Path", name="/etc/bar.conf", negate="tRUe")) + def test_include_element_altsrc(self): + ix = self.get_obj() + metadata = Mock() + entry = lxml.etree.Element("Path", name="/etc/bar.conf", + realname="/etc/foo.conf") + inc = lambda tag, **attrs: \ + ix._include_element(lxml.etree.Element(tag, **attrs), + metadata, entry) + + self.assertFalse(inc("Path", name="/etc/bar.conf")) + self.assertFalse(inc("Path", name="/etc/foo.conf", negate="true")) + self.assertFalse(inc("Path", name="/etc/foo.conf", negate="tRuE")) + self.assertTrue(inc("Path", name="/etc/foo.conf")) + self.assertTrue(inc("Path", name="/etc/foo.conf", negate="false")) + self.assertTrue(inc("Path", name="/etc/foo.conf", negate="faLSe")) + self.assertTrue(inc("Path", name="/etc/bar.conf", negate="true")) + self.assertTrue(inc("Path", name="/etc/bar.conf", negate="tRUe")) + + def test_BindEntry(self): ix = self.get_obj() entry = lxml.etree.Element("Path", name=self.path) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py index 873ebd837..63403208b 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py @@ -1,6 +1,7 @@ import os import sys import lxml.etree +import Bcfg2.Server.Plugins.Cfg from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator import * from Bcfg2.Server.Plugin import PluginExecutionError @@ -21,7 +22,7 @@ from TestServer.TestPlugins.TestCfg.Test_init import TestCfgGenerator class TestCfgEncryptedGenerator(TestCfgGenerator): test_obj = CfgEncryptedGenerator - @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") + @skipUnless(HAS_CRYPTO, "M2Crypto is not available") def setUp(self): TestCfgGenerator.setUp(self) @@ -29,6 +30,7 @@ class TestCfgEncryptedGenerator(TestCfgGenerator): "Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator.bruteforce_decrypt") def test_handle_event(self, mock_decrypt): @patch("Bcfg2.Server.Plugins.Cfg.CfgGenerator.handle_event") + @patch("Bcfg2.Options.setup.lax_decryption", False) def inner(mock_handle_event): def reset(): mock_decrypt.reset_mock() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedJinja2Generator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedJinja2Generator.py new file mode 100644 index 000000000..6857f933b --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedJinja2Generator.py @@ -0,0 +1,46 @@ +import os +import sys +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator import * + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * + +try: + from TestServer.TestPlugins.TestCfg.TestCfgJinja2Generator import \ + TestCfgJinja2Generator + from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import HAS_JINJA2 +except ImportError: + TestCfgJinja2Generator = object + HAS_JINJA2 = False + +try: + from TestServer.TestPlugins.TestCfg.TestCfgEncryptedGenerator import \ + TestCfgEncryptedGenerator + from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator import HAS_CRYPTO +except ImportError: + TestCfgEncryptedGenerator = object + HAS_CRYPTO = False + + +class TestCfgEncryptedJinja2Generator(TestCfgJinja2Generator, + TestCfgEncryptedGenerator): + test_obj = CfgEncryptedJinja2Generator + + @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") + @skipUnless(HAS_JINJA2, "Jinja2 libraries not found, skipping") + def setUp(self): + pass + + def test_handle_event(self): + TestCfgEncryptedGenerator.test_handle_event(self) + + def test_get_data(self): + TestCfgJinja2Generator.test_get_data(self) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgJinja2Generator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgJinja2Generator.py new file mode 100644 index 000000000..036380d56 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgJinja2Generator.py @@ -0,0 +1,47 @@ +import os +import sys +import lxml.etree +from mock import Mock, MagicMock, patch +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import * + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * +from TestServer.TestPlugins.TestCfg.Test_init import TestCfgGenerator + + +class TestCfgJinja2Generator(TestCfgGenerator): + test_obj = CfgJinja2Generator + + @skipUnless(HAS_JINJA2, "Jinja2 libraries not found, skipping") + def setUp(self): + TestCfgGenerator.setUp(self) + set_setup_default("repository", datastore) + + @patch("Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.Template") + @patch("Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.get_template_data") + def test_get_data(self, mock_get_template_data, mock_Template): + ccg = self.get_obj() + ccg.data = "data" + entry = lxml.etree.Element("Path", name="/test.txt") + metadata = Mock() + + template_vars = dict(name=entry.get("name"), + metadata=metadata, + path=ccg.name, + source_path=ccg.name, + repo=datastore) + mock_get_template_data.return_value = template_vars + + self.assertEqual(ccg.get_data(entry, metadata), + mock_Template.return_value.render.return_value) + mock_Template.assert_called_with("data".decode(Bcfg2.Options.setup.encoding)) + tmpl = mock_Template.return_value + tmpl.render.assert_called_with(template_vars) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDecisions.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDecisions.py index 537ceb4ff..8b4df8abb 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDecisions.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDecisions.py @@ -52,9 +52,9 @@ class TestDecisions(TestPlugin, TestDecision): metadata = Mock() self.assertEqual(d.GetDecisions(metadata, "whitelist"), - d.whitelist.get_decision.return_value) - d.whitelist.get_decision.assert_called_with(metadata) + d.whitelist.get_decisions.return_value) + d.whitelist.get_decisions.assert_called_with(metadata) self.assertEqual(d.GetDecisions(metadata, "blacklist"), - d.blacklist.get_decision.return_value) - d.blacklist.get_decision.assert_called_with(metadata) + d.blacklist.get_decisions.return_value) + d.blacklist.get_decisions.assert_called_with(metadata) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index 68313e6fb..32766b5c1 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -1,4 +1,5 @@ import os +import re import sys import shutil import tempfile @@ -229,10 +230,12 @@ group: group:with:colons def setUp(self): Bcfg2TestCase.setUp(self) set_setup_default("probes_db") + set_setup_default("probes_allowed_groups", [re.compile(".*")]) self.datastore = None Bcfg2.Server.Cache.expire("Probes") def tearDown(self): + Bcfg2.Server.Cache.expire("Probes") if self.datastore is not None: shutil.rmtree(self.datastore) self.datastore = None @@ -275,8 +278,34 @@ group: group:with:colons def test_probes_db(self): """ Set and retrieve probe data with database enabled """ Bcfg2.Options.setup.probes_db = True + syncdb(TestProbesDB) self._perform_tests() + def test_allowed_cgroups(self): + """ Test option to only allow probes to set certain groups """ + probes = self.get_obj() + + test_text = """a couple lines +of freeform text +""" + test_groups = ["group", "group2", "group-with-dashes"] + test_probe_data = lxml.etree.Element("Probe", name="test") + test_probe_data.text = test_text + for group in test_groups: + test_probe_data.text += "group:%s\n" % group + + client = Mock() + groups, data = probes.ReceiveDataItem(client, test_probe_data) + self.assertItemsEqual(groups, test_groups) + self.assertEqual(data, test_text) + + old_allowed_groups = Bcfg2.Options.setup.probes_allowed_groups + Bcfg2.Options.setup.probes_allowed_groups = [re.compile(r'^group.?$')] + groups, data = probes.ReceiveDataItem(client, test_probe_data) + self.assertItemsEqual(groups, ['group', 'group2']) + self.assertEqual(data, test_text) + Bcfg2.Options.setup.probes_allowed_groups = old_allowed_groups + def _perform_tests(self): p = self.get_obj() diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 17fac4fe4..77b170809 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -38,6 +38,7 @@ contingent_checks = { ("yum",): {"lib/Bcfg2/Client/Tools": ["YUM.py"]}, ("genshi",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgGenshiGenerator.py"]}, ("Cheetah",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgCheetahGenerator.py"]}, + ("jinja2",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgJinja2Generator.py"]}, ("M2Crypto",): {"lib/Bcfg2": ["Encryption.py"], "lib/Bcfg2/Server/Plugins/Cfg": ["CfgEncryptedGenerator.py"]}, @@ -45,6 +46,8 @@ contingent_checks = { ["CfgEncryptedGenshiGenerator.py"]}, ("M2Crypto", "Cheetah"): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgEncryptedCheetahGenerator.py"]}, + ("M2Crypto", "jinja2"): {"lib/Bcfg2/Server/Plugins/Cfg": + ["CfgEncryptedJinja2Generator.py"]}, } # perform only error checking on the listed files @@ -327,7 +330,7 @@ class TestPylint(CodeTestCase): class TestPEP8(CodeTestCase): __test__ = True - command = ["pep8", "--ignore=E125,E501"] + command = ["pep8", "--ignore=E125,E129,E501"] def _test_errors(self, files, extra_args=None): pass diff --git a/testsuite/before_install.sh b/testsuite/before_install.sh deleted file mode 100755 index 5f1a59aaf..000000000 --- a/testsuite/before_install.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# before_install script for Travis-CI - -PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') - -sudo apt-get update -qq -sudo apt-get install -qq swig libxml2-utils -if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then - if [[ ${PYVER:0:1} == "2" ]]; then - sudo apt-get install -qq python-selinux python-pylibacl yum - fi -fi diff --git a/testsuite/common.py b/testsuite/common.py index 849c22ef3..5a08f8db5 100644 --- a/testsuite/common.py +++ b/testsuite/common.py @@ -129,8 +129,12 @@ class Bcfg2TestCase(TestCase): "\nSecond: %s" % lxml.etree.tostring(el2) self.assertEqual(el1.tag, el2.tag, msg=fullmsg % "Tags differ") - self.assertEqual(el1.text, el2.text, - msg=fullmsg % "Text content differs") + if el1.text is not None and el2.text is not None: + self.assertEqual(el1.text.strip(), el2.text.strip(), + msg=fullmsg % "Text content differs") + else: + self.assertEqual(el1.text, el2.text, + msg=fullmsg % "Text content differs") self.assertItemsEqual(el1.attrib.items(), el2.attrib.items(), msg=fullmsg % "Attributes differ") self.assertEqual(len(el1.getchildren()), diff --git a/testsuite/ext/exception_messages.py b/testsuite/ext/exception_messages.py index 877ba42a1..cd3d7112c 100644 --- a/testsuite/ext/exception_messages.py +++ b/testsuite/ext/exception_messages.py @@ -1,16 +1,30 @@ -from logilab import astng -from pylint.interfaces import IASTNGChecker +try: + from logilab import astng as ast + from pylint.interfaces import IASTNGChecker as IChecker + PYLINT = 0 # pylint 0.something +except ImportError: + import astroid as ast + from pylint.interfaces import IAstroidChecker as IChecker + PYLINT = 1 # pylint 1.something from pylint.checkers import BaseChecker from pylint.checkers.utils import safe_infer +if PYLINT == 0: + # this is not quite correct; later versions of pylint 0.* wanted a + # three-tuple for messages as well + msg = ('Exception raised without arguments', + 'Used when an exception is raised without any arguments') +else: + msg = ('Exception raised without arguments', + 'exception-without-args', + 'Used when an exception is raised without any arguments') +msgs = {'R9901': msg} + class ExceptionMessageChecker(BaseChecker): - __implements__ = IASTNGChecker + __implements__ = IChecker name = 'Exception Messages' - msgs = \ - {'R9901': ('Exception raised without arguments', - 'Used when an exception is raised without any arguments')} options = ( ('exceptions-without-args', dict(default=('NotImplementedError',), @@ -23,9 +37,9 @@ class ExceptionMessageChecker(BaseChecker): def visit_raise(self, node): if node.exc is None: return - if isinstance(node.exc, astng.Name): + if isinstance(node.exc, ast.Name): raised = safe_infer(node.exc) - if (isinstance(raised, astng.Class) and + if (isinstance(raised, ast.Class) and raised.name not in self.config.exceptions_without_args): self.add_message('R9901', node=node.exc) diff --git a/testsuite/install.sh b/testsuite/install.sh index 8721c8015..bbbd9ae76 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -1,8 +1,11 @@ -#!/bin/bash +#!/bin/bash -ex # install script for Travis-CI -pip install -r testsuite/requirements.txt --use-mirrors +sudo apt-get update -qq +sudo apt-get install swig libxml2-utils + +pip install -r testsuite/requirements.txt PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') @@ -11,22 +14,15 @@ if [[ ${PYVER:0:1} == "2" && $PYVER != "2.7" ]]; then fi if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then - pip install --use-mirrors PyYAML pyinotify boto - if [[ $PYVER == "2.5" ]]; then - # markdown 2.2+ doesn't work on py2.5 - pip install --use-mirrors simplejson 'markdown<2.2' 'django<1.4.9' - else - pip install 'django<1.5' - fi + sudo apt-get install -y yum libaugeas0 augeas-lenses libacl1-dev libssl-dev + + pip install --use-mirrors PyYAML pyinotify boto pylibacl 'django<1.5' \ + Jinja2 + easy_install https://fedorahosted.org/released/python-augeas/python-augeas-0.4.1.tar.gz if [[ ${PYVER:0:1} == "2" ]]; then # django supports py3k, but South doesn't, and the django bits # in bcfg2 require South - pip install cheetah 'South<0.8' M2Crypto - fi -else - # python < 2.6 requires M2Crypto for SSL communication, not just - # for encryption support - if [[ $PYVER == "2.5" || $PYVER == "2.4" ]]; then - pip install --use-mirrors M2crypto + pip install cheetah 'South<0.8' + pip install m2crypto fi fi diff --git a/testsuite/pylintrc.conf b/testsuite/pylintrc.conf index e13a51d0d..1d3ba8c88 100644 --- a/testsuite/pylintrc.conf +++ b/testsuite/pylintrc.conf @@ -99,6 +99,10 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme # evaluation report (RP0004). comment=no +# Template used to display messages. This is a python new-style format string +# used to format the massage information. See doc for all details +msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} + [VARIABLES] @@ -131,6 +135,9 @@ ignore-docstrings=yes # Maximum number of characters on a single line. max-line-length=79 +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )?(<?https?://\S+>?|:(func|class):.*)$ + # Maximum number of lines in a module max-module-lines=1000 @@ -247,8 +254,10 @@ max-locals=20 # Maximum number of return / yield for function / method body max-returns=6 -# Maximum number of branch for function / method body +# Maximum number of branch for function / method body (max-branchs is +# pylint 0.x, max-branches is 1.0) max-branchs=18 +max-branches=18 # Maximum number of statements in function / method body max-statements=75 diff --git a/tools/bcfg2-cron b/tools/bcfg2-cron index fe0e6e90d..9a93c2e44 100755 --- a/tools/bcfg2-cron +++ b/tools/bcfg2-cron @@ -1,6 +1,6 @@ #!/bin/sh # -# Script to run bcfg2 with cron. +# Script to run bcfg2 with cron. # # This script is designed so that bcfg2-cron can be invoked from both # /etc/cron.daily and /etc/cron.hourly. This allows the administrators to diff --git a/tools/export.py b/tools/export.py index bdb85de41..df3c810d5 100755 --- a/tools/export.py +++ b/tools/export.py @@ -163,43 +163,6 @@ E.G. 1.2.0pre1 is a valid version. print(help_message) quit() - if version_info['build'] == '': - rpmchangelog = ["* %s %s <%s> %s-1\n" % - (datetime.datetime.now().strftime("%a %b %d %Y"), - name, email, version_release), - "- New upstream release\n", "\n"] - else: - rpmchangelog = ["* %s %s <%s> %s-0.%s.%s\n" % - (datetime.datetime.now().strftime("%a %b %d %Y"), - name, email, version_release, - version_info['build'][-1], version_info['build']), - "- New upstream release\n", "\n"] - - # write out the new RPM changelog - specs = ["misc/bcfg2.spec", - "misc/bcfg2-selinux.spec"] - if options.dryrun: - print("*** Add the following to the top of the %%changelog section in %s:\n%s\n" - % (rpmchangelog, " and ".join(specs))) - else: - for fname in specs: - try: - lines = open(fname).readlines() - for lineno in range(len(lines)): - if lines[lineno].startswith("%changelog"): - break - else: - print("No %changelog section found in %s" % fname) - continue - for line in reversed(rpmchangelog): - lines.insert(lineno + 1, line) - open(fname, 'w').write("".join(lines)) - except: - err = sys.exc_info()[1] - print("Could not write %s: %s" % (fname, err)) - print(help_message) - quit() - # update solaris version find_and_replace('solaris/Makefile', 'VERS=', 'VERS=%s-1\n' % version, @@ -294,25 +257,23 @@ E.G. 1.2.0pre1 is a valid version. startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2.spec', 'BuildRoot', - 'BuildRoot: %%{_tmppath}/%%{name}-%%{version}%s-%%{release}-root-%%(%%{__id_u} -n)\n' % - version_info['build'], + 'BuildRoot: %{_tmppath}/%{name}-%{version}%{?_pre_rc}-%{release}-root-%(%{__id_u} -n)\n', startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2-selinux.spec', 'BuildRoot', - 'BuildRoot: %%{_tmppath}/%%{name}-%%{version}%s-%%{release}-root-%%(%%{__id_u} -n)\n' % - version_info['build'], + 'BuildRoot: %{_tmppath}/%{name}-%{version}%{?_pre_rc}-%{release}-root-%(%{__id_u} -n)\n', startswith=True, dryrun=options.dryrun) # fix pre problem noted in # http://trac.mcs.anl.gov/projects/bcfg2/ticket/1129 find_and_replace('misc/bcfg2.spec', 'Source0', - 'Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%%{name}-%%{version}%s.tar.gz\n' % version_info["build"], + 'Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}%{?_pre_rc}.tar.gz\n', startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2-selinux.spec', 'Source0', - 'Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%%{name}-%%{version}%s.tar.gz\n' % version_info["build"], + 'Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}%{?_pre_rc}.tar.gz\n', startswith=True, dryrun=options.dryrun) # update the version in reports |