diff options
69 files changed, 1016 insertions, 445 deletions
diff --git a/.travis.yml b/.travis.yml index 655d9fad5..73b8a9594 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ python: - "2.5" - "2.6" - "2.7" - - "3.2" env: - WITH_OPTIONAL_DEPS=yes - WITH_OPTIONAL_DEPS=no diff --git a/debian/bcfg2-report-collector.init b/debian/bcfg2-report-collector.init index 2d182385a..df7b751cb 100755 --- a/debian/bcfg2-report-collector.init +++ b/debian/bcfg2-report-collector.init @@ -32,6 +32,7 @@ test -x $DAEMON || exit 5 # Internal variables BINARY=$(basename $DAEMON) +RETVAL=0 start () { echo -n "Starting Configuration Report Collector: " @@ -85,22 +86,26 @@ status () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status + RETVAL=$? ;; restart|reload|force-reload) stop sleep 5 start + RETVAL=$? ;; *) log_success_msg "Usage: $0 {start|stop|status|reload|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/debian/bcfg2-server.init b/debian/bcfg2-server.init index 8de16b9b5..04774c063 100755 --- a/debian/bcfg2-server.init +++ b/debian/bcfg2-server.init @@ -41,6 +41,7 @@ test -x $DAEMON || exit 5 # Internal variables BINARY=$(basename $DAEMON) +RETVAL=0 start () { echo -n "Starting Configuration Management Server: " @@ -91,22 +92,26 @@ status () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status + RETVAL=$? ;; restart|reload|force-reload) stop sleep 5 start + RETVAL=$? ;; *) log_success_msg "Usage: $0 {start|stop|status|reload|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/debian/bcfg2.init b/debian/bcfg2.init index 4f83adbf6..b2e47b346 100755 --- a/debian/bcfg2.init +++ b/debian/bcfg2.init @@ -47,6 +47,7 @@ fi # Internal variables BINARY=$(basename $BCFG2) +RETVAL=0 # Include lsb functions . /lib/lsb/init-functions @@ -70,17 +71,19 @@ start () { case "$1" in start) start + RETVAL=$? ;; stop|status) - exit 0 + RETVAL=0 ;; restart|force-reload) start + RETVAL=$? ;; *) echo "Usage: $0 {start|stop|status|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/doc/_templates/indexsidebar.html b/doc/_templates/indexsidebar.html index 39916315d..2133cdcc5 100644 --- a/doc/_templates/indexsidebar.html +++ b/doc/_templates/indexsidebar.html @@ -7,5 +7,6 @@ <ul> <li><a href="http://docs.bcfg2.org/1.1/">Bcfg2 1.1 (stable)</a></li> <li><a href="http://docs.bcfg2.org/1.2/">Bcfg2 1.2 (stable)</a></li> + <li><a href="http://docs.bcfg2.org/1.3/">Bcfg2 1.3 (stable)</a></li> <li><a href="http://docs.bcfg2.org/dev/">Bcfg2 development documentation</a></li> </ul> diff --git a/doc/development/documentation.txt b/doc/development/documentation.txt index 2a3cf46d1..4d8a7c9f8 100644 --- a/doc/development/documentation.txt +++ b/doc/development/documentation.txt @@ -8,13 +8,14 @@ There are two parts of documentation in the Bcfg2 project: -* The wiki -* The manual +* The Wiki_ +* The Manual_ The wiki ======== .. _Wiki: http://bcfg2.org +.. _Manual: http://docs.bcfg2.org .. _Trac: http://trac.edgewall.org/ .. _OpenID: https://openid.org/ .. _MCS: http://www.mcs.anl.gov/ @@ -31,9 +32,9 @@ The manual ========== .. _rst: http://en.wikipedia.org/wiki/ReStructuredText .. _Sphinx: http://sphinx.pocoo.org -.. _Docutils: +.. _Docutils: http://docutils.sourceforge.net -The source for the manual is located in the ``doc/`` directory in the +The source for the Manual_ is located in the ``doc/`` directory in the git repository or in the source tarball. All files are written in rst_ (ReStructuredText) format. Sphinx_ is used to build the documentation from the restructured text sources. @@ -49,11 +50,20 @@ Building the Manual apt-get -t lenny-backports install python-sphinx - * The needed tools for Fedora based systems are in the `Fedora + * The tools for Fedora based systems are in the `Fedora Package Collection <https://admin.fedoraproject.org/pkgdb>`_; installation can be done easily with Yum:: yum -y install python-sphinx python-docutils + + * The tools for RHEL6-based systems are in the base distribution; you can install them with Yum:: + + yum -y install python-sphinx python-docutils + + * The tools for RHEL5-based systems are in the `Extra Packages for Enterprise Linux(EPEL) <https://fedoraproject.org/wiki/EPEL>`_ repository; if your system is configured for EPEL, you can install them with Yum:: + + yum -y install python-sphinx python-docutils + * Additionally, to build the PDF version: @@ -80,14 +90,14 @@ Documentation Style Guide for Bcfg2 =================================== This is a style guide to use when creating documentation for Bcfg2. It -is meant to be helpful, not a hinderence. +is meant to be helpful, not a hindrance. Basics ------ **Bcfg2** - When referring to project, Bcfg2 is the preferred use of cases. + When referring to project, Bcfg2 is the preferred use of case. **Monospace fonts** @@ -97,8 +107,7 @@ Basics **Repository** When used alone this refers to a Bcfg2 :term:`repository`. When there - is a chance for confusion, for instance in documents also talking - about :term:`VCS`, be sure to use the longer Bcfg2 :term:`repository`. + is a chance for confusion, for instance in documents that also discuss :term:`VCS`, be sure to use the longer phrase "Bcfg2 :term:`repository`". Sections -------- diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index b3028e9e1..9de3f868f 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -53,40 +53,41 @@ Prerequisites Install ------- -Be sure to include the specified fields included in the example -``bcfg2.conf`` file. These can be specified in either ``/etc/bcfg2.conf``, -if it is readable by the webserver user, or ``/etc/bcfg2-web.conf``. Any -database supported by `Django <http://www.djangoproject.com>`_ can be used. -As of version 1.3, `South <http://south.aeracode.org>`_ is used to control -schema changes. If your database is not supported by South, any updates -will need to be applied manually. Sqlite is configured by default. -Please see the :ref:`reporting-databases` section to configure alternative -databases. -.. warning:: +1. Be sure to include the specified fields included in the example + ``bcfg2.conf`` file. These can be specified in either + ``/etc/bcfg2.conf``, if it is readable by the webserver user, + or ``/etc/bcfg2-web.conf``. Any database supported by `Django + <http://www.djangoproject.com>`_ can be used. As of version 1.3, + `South <http://south.aeracode.org>`_ is used to control schema changes. + If your database is not supported by South, any updates will need to + be applied manually. Sqlite is configured by default. Please see the + :ref:`reporting-databases` section to configure alternative databases. - If you are using an sqlite database, the directory containing the - database file will need to be writable by the web server. The reason - for this is that sqlite will create another file for its journal - when it tries to update the database file. + .. warning:: -.. note:: + If you are using an sqlite database, the directory containing the + database file will need to be writable by the web server. The reason + for this is that sqlite will create another file for its journal + when it tries to update the database file. + + .. note:: - Distributed environments can share a single remote database for - reporting. + Distributed environments can share a single remote database for + reporting. -After configuring your database be sure to run `bcfg2-admin reports init` -to create the schema. +2. After configuring your database be sure to run ``bcfg2-admin reports + init`` to create the schema. -To enable statistics collection in the bcfg2-server, add -:ref:`server-plugins-statistics-reporting` to the **plugins** -line in your ``bcfg2.conf`` and restart the bcfg2-server. A report collecting -daemon should be run to import the collected statistics into the backend. -Please see the section :ref:`Report Collector <report_collector>` for more -information. +3. To enable statistics collection in the bcfg2-server, add + :ref:`server-plugins-statistics-reporting` to the **plugins** + line in your ``bcfg2.conf`` and restart the bcfg2-server. A report + collecting daemon should be run to import the collected statistics + into the backend. Please see the section :ref:`Report Collector + <report_collector>` for more information. -Detailed installation instructions can be found :ref:`here -<appendix-guides-web-reports-install>`. + Detailed installation instructions can be found :ref:`here + <appendix-guides-web-reports-install>`. .. _dynamic-http-install: @@ -175,7 +176,7 @@ Upgrading .. note:: After the database is upgraded all of the old tables are left - intact. To remove them any table starting with reports_ can + intact. To remove them any table starting with **reports\_** can be dropped. 4. `(Optional)` Run the :ref:`Report Collector <report_collector>` @@ -199,11 +200,6 @@ An example using the defaults is listed below:: host = port = - [statistics] - config = /etc/bcfg2-web.conf - time_zone = - web_debug = False - [reporting] transport = DirectStore web_prefix = @@ -241,6 +237,8 @@ section: statistics ^^^^^^^^^^ +.. deprecated: 1.3.0 + * config: The config file to be read for additional reporting data. This is used to restrict what can be read by the web server. diff --git a/doc/server/plugins/connectors/grouplogic.txt b/doc/server/plugins/connectors/grouplogic.txt new file mode 100644 index 000000000..b9a5b00d6 --- /dev/null +++ b/doc/server/plugins/connectors/grouplogic.txt @@ -0,0 +1,122 @@ +.. -*- mode: rst -*- + +.. _server-plugins-connectors-grouplogic: + +========== +GroupLogic +========== + +.. versionadded:: 1.3.2 + +GroupLogic is a connector plugin that lets you use an XML Genshi +template to dynamically set additional groups for clients. + +Usage +===== + +To use the GroupLogic plugin, first do ``mkdir +/var/lib/bcfg2/GroupLogic``. Add ``GroupLogic`` to your ``plugins`` +line in ``/etc/bcfg2.conf``. Next, create +``/var/lib/bcfg2/GroupLogic/groups.xml``: + +.. code-block:: xml + + <GroupLogic xmlns:py="http://genshi.edgewall.org/"> + </GroupLogic> + +``groups.xml`` is structured very similarly to the +:ref:`server-plugins-grouping-metadata` ``groups.xml``. A Group tag +that contains no children is a declaration of membership; a Group or +Client tag that does contain children is a conditional. + +Unlike ``Metadata/groups.xml``, GroupLogic supports genshi templating, +so you can dynamically create groups. ``GroupLogic/groups.xml`` is +rendered for each client, and the groups set in it are added to the +client metadata. + +.. note:: + + Also unlike ``Metadata/groups.xml``, GroupLogic can not be used to + associate bundles with clients directly, or to negate groups. But + you can use GroupLogic to assign a group that is associated with a + bundle in Metadata. + +Consider the case where you have four environments -- dev, test, +staging, and production -- and four components to a web application -- +the frontend, the API, the database server, and the caching proxy. In +order to make files specific to the component *and* to the +environment, you need groups to describe each combination: +webapp-frontend-dev, webapp-frontend-test, and so on. You *could* do +this in ``Metadata/groups.xml``: + +.. code-block:: xml + + <Groups> + <Group name="webapp-frontend"> + <Group name="dev"> + <Group name="webapp-frontend-dev"/> + </Group> + <Group name="test"> + <Group name="webapp-frontend-test"/> + </Group> + ... + </Group> + <Group name="webapp-api"> + ... + </Group> + ... + </Groups> + +Creating the sixteen groups this way is incredibly tedious, and this +is a quite *small* site. GroupLogic can automate this process. + +Assume that we've declared the groups thusly in +``Metadata/groups.xml``: + +.. code-block:: xml + + <Groups> + <Group name="webapp-frontend" category="webapp-component"/> + <Group name="webapp-api" category="webapp-component"/> + <Group name="webapp-db" category="webapp-component"/> + <Group name="webapp-proxy" category="webapp-component"/> + <Group name="dev" category="environment"/> + <Group name="test" category="environment"/> + <Group name="staging" category="environment"/> + <Group name="prod" category="environment"/> + </Groups> + +One way to automate the creation of the groups would be to simply +generate the tedious config: + +.. code-block:: xml + + <GroupLogic xmlns:py="http://genshi.edgewall.org/"> + <py:for each="component in metadata.query.all_groups_in_category("webapp-component")> + <Group name="${component}"> + <py:for each="env in metadata.query.all_groups_in_category("environment")> + <Group name="${env}"> + <Group name="${component}-${env}"/> + </Group> + </py:for> + </Group> + </py:for> + </GroupLogic> + +But, since ``GroupLogic/groups.xml`` is rendered for each client +individually, there's a more elegant way to accomplish the same thing: + +.. code-block:: xml + + <GroupLogic xmlns:py="http://genshi.edgewall.org/"> + <?python +component = metadata.group_in_category("webapp-component") +env = metadata.group_in_category("environment") + ?> + <py:if test="component and env"> + <Group name="${component}-${env}"/> + </py:if> + </GroupLogic> + +This gets only the component and environment for the current client, +and, if both are set, sets the single appropriate group. diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index 73145fd6b..606e1e128 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -434,7 +434,7 @@ configs. Simply add entries like these to the appropriate bundles: .. code-block:: xml <Path name="/etc/yum.repos.d/bcfg2.repo"/> - <Path name="/etc/apt/sources.d/bcfg2"/> + <Path name="/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list"/> If you want to change the path to either of those files, you can set ``yum_config`` or ``apt_config`` in ``bcfg2.conf`` to the path to the @@ -702,25 +702,25 @@ It understands the following directives: [packages] section ------------------ -+-------------+------------------------------------------------------+----------+-----------------------------+ -| Name | Description | Values | Default | -+=============+======================================================+==========+=============================+ -| resolver | Enable dependency resolution | Boolean | True | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| metadata | Enable metadata processing. Disabling ``metadata`` | Boolean | True | -| | implies disabling ``resolver`` as well. | | | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| yum_config | The path at which to generate Yum configs. | String | /etc/yum.repos.d/bcfg2.repo | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| apt_config | The path at which to generate APT configs. | String | /etc/apt/sources.d/bcfg2 | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| gpg_keypath | The path on the client RPM GPG keys will be copied | String | /etc/pki/rpm-gpg | -| | to before they are imported on the client. | | | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| version | Set the version attribute used when binding Packages | any|auto | auto | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| cache | Path where Packages will store its cache | String | <repo>/Packages/cache | -+-------------+------------------------------------------------------+----------+-----------------------------+ ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| Name | Description | Values | Default | ++=============+======================================================+==========+===================================================================+ +| resolver | Enable dependency resolution | Boolean | True | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| metadata | Enable metadata processing. Disabling ``metadata`` | Boolean | True | +| | implies disabling ``resolver`` as well. | | | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| yum_config | The path at which to generate Yum configs. | String | /etc/yum.repos.d/bcfg2.repo | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| apt_config | The path at which to generate APT configs. | String | /etc/apt/sources.list.d/bcfg2-packages-generated-sources.list | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| gpg_keypath | The path on the client RPM GPG keys will be copied | String | /etc/pki/rpm-gpg | +| | to before they are imported on the client. | | | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| version | Set the version attribute used when binding Packages | any|auto | auto | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| cache | Path where Packages will store its cache | String | <repo>/Packages/cache | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ [packages:yum] section diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index 2789411e7..845006115 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -117,8 +117,13 @@ describe the attributes available for various Path types. Note that ``secontext`` below expects a full context, not just the type. For instance, "``system_u:object_r:etc_t:s0``", not just ``etc_t``. You can also specify "``__default__``", which will restore -the context of the file to the default set by policy. See -:ref:`server-selinux` for more information. +the context of the file to the default set by policy. If a file has +no default context rule, and you don't wish to set one, you can +specify ``secontext=''`` (i.e., an empty ``secontext``), in which case +the client will not try to manage the SELinux context of the file at +all. + +See :ref:`server-selinux` for more information. Attributes common to all Path tags: diff --git a/doc/server/plugins/generators/sslca.txt b/doc/server/plugins/generators/sslca.txt index cab7eb233..7ef358a31 100644 --- a/doc/server/plugins/generators/sslca.txt +++ b/doc/server/plugins/generators/sslca.txt @@ -156,7 +156,7 @@ Example .. code-block:: xml <CertInfo> - <SubjectAltName>test.example.com</SubjectAltName> + <subjectAltName>test.example.com</subjectAltName> <Group name="apache"> <Cert key="/etc/pki/tls/private/foo.key" days="730"/> </Group> diff --git a/doc/server/plugins/grouping/metadata.txt b/doc/server/plugins/grouping/metadata.txt index fe0d2683e..32834b458 100644 --- a/doc/server/plugins/grouping/metadata.txt +++ b/doc/server/plugins/grouping/metadata.txt @@ -119,20 +119,19 @@ a simple ``groups.xml`` file: <Group name='oracle-server'> <Group name='selinux-enabled' negate='true'/> </Group> - <Client name='foo.eample.com'> + <Client name='foo.example.com'> <Group name='oracle-server'/> <Group name='apache-server'/> </Client> </Groups> -A Group or Client tag that does not contain any child tags is a -declaration of membership; a Group or Client tag that does contain -children is a conditional. So the example above does not assign -either the ``rhel5`` or ``rhel6`` groups to machines in the -``mail-server`` group, but conditionally assigns the -``sendmail-server`` or ``postfix-server`` groups depending on the OS -of the client. (Presumably in this example the OS groups are set by a -probe.) +A Group tag that does not contain any child tags is a declaration of +membership; a Group or Client tag that does contain children is a +conditional. So the example above does not assign either the +``rhel5`` or ``rhel6`` groups to machines in the ``mail-server`` +group, but conditionally assigns the ``sendmail-server`` or +``postfix-server`` groups depending on the OS of the client. +(Presumably in this example the OS groups are set by a probe.) Consequently, a client that is RHEL 5 and a member of the ``mail-server`` profile group would also be a member of the diff --git a/doc/server/selinux.txt b/doc/server/selinux.txt index 9f54b0d68..79384970a 100644 --- a/doc/server/selinux.txt +++ b/doc/server/selinux.txt @@ -142,13 +142,13 @@ necessary. Duplicate Entries ----------------- -It may be necessary to use `BoundSELinux` tags if a single fcontext +It may be necessary to use `BoundSEFcontext` tags if a single fcontext needs two different SELinux types depending on whether it's a symlink or a plain file. For instance: .. code-block:: xml - <BoundSELinux type="fcontext" filetype="symlink" - name="/etc/localtime" selinuxtype="etc_t"/> - <BoundSELinux type="fcontext" filetype="regular" - name="/etc/localtime" selinuxtype="locale_t"/> + <BoundSEFcontext filetype="symlink" + name="/etc/localtime" selinuxtype="etc_t"/> + <BoundSEFcontext filetype="regular" + name="/etc/localtime" selinuxtype="locale_t"/> diff --git a/gentoo/bcfg2-1.3.0.ebuild b/gentoo/bcfg2-1.3.0.ebuild index e600448d9..4d8530b02 100644 --- a/gentoo/bcfg2-1.3.0.ebuild +++ b/gentoo/bcfg2-1.3.0.ebuild @@ -1,13 +1,13 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Header: $ -EAPI="4" +EAPI=4 -PYTHON_DEPEND="2:2.6" +PYTHON_DEPEND="*:2.6" SUPPORT_PYTHON_ABIS="1" # ssl module required. -RESTRICT_PYTHON_ABIS="2.4 2.5 3.*" +RESTRICT_PYTHON_ABIS="2.5" inherit distutils eutils @@ -15,7 +15,7 @@ DESCRIPTION="configuration management tool" HOMEPAGE="http://bcfg2.org" SRC_URI="ftp://ftp.mcs.anl.gov/pub/bcfg/${P}.tar.gz" -LICENSE="BSD" +LICENSE="BSD-2" SLOT="0" KEYWORDS="~amd64 ~x86 ~amd64-linux ~x86-linux ~x64-solaris" IUSE="doc cheetah genshi server" diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index e6b21d76c..d8eb8e5de 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -153,7 +153,7 @@ Requires: bcfg2-server = %{version} # cherrypy 3.2.3 actually doesn't exist yet, but 3.2.2 has bugs that # prevent it from working: # https://bitbucket.org/cherrypy/cherrypy/issue/1154/assertionerror-in-recv-when-ssl-is-enabled -Requires: python-cherrypy > 3.2.2 +Requires: python-cherrypy > 3.3 %description server-cherrypy Bcfg2 helps system administrators produce a consistent, reproducible, @@ -523,4 +523,3 @@ fi * Fri Sep 15 2006 Narayan Desai <desai@mcs.anl.gov> - 0.8.4-1 - Initial log - diff --git a/redhat/scripts/bcfg2-report-collector.init b/redhat/scripts/bcfg2-report-collector.init index a8e23f080..43e875a6b 100755 --- a/redhat/scripts/bcfg2-report-collector.init +++ b/redhat/scripts/bcfg2-report-collector.init @@ -32,6 +32,7 @@ test -x $DAEMON || exit 5 # Internal variables BINARY=$(basename $DAEMON) +RETVAL=0 start () { echo -n "Starting Configuration Report Collector: " @@ -79,22 +80,26 @@ status () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status + RETVAL=$? ;; restart|reload|force-reload) stop sleep 5 start + RETVAL=$? ;; *) echo "Usage: $0 {start|stop|status|reload|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/redhat/scripts/bcfg2-server.init b/redhat/scripts/bcfg2-server.init index ffac6ac3d..c4412d1c3 100755 --- a/redhat/scripts/bcfg2-server.init +++ b/redhat/scripts/bcfg2-server.init @@ -59,9 +59,11 @@ stop () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status $prog @@ -71,6 +73,7 @@ case "$1" in stop sleep 5 start + RETVAL=$? ;; *) echo $"Usage: $0 {start|stop|status|restart|reload|force-reload}" diff --git a/redhat/scripts/bcfg2.init b/redhat/scripts/bcfg2.init index 5cfdf47bc..9c26434ff 100755 --- a/redhat/scripts/bcfg2.init +++ b/redhat/scripts/bcfg2.init @@ -54,12 +54,14 @@ start () { case "$1" in start) start + RETVAL=$? ;; stop|status) - exit 0 + RETVAL=0 ;; restart|force-reload) start + RETVAL=$? ;; *) echo "Usage: $0 {start|stop|status|restart|force-reload}" diff --git a/schemas/grouplogic.xsd b/schemas/grouplogic.xsd new file mode 100644 index 000000000..bf43bceb3 --- /dev/null +++ b/schemas/grouplogic.xsd @@ -0,0 +1,110 @@ +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> + + <xsd:annotation> + <xsd:documentation> + GroupLogic schema for bcfg2 + </xsd:documentation> + </xsd:annotation> + + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + + <xsd:complexType name="GroupLogicDeclarationType"> + <xsd:annotation> + <xsd:documentation> + A **GroupLogicDeclarationType** declares a Group to be added + to a client. + </xsd:documentation> + </xsd:annotation> + <xsd:attribute type='xsd:string' name='name' use='required'> + <xsd:annotation> + <xsd:documentation> + The group name + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name="GroupLogicType"> + <xsd:annotation> + <xsd:documentation> + The top-level tag of a GroupLogic configuration file. + </xsd:documentation> + </xsd:annotation> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:group ref="py:genshiElements"/> + <xsd:element name='Group' type='GroupLogicDeclarationType'/> + <xsd:element name='Group' type='GroupLogicContainerType'> + <xsd:annotation> + <xsd:documentation> + Elements within Group tags only apply to clients that are + members of that group (or vice-versa; see #element_negate + below) + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name='Client' type='GroupLogicContainerType'> + <xsd:annotation> + <xsd:documentation> + Elements within Client tags only apply to the named client + (or vice-versa; see #element_negate below) + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name='GroupLogic' type='GroupLogicType'> + <xsd:annotation> + <xsd:documentation> + Nesting GroupLogic tags is allowed in order to support + XInclude. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + </xsd:choice> + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name="GroupLogicContainerType"> + <xsd:annotation> + <xsd:documentation> + A **GroupLogicContainerType** is a tag used to provide logic. + Child entries of a GroupLogicContainerType tag only apply to + machines that match the condition specified -- either + membership in a group, or a matching client name. + :xml:attribute:`GroupLogicContainerType:negate` can be set to + negate the sense of the match. + </xsd:documentation> + </xsd:annotation> + <xsd:complexContent> + <xsd:extension base="GroupLogicType"> + <xsd:attribute type='xsd:string' name='name' use='required'> + <xsd:annotation> + <xsd:documentation> + The group name + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type='xsd:string' name='negate'> + <xsd:annotation> + <xsd:documentation> + Negate the sense of this group or client; i.e., entries + within this tag are only used on clients that are not + members of the group, or that have hostnames that do not + match. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:extension> + </xsd:complexContent> + </xsd:complexType> + + <xsd:element name='GroupLogic' type='GroupLogicType'> + <xsd:annotation> + <xsd:documentation> + A GroupLogic file is a genshi file that can be used to + dynamically add additional groups to a client. + </xsd:documentation> + </xsd:annotation> + </xsd:element> +</xsd:schema> diff --git a/schemas/selinux.xsd b/schemas/selinux.xsd index 760953e34..3651549f5 100644 --- a/schemas/selinux.xsd +++ b/schemas/selinux.xsd @@ -80,6 +80,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to this port + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -127,6 +134,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to files matching this specification + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -157,6 +171,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to this node + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -205,6 +226,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to this user + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -235,6 +263,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to this user + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -258,6 +293,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to this interface + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/types.xsd b/schemas/types.xsd index 524b327c5..31fea26a2 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -422,8 +422,10 @@ <xsd:attribute type="xsd:string" name="gecos"> <xsd:annotation> <xsd:documentation> - Human-readable user name or comment. If this is not set, - the GECOS will be the same as the username. + This field is typically used to record general information + about the account or its user(s) such as their real name + and phone number. If this is not set, the GECOS will be + the same as the username. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index 5633764a8..1676ee717 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -91,7 +91,10 @@ class Client(object): try: script.write("#!%s\n" % (probe.attrib.get('interpreter', '/bin/sh'))) - script.write(probe.text) + if sys.hexversion >= 0x03000000: + script.write(probe.text) + else: + script.write(probe.text.encode('utf-8')) script.close() os.chmod(scriptname, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | @@ -105,7 +108,10 @@ class Client(object): self._probe_failure(name, "Return value %s" % rv) self.logger.info("Probe %s has result:" % name) self.logger.info(rv.stdout) - ret.text = rv.stdout + if sys.hexversion >= 0x03000000: + ret.text = rv.stdout + else: + ret.text = rv.stdout.decode('utf-8') finally: os.unlink(scriptname) except SystemExit: @@ -167,7 +173,7 @@ class Client(object): self.proxy.RecvProbeData( Bcfg2.Client.XML.tostring( probedata, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False).decode('utf-8')) except Bcfg2.Proxy.ProxyError: err = sys.exc_info()[1] self.fatal_error("Failed to upload probe data: %s" % err) @@ -229,7 +235,7 @@ class Client(object): self.fatal_error("Failed to get decision list: %s" % err) try: - rawconfig = self.proxy.GetConfig().encode('UTF-8') + rawconfig = self.proxy.GetConfig().encode('utf-8') except Bcfg2.Proxy.ProxyError: err = sys.exc_info()[1] self.fatal_error("Failed to download configuration from " @@ -247,7 +253,7 @@ class Client(object): self.logger.info("Starting Bcfg2 client run at %s" % times['start']) - rawconfig = self.get_config(times=times) + rawconfig = self.get_config(times=times).decode('utf-8') if self.setup['cache']: try: @@ -324,7 +330,7 @@ class Client(object): self.proxy.RecvStats( Bcfg2.Client.XML.tostring( feedback, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False).decode('utf-8')) except Bcfg2.Proxy.ProxyError: err = sys.exc_info()[1] self.logger.error("Failed to upload configuration statistics: " diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index ada5320b8..850e58d9d 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -417,15 +417,18 @@ class Frame(object): bundle.get('name') not in self.setup['bundle']): # prune out unspecified bundles when running with -b continue + if bundle in mbundles: + self.logger.debug("Bundle %s was modified" % bundle) + func = "BundleUpdated" + else: + self.logger.debug("Bundle %s was not modified" % bundle) + func = "BundleNotUpdated" for tool in self.tools: try: - if bundle in mbundles: - tool.BundleUpdated(bundle, self.states) - else: - tool.BundleNotUpdated(bundle, self.states) + getattr(tool, func)(bundle, self.states) except: - self.logger.error("%s.BundleNotUpdated() call failed:" % - tool.name, exc_info=1) + self.logger.error("%s.%s() call failed:" % + (tool.name, func), exc_info=1) def Remove(self): """Remove extra entries.""" @@ -447,15 +450,16 @@ class Frame(object): self.logger.info('Incorrect entries: %d' % list(self.states.values()).count(False)) if phase == 'final' and list(self.states.values()).count(False): - for entry in self.states.keys(): + for entry in sorted(self.states.keys(), key=lambda e: e.tag + ":" + + e.get('name')): if not self.states[entry]: etype = entry.get('type') if etype: self.logger.info("%s:%s:%s" % (entry.tag, etype, entry.get('name'))) else: - self.logger.info(" %s:%s" % (entry.tag, - entry.get('name'))) + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) self.logger.info('Total managed entries: %d' % len(list(self.states.values()))) self.logger.info('Unmanaged entries: %d' % len(self.extra)) @@ -467,8 +471,8 @@ class Frame(object): self.logger.info("%s:%s:%s" % (entry.tag, etype, entry.get('name'))) else: - self.logger.info(" %s:%s" % (entry.tag, - entry.get('name'))) + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) if ((list(self.states.values()).count(False) == 0) and not self.extra): self.logger.info('All entries correct.') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 9b95d2234..168c35c98 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -34,13 +34,11 @@ class POSIXFile(POSIXTool): def _get_data(self, entry): """ Get a tuple of (<file data>, <is binary>) for the given entry """ - is_binary = False - if entry.get('encoding', 'ascii') == 'base64': - tempdata = b64decode(entry.text) - is_binary = True - - elif entry.get('empty', 'false') == 'true': + is_binary = entry.get('encoding', 'ascii') == 'base64' + if entry.get('empty', 'false') == 'true' or not entry.text: tempdata = '' + elif is_binary: + tempdata = b64decode(entry.text) else: tempdata = entry.text if isinstance(tempdata, unicode) and unicode != str: diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 11f331ddb..16fe0acb5 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -275,7 +275,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if path is None: path = entry.get("name") context = entry.get("secontext") - if context is None: + if not context: # no context listed return True @@ -520,13 +520,19 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): "Current mtime is %s but should be %s" % (path, mtime, entry.get('mtime'))) - if HAS_SELINUX and entry.get("secontext"): + if HAS_SELINUX: + wanted_secontext = None if entry.get("secontext") == "__default__": - wanted_secontext = \ - selinux.matchpathcon(path, 0)[1].split(":")[2] + try: + wanted_secontext = \ + selinux.matchpathcon(path, 0)[1].split(":")[2] + except OSError: + errors.append("%s has no default SELinux context" % + entry.get("name")) else: wanted_secontext = entry.get("secontext") - if attrib['current_secontext'] != wanted_secontext: + if (wanted_secontext and + attrib['current_secontext'] != wanted_secontext): errors.append("SELinux context for path %s is incorrect. " "Current context is %s but should be %s" % (path, attrib['current_secontext'], diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index 552b27842..4b78581f7 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -12,6 +12,15 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'rc-update')] __req__ = {'Service': ['name', 'status']} + def get_enabled_svcs(self): + """ + Return a list of all enabled services. + """ + return [line.split()[0] + for line in self.cmd.run(['/bin/rc-status', + '-s']).stdout.splitlines() + if 'started' in line] + def VerifyService(self, entry, _): """ Verify Service status for entry. @@ -21,9 +30,12 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): if entry.get('status') == 'ignore': return True + # get a list of all started services + allsrv = self.get_enabled_svcs() + # check if service is enabled - result = self.cmd.run(["/sbin/rc-update", "show", "default"]) - is_enabled = entry.get("name") in result.stdout + result = self.cmd.run(["/sbin/rc-update", "show", "default"]).stdout + is_enabled = entry.get("name") in result # check if init script exists try: @@ -34,8 +46,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): return False # check if service is enabled - result = self.cmd.run(self.get_svc_command(entry, "status")) - is_running = "started" in result.stdout + is_running = entry.get('name') in allsrv if entry.get('status') == 'on' and not (is_enabled and is_running): entry.set('current_status', 'off') @@ -70,10 +81,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): def FindExtra(self): """Locate extra rc-update services.""" - allsrv = [line.split()[0] - for line in self.cmd.run(['/bin/rc-status', - '-s']).stdout.splitlines() - if 'started' in line] + allsrv = self.get_enabled_svcs() self.logger.debug('Found active services:') self.logger.debug(allsrv) specified = [srv.get('name') for srv in self.getSupportedEntries()] diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 0041ce61a..0b4aba60d 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -204,7 +204,16 @@ class SELinuxEntryHandler(object): type, if the records object supports the customized() method """ if hasattr(self.records, "customized") and self.custom_re: - return dict([(k, self.all_records[k]) for k in self.custom_keys]) + rv = dict() + for key in self.custom_keys: + if key in self.all_records: + rv[key] = self.all_records[key] + else: + self.logger.warning("SELinux %s %s customized, but no " + "record found. This may indicate an " + "error in your SELinux policy." % + (self.etype, key)) + return rv else: # ValueError is really a pretty dumb exception to raise, # but that's what the seobject customized() method raises @@ -491,7 +500,8 @@ class SELinuxSeportHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding and modifying entries """ (port, proto) = entry.get("name").split("/") - return (port, proto, '', entry.get("selinuxtype")) + return (port, proto, entry.get("mlsrange", ""), + entry.get("selinuxtype")) def _deleteargs(self, entry): return tuple(entry.get("name").split("/")) @@ -564,7 +574,7 @@ class SELinuxSefcontextHandler(SELinuxEntryHandler): """ argument list for adding, modifying, and deleting entries """ return (entry.get("name"), entry.get("selinuxtype"), self.filetypeargs[entry.get("filetype", "all")], - '', '') + entry.get("mlsrange", ""), '') def primarykey(self, entry): return ":".join([entry.tag, entry.get("name"), @@ -599,7 +609,7 @@ class SELinuxSenodeHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ (addr, netmask) = entry.get("name").split("/") - return (addr, netmask, entry.get("proto"), "", + return (addr, netmask, entry.get("proto"), entry.get("mlsrange", ""), entry.get("selinuxtype")) @@ -611,7 +621,8 @@ class SELinuxSeloginHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ - return (entry.get("name"), entry.get("selinuxuser"), "") + return (entry.get("name"), entry.get("selinuxuser"), + entry.get("mlsrange", "")) class SELinuxSeuserHandler(SELinuxEntryHandler): @@ -651,15 +662,16 @@ class SELinuxSeuserHandler(SELinuxEntryHandler): # prefix. see the comment in Install() above for more # details. rv = [entry.get("name"), - entry.get("roles", "").replace(" ", ",").split(",")] + entry.get("roles", "").replace(" ", ",").split(","), + '', entry.get("mlsrange", "")] if self.needs_prefix: - rv.extend(['', '', entry.get("prefix")]) + rv.append(entry.get("prefix")) else: key = self._key(entry) if key in self.all_records: attrs = self._key2attrs(key) if attrs['prefix'] != entry.get("prefix"): - rv.extend(['', '', entry.get("prefix")]) + rv.append(entry.get("prefix")) return tuple(rv) @@ -671,7 +683,8 @@ class SELinuxSeinterfaceHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ - return (entry.get("name"), '', entry.get("selinuxtype")) + return (entry.get("name"), entry.get("mlsrange", ""), + entry.get("selinuxtype")) class SELinuxSepermissiveHandler(SELinuxEntryHandler): diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index c9fae7fc7..c30c0a13a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -131,10 +131,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): def __init__(self, logger, setup, config): self.yumbase = self._loadYumBase(setup=setup, logger=logger) Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) - self.ignores = [entry.get('name') for struct in config \ - for entry in struct \ - if entry.tag == 'Path' and \ - entry.get('type') == 'ignore'] + self.ignores = [] + for struct in config: + self.ignores.extend([entry.get('name') + for entry in struct + if (entry.tag == 'Path' and + entry.get('type') == 'ignore')]) self.instance_status = {} self.extra_instances = [] self.modlists = {} @@ -293,8 +295,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): group. """ missing = Bcfg2.Client.Tools.PkgTool.missing_attrs(self, entry) - if entry.get('name', None) == None and \ - entry.get('group', None) == None: + if (entry.get('name', None) is None and + entry.get('group', None) is None): missing += ['name', 'group'] return missing @@ -422,10 +424,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if entry.get('group'): self.logger.debug("Verifying packages for group %s" % - entry.get('group')) + entry.get('group')) else: self.logger.debug("Verifying package instances for %s" % - entry.get('name')) + entry.get('name')) self.verify_cache = dict() # Used for checking multilib packages self.modlists[entry] = modlist @@ -434,10 +436,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool): package_fail = False qtext_versions = [] virt_pkg = False - pkg_checks = self.pkg_checks and \ - entry.get('pkg_checks', 'true').lower() == 'true' - pkg_verify = self.pkg_verify and \ - entry.get('pkg_verify', 'true').lower() == 'true' + pkg_checks = (self.pkg_checks and + entry.get('pkg_checks', 'true').lower() == 'true') + pkg_verify = (self.pkg_verify and + entry.get('pkg_verify', 'true').lower() == 'true') yum_group = False if entry.get('name') == 'gpg-pubkey': @@ -455,15 +457,13 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if d] group_type = entry.get('choose', 'default') if group_type in ['default', 'optional', 'all']: - group_packages += [p - for p, d in - group.default_packages.items() - if d] + group_packages += [ + p for p, d in group.default_packages.items() + if d] if group_type in ['optional', 'all']: - group_packages += [p - for p, d in - group.optional_packages.items() - if d] + group_packages += [ + p for p, d in group.optional_packages.items() + if d] if len(group_packages) == 0: self.logger.error("No packages found for group %s" % entry.get("group")) @@ -489,7 +489,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): else: all_pkg_objs = \ self.yumbase.rpmdb.searchNevra(name=entry.get('name')) - if len(all_pkg_objs) == 0 and yum_group != True: + if len(all_pkg_objs) == 0 and yum_group is not True: # Some sort of virtual capability? Try to resolve it all_pkg_objs = self.yumbase.rpmdb.searchProvides(entry.get('name')) if len(all_pkg_objs) > 0: @@ -567,9 +567,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool): pkg_objs = [po for po in all_pkg_objs] else: pkg_objs = [po for po in all_pkg_objs - if po.checkPrco('provides', - (nevra["name"], 'EQ', - tuple(vlist)))] + if po.checkPrco('provides', + (nevra["name"], 'EQ', + tuple(vlist)))] elif entry.get('name') == 'gpg-pubkey': if 'version' not in nevra: self.logger.warning("Skipping verify: gpg-pubkey without " @@ -622,7 +622,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if self.setup.get('quick', False): # Passed -q on the command line continue - if not (pkg_verify and \ + if not (pkg_verify and inst.get('pkg_verify', 'true').lower() == 'true'): continue @@ -648,8 +648,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Now take out the Yum specific objects / modlists / unproblems ignores = [ig.get('name') for ig in entry.findall('Ignore')] + \ - [ig.get('name') for ig in inst.findall('Ignore')] + \ - self.ignores + [ig.get('name') for ig in inst.findall('Ignore')] + \ + self.ignores for fname, probs in list(vrfy_result.items()): if fname in modlist: self.logger.debug(" %s in modlist, skipping" % fname) @@ -737,8 +737,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool): for pkg in pkg_objs: self.logger.debug(" Extra Instance Found: %s" % str(pkg)) Bcfg2.Client.XML.SubElement(extra_entry, 'Instance', - epoch=pkg.epoch, name=pkg.name, version=pkg.version, - release=pkg.release, arch=pkg.arch) + epoch=pkg.epoch, name=pkg.name, + version=pkg.version, + release=pkg.release, arch=pkg.arch) if pkg_objs == []: return None @@ -782,7 +783,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): ver = yum.misc.keyIdToRPMVer(gpg['keyid']) rel = yum.misc.keyIdToRPMVer(gpg['timestamp']) if not (ver == inst.get('version') and rel == inst.get('release')): - self.logger.info("GPG key file %s does not match gpg-pubkey-%s-%s"\ + self.logger.info("GPG key file %s does not match gpg-pubkey-%s-%s" % (key_file, inst.get('version'), inst.get('release'))) return False @@ -791,20 +792,21 @@ class YUM(Bcfg2.Client.Tools.PkgTool): gpg['timestamp']) == 0: result = tset.pgpImportPubkey(yum.misc.procgpgkey(rawkey)) else: - self.logger.debug("gpg-pubkey-%s-%s already installed"\ - % (inst.get('version'), - inst.get('release'))) + self.logger.debug("gpg-pubkey-%s-%s already installed" % + (inst.get('version'), inst.get('release'))) return True if result != 0: - self.logger.debug("Unable to install %s-%s" % \ - (self.instance_status[inst].get('pkg').get('name'), - nevra2string(inst))) + self.logger.debug( + "Unable to install %s-%s" % + (self.instance_status[inst].get('pkg').get('name'), + nevra2string(inst))) return False else: - self.logger.debug("Installed %s-%s-%s" % \ - (self.instance_status[inst].get('pkg').get('name'), - inst.get('version'), inst.get('release'))) + self.logger.debug( + "Installed %s-%s-%s" % + (self.instance_status[inst].get('pkg').get('name'), + inst.get('version'), inst.get('release'))) return True def _runYumTransaction(self): @@ -898,7 +900,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Remove extra instances. # Can not reverify because we don't have a package entry. if self.extra_instances is not None and len(self.extra_instances) > 0: - if (self.setup.get('remove') == 'all' or \ + if (self.setup.get('remove') == 'all' or self.setup.get('remove') == 'packages'): self.Remove(self.extra_instances) else: @@ -913,7 +915,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Figure out which instances of the packages actually need something # doing to them and place in the appropriate work 'queue'. for pkg in packages: - insts = [pinst for pinst in pkg \ + insts = [pinst for pinst in pkg if pinst.tag in ['Instance', 'Package']] if insts: for inst in insts: @@ -1006,10 +1008,11 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if not self.setup['kevlar']: for pkg_entry in [p for p in packages if self.canVerify(p)]: - self.logger.debug("Reverifying Failed Package %s" \ - % (pkg_entry.get('name'))) - states[pkg_entry] = self.VerifyPackage(pkg_entry, - self.modlists.get(pkg_entry, [])) + self.logger.debug("Reverifying Failed Package %s" % + pkg_entry.get('name')) + states[pkg_entry] = \ + self.VerifyPackage(pkg_entry, + self.modlists.get(pkg_entry, [])) for entry in [ent for ent in packages if states[ent]]: self.modified.append(entry) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 8c8c4fd94..e40ef750b 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -3,9 +3,7 @@ import os import sys import select -from Bcfg2.Compat import input, walk_packages # pylint: disable=W0622 - -__all__ = [m[1] for m in walk_packages(path=__path__)] +from Bcfg2.Compat import input # pylint: disable=W0622 def prompt(msg): diff --git a/src/lib/Bcfg2/Encryption.py b/src/lib/Bcfg2/Encryption.py index 2b4ba6237..b4674d72f 100755 --- a/src/lib/Bcfg2/Encryption.py +++ b/src/lib/Bcfg2/Encryption.py @@ -27,7 +27,7 @@ ALGORITHM = "aes_256_cbc" #: Default initialization vector. For best security, you should use a #: unique IV for each message. :func:`ssl_encrypt` does this in an #: automated fashion. -IV = '\0' * 16 +IV = r'\0' * 16 #: The config file section encryption options and passphrases are #: stored in @@ -116,9 +116,11 @@ def ssl_decrypt(data, passwd, algorithm=ALGORITHM): # base64-decode the data data = b64decode(data) salt = data[8:16] + # pylint: disable=E1101 hashes = [md5(passwd + salt).digest()] for i in range(1, 3): hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + # pylint: enable=E1101 key = hashes[0] + hashes[1] iv = hashes[2] @@ -144,9 +146,11 @@ def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): if salt is None: salt = Rand.rand_bytes(8) + # pylint: disable=E1101 hashes = [md5(passwd + salt).digest()] for i in range(1, 3): hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + # pylint: enable=E1101 key = hashes[0] + hashes[1] iv = hashes[2] diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 66e987b91..c7604c5c4 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -401,7 +401,8 @@ CFILE = \ Option('Specify configuration file', default=DEFAULT_CONFIG_LOCATION, cmd='-C', - odesc='<conffile>') + odesc='<conffile>', + env="BCFG2_CONFIG") LOCKFILE = \ Option('Specify lockfile', default='/var/lock/bcfg2.run', @@ -534,6 +535,11 @@ SERVER_FAM_IGNORE = \ 'SCCS', '.svn', '4913', '.gitignore'], cf=('server', 'ignore_files'), cook=list_split) +SERVER_FAM_BLOCK = \ + Option('FAM blocks on startup until all events are processed', + default=False, + cook=get_bool, + cf=('server', 'fam_blocking')) SERVER_LISTEN_ALL = \ Option('Listen on all interfaces', default=False, @@ -1082,6 +1088,15 @@ VERBOSE = \ cmd='-v', cook=get_bool, cf=('logging', 'verbose')) +LOG_PERFORMANCE = \ + Option("Periodically log performance statistics", + default=False, + cf=('logging', 'performance')) +PERFLOG_INTERVAL = \ + Option("Performance statistics logging interval in seconds", + default=300.0, + cook=get_timeout, + cf=('logging', 'performance_interval')) # Plugin-specific options CFG_VALIDATION = \ @@ -1156,6 +1171,7 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, password=SERVER_PASSWORD, filemonitor=SERVER_FILEMONITOR, ignore=SERVER_FAM_IGNORE, + fam_blocking=SERVER_FAM_BLOCK, location=SERVER_LOCATION, key=SERVER_KEY, cert=SERVER_CERT, @@ -1164,7 +1180,9 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, web_configfile=WEB_CFILE, backend=SERVER_BACKEND, vcs_root=SERVER_VCS_ROOT, - authentication=SERVER_AUTHENTICATION) + authentication=SERVER_AUTHENTICATION, + perflog=LOG_PERFORMANCE, + perflog_interval=PERFLOG_INTERVAL) CRYPT_OPTIONS = dict(encrypt=ENCRYPT, decrypt=DECRYPT, @@ -1269,6 +1287,11 @@ TEST_COMMON_OPTIONS = dict(noseopts=TEST_NOSEOPTS, xunit=TEST_XUNIT, validate=CFG_VALIDATION) +INFO_COMMON_OPTIONS = dict(ppath=PARANOID_PATH, + max_copies=PARANOID_MAX_COPIES) +INFO_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS) +INFO_COMMON_OPTIONS.update(SERVER_COMMON_OPTIONS) + class OptionParser(OptionSet): """ diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index b2b9fcc2e..9246c0f87 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -24,6 +24,7 @@ from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus version = sys.version_info[:2] has_py26 = version >= (2, 6) +has_py32 = version >= (3, 2) __all__ = ["ComponentProxy", "RetryMethod", @@ -173,8 +174,12 @@ class SSLHTTPConnection(httplib.HTTPConnection): """ if not has_py26: httplib.HTTPConnection.__init__(self, host, port, strict) - else: + elif not has_py32: httplib.HTTPConnection.__init__(self, host, port, strict, timeout) + else: + # the strict parameter is deprecated. + # HTTP 0.9-style "Simple Responses" are not supported anymore. + httplib.HTTPConnection.__init__(self, host, port, timeout=timeout) self.key = key self.cert = cert self.ca = ca diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py index 0a0f032e5..c7d5c512a 100644 --- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -87,7 +87,7 @@ class LocalFilesystem(TransportBase): # using a tmpfile to hopefully avoid the file monitor from grabbing too # soon - saved = open(tmp_file, 'w') + saved = open(tmp_file, 'wb') try: saved.write(payload) except IOError: @@ -123,7 +123,7 @@ class LocalFilesystem(TransportBase): self.debug_log("Handling event %s" % event.filename) payload = os.path.join(self.work_path, event.filename) try: - payloadfd = open(payload, "r") + payloadfd = open(payload, "rb") interaction = cPickle.load(payloadfd) payloadfd.close() os.unlink(payload) diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py index 5c51dad1e..73bdd0b3a 100644 --- a/src/lib/Bcfg2/Reporting/Transport/__init__.py +++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py @@ -2,11 +2,11 @@ Public transport routines """ -import traceback - +import sys from Bcfg2.Reporting.Transport.base import TransportError, \ TransportImportError + def load_transport(transport_name, setup): """ Try to load the transport. Raise TransportImportError on failure @@ -18,13 +18,14 @@ def load_transport(transport_name, setup): try: mod = __import__(transport_name) except: - raise TransportImportError("Unavailable") + raise TransportImportError("Error importing transport %s: %s" % + (transport_name, sys.exc_info()[1])) try: - cls = getattr(mod, transport_name) - return cls(setup) + return getattr(mod, transport_name)(setup) except: - raise TransportImportError("Transport unavailable: %s" % - traceback.format_exc().splitlines()[-1]) + raise TransportImportError("Error instantiating transport %s: %s" % + (transport_name, sys.exc_info()[1])) + def load_transport_from_config(setup): """Load the transport in the config... eventually""" @@ -32,4 +33,3 @@ def load_transport_from_config(setup): return load_transport(setup['reporting_transport'], setup) except KeyError: raise TransportImportError('Transport missing in config') - diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 4be509f53..e63c180a8 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -139,9 +139,11 @@ class Interaction(models.Model): posixgroups = models.ManyToManyField("POSIXGroupEntry") failures = models.ManyToManyField("FailureEntry") - entry_types = ('actions', 'packages', 'paths', 'services', 'sebooleans', - 'seports', 'sefcontexts', 'senodes', 'selogins', 'seusers', - 'seinterfaces', 'sepermissives', 'semodules', 'posixusers', + entry_types = ('actions', 'failures', 'packages', + 'paths', 'services', 'sebooleans', + 'seports', 'sefcontexts', 'senodes', + 'selogins', 'seusers', 'seinterfaces', + 'sepermissives', 'semodules', 'posixusers', 'posixgroups') # Formerly InteractionMetadata diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index 13c0563ec..93e42305c 100644 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -3,6 +3,7 @@ import getopt import lxml.etree import sys import Bcfg2.Server.Admin +from Bcfg2.Server.Plugin import PullSource class Minestruct(Bcfg2.Server.Admin.StructureMode): @@ -39,7 +40,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): try: extra = set() - for source in self.bcore.pull_sources: + for source in self.bcore.plugins_by_type(PullSource): for item in source.GetExtra(client): extra.add(item) except: diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index 9f1b3d138..8001425df 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -6,6 +6,7 @@ import sys import getopt import select import Bcfg2.Server.Admin +from Bcfg2.Server.Plugin import PullSource, Generator from Bcfg2.Compat import input # pylint: disable=W0622 @@ -62,13 +63,14 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): given client/entry from statistics. """ new_entry = {'type': etype, 'name': ename} - for plugin in self.bcore.pull_sources: + pull_sources = self.bcore.plugins_by_type(PullSource) + for plugin in pull_sources: try: (owner, group, mode, contents) = \ plugin.GetCurrentEntry(client, etype, ename) break except Bcfg2.Server.Plugin.PluginExecutionError: - if plugin == self.bcore.pull_sources[-1]: + if plugin == pull_sources[-1]: print("Pull Source failure; could not fetch current state") raise SystemExit(1) @@ -121,8 +123,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): meta = self.bcore.build_metadata(client) # Find appropriate plugin in bcore - glist = [gen for gen in self.bcore.generators if - ename in gen.Entries.get(etype, {})] + glist = [gen for gen in self.bcore.plugins_by_type(Generator) + if ename in gen.Entries.get(etype, {})] if len(glist) != 1: self.errExit("Got wrong numbers of matching generators for entry:" "%s" % ([g.name for g in glist])) diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 4d7453840..c3302f1d0 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -9,12 +9,12 @@ from Bcfg2.Server.Core import BaseCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse from Bcfg2.SSLServer import XMLRPCServer -from lockfile import LockFailed +from lockfile import LockFailed, LockTimeout # pylint: disable=E0611 try: - from daemon.pidfile import PIDLockFile + from daemon.pidfile import TimeoutPIDLockFile except ImportError: - from daemon.pidlockfile import PIDLockFile + from daemon.pidlockfile import TimeoutPIDLockFile # pylint: enable=E0611 @@ -33,7 +33,8 @@ class Core(BaseCore): gid=self.setup['daemon_gid'], umask=int(self.setup['umask'], 8)) if self.setup['daemon']: - daemon_args['pidfile'] = PIDLockFile(self.setup['daemon']) + daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'], + acquire_timeout=5) #: The :class:`daemon.DaemonContext` used to drop #: privileges, write the PID file (with :class:`PidFile`), #: and daemonize this core. @@ -89,6 +90,11 @@ class Core(BaseCore): err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False + except LockTimeout: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: Failed to acquire lock " + "on %s" % (self.name, self.setup['daemon'])) + return False def _run(self): """ Create :attr:`server` to start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index deb9065a5..ab8cda3da 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -19,8 +19,9 @@ from Bcfg2.Cache import Cache import Bcfg2.Statistics from itertools import chain from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 -from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError, \ - track_statistics +from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 +from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 +from Bcfg2.Server.Plugin import track_statistics try: import psyco @@ -96,6 +97,7 @@ class BaseCore(object): .. automethod:: _block .. ----- .. automethod:: _file_monitor_thread + .. automethod:: _perflog_thread """ #: The Bcfg2 repository directory self.datastore = setup['repo'] @@ -174,6 +176,9 @@ class BaseCore(object): #: the first one loaded wins. self.plugin_blacklist = {} + #: The Metadata plugin + self.metadata = None + #: Revision of the Bcfg2 specification. This will be sent to #: the client in the configuration, and can be set by a #: :class:`Bcfg2.Server.Plugin.interfaces.Version` plugin. @@ -235,71 +240,6 @@ class BaseCore(object): self.logger.error("Failed to set ownership of database " "at %s: %s" % (db_settings['NAME'], err)) - if '' in setup['plugins']: - setup['plugins'].remove('') - - for plugin in setup['plugins']: - if not plugin in self.plugins: - self.init_plugin(plugin) - # Remove blacklisted plugins - for plugin, blacklist in list(self.plugin_blacklist.items()): - if len(blacklist) > 0: - self.logger.error("The following plugins conflict with %s;" - "Unloading %s" % (plugin, blacklist)) - for plug in blacklist: - del self.plugins[plug] - - # Log experimental plugins - expl = [plug for plug in list(self.plugins.values()) - if plug.experimental] - if expl: - self.logger.info("Loading experimental plugin(s): %s" % - (" ".join([x.name for x in expl]))) - self.logger.info("NOTE: Interfaces subject to change") - - # Log deprecated plugins - depr = [plug for plug in list(self.plugins.values()) - if plug.deprecated] - if depr: - self.logger.info("Loading deprecated plugin(s): %s" % - (" ".join([x.name for x in depr]))) - - # Find the metadata plugin and set self.metadata - mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata) - if len(mlist) >= 1: - #: The Metadata plugin - self.metadata = mlist[0] - if len(mlist) > 1: - self.logger.error("Multiple Metadata plugins loaded; " - "using %s" % self.metadata) - else: - self.logger.error("No Metadata plugin loaded; " - "failed to instantiate Core") - raise CoreInitError("No Metadata Plugin") - - #: The list of plugins that handle - #: :class:`Bcfg2.Server.Plugin.interfaces.Statistics` - self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics) - - #: The list of plugins that implement the - #: :class:`Bcfg2.Server.Plugin.interfaces.PullSource` - #: interface - self.pull_sources = \ - self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) - - #: The list of - #: :class:`Bcfg2.Server.Plugin.interfaces.Generator` plugins - self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator) - - #: The list of plugins that handle - #: :class:`Bcfg2.Server.Plugin.interfaces.Structure` - #: generation - self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure) - - #: The list of plugins that implement the - #: :class:`Bcfg2.Server.Plugin.interfaces.Connector` interface - self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector) - #: The CA that signed the server cert self.ca = setup['ca'] @@ -317,6 +257,12 @@ class BaseCore(object): threading.Thread(name="%sFAMThread" % setup['filemonitor'], target=self._file_monitor_thread) + self.perflog_thread = None + if self.setup['perflog']: + self.perflog_thread = \ + threading.Thread(name="PerformanceLoggingThread", + target=self._perflog_thread) + #: A :func:`threading.Lock` for use by #: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set` self.lock = threading.Lock() @@ -325,10 +271,6 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() - if self.debug_flag: - # enable debugging on everything else. - self.plugins[plugin].set_debug(self.debug_flag) - def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -349,11 +291,23 @@ class BaseCore(object): if isinstance(plugin, base_cls)], key=lambda p: (p.sort_order, p.name)) + def _perflog_thread(self): + """ The thread that periodically logs performance statistics + to syslog. """ + self.logger.debug("Performance logging thread starting") + while not self.terminate.isSet(): + self.terminate.wait(self.setup['perflog_interval']) + for name, stats in self.get_statistics(None).items(): + self.logger.info("Performance statistics: " + "%s min=%.06f, max=%.06f, average=%.06f, " + "count=%d" % ((name, ) + stats)) + def _file_monitor_thread(self): """ The thread that runs the :class:`Bcfg2.Server.FileMonitor.FileMonitor`. This also queries :class:`Bcfg2.Server.Plugin.interfaces.Version` plugins for the current revision of the Bcfg2 repo. """ + self.logger.debug("File monitor thread starting") famfd = self.fam.fileno() terminate = self.terminate while not terminate.isSet(): @@ -372,7 +326,7 @@ class BaseCore(object): def _update_vcs_revision(self): """ Update the revision of the current configuration on-disk from the VCS plugin """ - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version): + for plugin in self.plugins_by_type(Version): try: newrev = plugin.get_revision() if newrev != self.revision: @@ -384,6 +338,58 @@ class BaseCore(object): (plugin.name, sys.exc_info()[1])) self.revision = '-1' + def load_plugins(self): + """ Load all plugins, setting + :attr:`Bcfg2.Server.Core.BaseCore.plugins` and + :attr:`Bcfg2.Server.Core.BaseCore.metadata` as side effects. + This does not start plugin threads; that is done later, in + :func:`Bcfg2.Server.Core.BaseCore.run` """ + while '' in self.setup['plugins']: + self.setup['plugins'].remove('') + + for plugin in self.setup['plugins']: + if not plugin in self.plugins: + self.init_plugin(plugin) + + # Remove blacklisted plugins + for plugin, blacklist in list(self.plugin_blacklist.items()): + if len(blacklist) > 0: + self.logger.error("The following plugins conflict with %s;" + "Unloading %s" % (plugin, blacklist)) + for plug in blacklist: + del self.plugins[plug] + + # Log experimental plugins + expl = [plug for plug in list(self.plugins.values()) + if plug.experimental] + if expl: + self.logger.info("Loading experimental plugin(s): %s" % + (" ".join([x.name for x in expl]))) + self.logger.info("NOTE: Interfaces subject to change") + + # Log deprecated plugins + depr = [plug for plug in list(self.plugins.values()) + if plug.deprecated] + if depr: + self.logger.info("Loading deprecated plugin(s): %s" % + (" ".join([x.name for x in depr]))) + + # Find the metadata plugin and set self.metadata + mlist = self.plugins_by_type(Metadata) + if len(mlist) >= 1: + self.metadata = mlist[0] + if len(mlist) > 1: + self.logger.error("Multiple Metadata plugins loaded; using %s" + % self.metadata) + else: + self.logger.error("No Metadata plugin loaded; " + "failed to instantiate Core") + raise CoreInitError("No Metadata Plugin") + + if self.debug_flag: + # enable debugging on plugins + self.plugins[plugin].set_debug(self.debug_flag) + def init_plugin(self, plugin): """ Import and instantiate a single plugin. The plugin is stored to :attr:`plugins`. @@ -468,8 +474,7 @@ class BaseCore(object): metadata.hostname)) start = time.time() try: - for plugin in \ - self.plugins_by_type(Bcfg2.Server.Plugin.ClientRunHooks): + for plugin in self.plugins_by_type(ClientRunHooks): try: getattr(plugin, hook)(metadata) except AttributeError: @@ -500,11 +505,10 @@ class BaseCore(object): :type data: list of lxml.etree._Element objects """ self.logger.debug("Validating structures for %s" % metadata.hostname) - for plugin in \ - self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): + for plugin in self.plugins_by_type(StructureValidator): try: plugin.validate_structures(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: + except ValidationError: err = sys.exc_info()[1] self.logger.error("Plugin %s structure validation failed: %s" % (plugin.name, err)) @@ -527,10 +531,10 @@ class BaseCore(object): :type data: list of lxml.etree._Element objects """ self.logger.debug("Validating goals for %s" % metadata.hostname) - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): + for plugin in self.plugins_by_type(GoalValidator): try: plugin.validate_goals(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: + except ValidationError: err = sys.exc_info()[1] self.logger.error("Plugin %s goal validation failed: %s" % (plugin.name, err.message)) @@ -548,8 +552,9 @@ class BaseCore(object): :returns: list of :class:`lxml.etree._Element` objects """ self.logger.debug("Getting structures for %s" % metadata.hostname) - structures = list(chain(*[struct.BuildStructures(metadata) - for struct in self.structures])) + structures = list( + chain(*[struct.BuildStructures(metadata) + for struct in self.plugins_by_type(Structure)])) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] missing = [b for b in metadata.bundles if b not in sbundles] if missing: @@ -634,8 +639,9 @@ class BaseCore(object): self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) - glist = [gen for gen in self.generators if - entry.get('name') in gen.Entries.get(entry.tag, {})] + generators = self.plugins_by_type(Generator) + glist = [gen for gen in generators + if entry.get('name') in gen.Entries.get(entry.tag, {})] if len(glist) == 1: return glist[0].Entries[entry.tag][entry.get('name')](entry, metadata) @@ -643,8 +649,8 @@ class BaseCore(object): generators = ", ".join([gen.name for gen in glist]) self.logger.error("%s %s served by multiple generators: %s" % (entry.tag, entry.get('name'), generators)) - g2list = [gen for gen in self.generators if - gen.HandlesEntry(entry, metadata)] + g2list = [gen for gen in generators + if gen.HandlesEntry(entry, metadata)] try: if len(g2list) == 1: return g2list[0].HandleEntry(entry, metadata) @@ -671,7 +677,7 @@ class BaseCore(object): revision=self.revision) try: meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: self.logger.error("Metadata consistency error for client %s" % client) return lxml.etree.Element("error", type='metadata error') @@ -718,7 +724,8 @@ class BaseCore(object): :type event: Bcfg2.Server.FileMonitor.Event """ if event.filename != self.cfile: - print("Got event for unknown file: %s" % event.filename) + self.logger.error("Got event for unknown file: %s" % + event.filename) return if event.code2str() == 'deleted': return @@ -755,16 +762,25 @@ class BaseCore(object): return False try: + self.load_plugins() + self.fam.start() self.fam_thread.start() self.fam.AddMonitor(self.cfile, self) + if self.perflog_thread is not None: + self.perflog_thread.start() - for plug in self.plugins_by_type(Bcfg2.Server.Plugin.Threaded): + for plug in self.plugins_by_type(Threaded): plug.start_threads() except: self.shutdown() raise + if self.setup['fam_blocking']: + time.sleep(1) + while self.fam.pending() != 0: + time.sleep(1) + self.set_debug(None, self.debug_flag) self._block() @@ -796,7 +812,7 @@ class BaseCore(object): """ self.logger.debug("Getting decision list for %s" % metadata.hostname) result = [] - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): + for plugin in self.plugins_by_type(Decision): try: result.extend(plugin.GetDecisions(metadata, mode)) except: @@ -815,7 +831,7 @@ class BaseCore(object): """ if not hasattr(self, 'metadata'): # some threads start before metadata is even loaded - raise Bcfg2.Server.Plugin.MetadataRuntimeError + raise MetadataRuntimeError("Metadata not loaded yet") if self.metadata_cache_mode == 'initial': # the Metadata plugin handles loading the cached data if # we're only caching the initial metadata object @@ -825,10 +841,11 @@ class BaseCore(object): if not imd: self.logger.debug("Building metadata for %s" % client_name) imd = self.metadata.get_initial_metadata(client_name) - for conn in self.connectors: + connectors = self.plugins_by_type(Connector) + for conn in connectors: grps = conn.get_additional_groups(imd) self.metadata.merge_additional_groups(imd, grps) - for conn in self.connectors: + for conn in connectors: data = conn.get_additional_data(imd) self.metadata.merge_additional_data(imd, conn.name, data) imd.query.by_name = self.build_metadata @@ -849,7 +866,7 @@ class BaseCore(object): meta = self.build_metadata(client_name) state = statistics.find(".//Statistics") if state.get('version') >= '2.0': - for plugin in self.statistics: + for plugin in self.plugins_by_type(Statistics): try: plugin.process_statistics(meta, statistics) except: @@ -891,11 +908,11 @@ class BaseCore(object): meta = self.build_metadata(client) else: meta = None - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: err = sys.exc_info()[1] self.critical_error("Client metadata resolution error for %s: %s" % (address[0], err)) - except Bcfg2.Server.Plugin.MetadataRuntimeError: + except MetadataRuntimeError: err = sys.exc_info()[1] self.critical_error('Metadata system runtime failure for %s: %s' % (address[0], err)) @@ -989,8 +1006,7 @@ class BaseCore(object): version)) try: self.metadata.set_version(client, version) - except (Bcfg2.Server.Plugin.MetadataConsistencyError, - Bcfg2.Server.Plugin.MetadataRuntimeError): + except (MetadataConsistencyError, MetadataRuntimeError): err = sys.exc_info()[1] self.critical_error("Unable to set version for %s: %s" % (client, err)) @@ -1010,7 +1026,7 @@ class BaseCore(object): client, metadata = self.resolve_client(address, cleanup_cache=True) self.logger.debug("Getting probes for %s" % client) try: - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): + for plugin in self.plugins_by_type(Probing): for probe in plugin.GetProbes(metadata): resp.append(probe) return lxml.etree.tostring(resp, @@ -1080,8 +1096,7 @@ class BaseCore(object): self.logger.debug("%s sets its profile to %s" % (client, profile)) try: self.metadata.set_profile(client, profile, address) - except (Bcfg2.Server.Plugin.MetadataConsistencyError, - Bcfg2.Server.Plugin.MetadataRuntimeError): + except (MetadataConsistencyError, MetadataRuntimeError): err = sys.exc_info()[1] self.critical_error("Unable to assert profile for %s: %s" % (client, err)) @@ -1103,7 +1118,7 @@ class BaseCore(object): config = self.BuildConfiguration(client) return lxml.etree.tostring(config, xml_declaration=False).decode('UTF-8') - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: self.critical_error("Metadata consistency failure for %s" % client) @exposed diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 37bc230d1..ae7c75804 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -40,7 +40,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", "SSLCA/**/cert.xml": "sslca-cert.xsd", - "SSLCA/**/key.xml": "sslca-key.xsd" + "SSLCA/**/key.xml": "sslca-key.xsd", + "GroupLogic/groups.xml": "grouplogic.xsd" } self.filelists = {} @@ -83,17 +84,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): else: self.LintError("properties-schema-not-found", "No schema found for %s" % filename) + # ensure that it at least parses + self.parse(filename) - def validate(self, filename, schemafile, schema=None): - """validate a file against the given lxml.etree.Schema. - return True on success, False on failure """ - if schema is None: - # if no schema object was provided, instantiate one - schema = self._load_schema(schemafile) - if not schema: - return False + def parse(self, filename): + """ Parse an XML file, raising the appropriate LintErrors if + it can't be parsed or read. Return the + lxml.etree._ElementTree parsed from the file. """ try: - datafile = lxml.etree.parse(filename) + return lxml.etree.parse(filename) except SyntaxError: lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) self.LintError("xml-failed-to-parse", @@ -106,6 +105,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Failed to open file %s" % filename) return False + def validate(self, filename, schemafile, schema=None): + """validate a file against the given lxml.etree.Schema. + return True on success, False on failure """ + if schema is None: + # if no schema object was provided, instantiate one + schema = self._load_schema(schemafile) + if not schema: + return False + datafile = self.parse(filename) if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 2b878d7e2..b14968d77 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -530,8 +530,8 @@ class XMLFileBacked(FileBacked): #: XInclude. self.extra_monitors = [] - if ((create or (self.create is not None and self.create)) - and not os.path.exists(self.name)): + if ((create is not None or self.create not in [None, False]) and + not os.path.exists(self.name)): toptag = create or self.create self.logger.warning("%s does not exist, creating" % self.name) if hasattr(toptag, "getroottree"): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index 581a997d8..c7b62f352 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -69,7 +69,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): the given client metadata, and may be obtained by doing ``self.XMLMatch(metadata)`` :type spec: lxml.etree._Element - :returns: None + :returns: string - The filename of the private key """ if spec is None: spec = self.XMLMatch(metadata) @@ -140,7 +140,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if spec is None: spec = self.XMLMatch(metadata) category = spec.get("category", self.category) - print("category=%s" % category) if category is None: per_host_default = "true" else: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 926172e57..ffe93c25b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -599,6 +599,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, else: try: if not isinstance(data, unicode): + if not isinstance(data, str): + data = data.decode('utf-8') data = u_str(data, self.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index c8362db41..44971aba7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -44,7 +44,7 @@ class Git(Version): else: cmd = ["git", "--git-dir", self.vcs_path, "--work-tree", self.vcs_root, "rev-parse", "HEAD"] - self.debug_log("Git: Running cmd") + self.debug_log("Git: Running %s" % cmd) proc = Popen(cmd, stdout=PIPE, stderr=PIPE) rv, err = proc.communicate() if proc.wait(): diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py new file mode 100644 index 000000000..810b273af --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -0,0 +1,47 @@ +""" GroupLogic is a connector plugin that lets you use an XML Genshi +template to dynamically set additional groups for clients. """ + +import os +import lxml.etree +import Bcfg2.Server.Plugin +try: + from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile +except ImportError: + # BundleTemplateFile missing means that genshi is missing. we + # import genshi to get the _real_ error + import genshi # pylint: disable=W0611 + + +class GroupLogicConfig(BundleTemplateFile): + """ Representation of the GroupLogic groups.xml file """ + create = lxml.etree.Element("GroupLogic", + nsmap=dict(py="http://genshi.edgewall.org/")) + + def __init__(self, name, fam): + BundleTemplateFile.__init__(self, name, + Bcfg2.Server.Plugin.Specificity(), None) + self.fam = fam + self.should_monitor = True + self.fam.AddMonitor(self.name, self) + + def _match(self, item, metadata): + if item.tag == 'Group' and not len(item.getchildren()): + return [item] + return BundleTemplateFile._match(self, item, metadata) + + +class GroupLogic(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + """ GroupLogic is a connector plugin that lets you use an XML + Genshi template to dynamically set additional groups for + clients. """ + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), + core.fam) + + def get_additional_groups(self, metadata): + return [el.get("name") + for el in self.config.get_xml_value(metadata).findall("Group")] diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index bdf3b87fe..71e81c1fe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -945,7 +945,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Client %s set as nonexistent group %s" % (client, group)) - def set_profile(self, client, profile, addresspair): + def set_profile(self, client, profile, # pylint: disable=W0221 + addresspair, require_public=True): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) @@ -957,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] - if not group.is_public: + if require_public and not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) @@ -1128,7 +1129,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = self.default if pgroup: - self.set_profile(client, pgroup, (None, None)) + self.set_profile(client, pgroup, (None, None), + require_public=False) profile = _add_group(pgroup) else: msg = "Cannot add new client %s; no default group set" % client diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 4eefd0722..bc2928fa6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -93,6 +93,8 @@ class AptSource(Source): self.logger.error("Packages: Failed to read file %s" % fname) raise for line in reader.readlines(): + if not isinstance(line, str): + line = line.decode('utf-8') words = str(line.strip()).split(':', 1) if words[0] == 'Package': pkgname = words[1].strip().rstrip() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index b4d481459..7ba374dd3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -315,7 +315,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :raises: OSError - If the saved data cannot be read :raises: cPickle.UnpicklingError - If the saved data is corrupt """ - data = open(self.cachefile) + data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, self.essentialpkgs) = cPickle.load(data) @@ -615,7 +615,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 self.logger.info("Packages: Updating %s" % url) fname = self.escape_url(url) try: - open(fname, 'w').write(fetch_url(url)) + open(fname, 'wb').write(fetch_url(url)) except ValueError: self.logger.error("Packages: Bad url string %s" % url) raise diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 4f163a1ab..efbca28cd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -18,7 +18,8 @@ from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" #: The default path for generated apt configs -APT_CONFIG_DEFAULT = "/etc/apt/sources.d/bcfg2" +APT_CONFIG_DEFAULT = \ + "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" class Packages(Bcfg2.Server.Plugin.Plugin, diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index a8001d9af..309b96475 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -63,7 +63,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data) + return str.__new__(cls, data.encode('utf-8')) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -153,7 +153,20 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): probe = lxml.etree.Element('probe') probe.set('name', os.path.basename(name)) probe.set('source', self.plugin_name) - probe.text = entry.data + if (metadata.version_info and + metadata.version_info > (1, 3, 1, '', 0)): + try: + probe.text = entry.data.decode('utf-8') + except AttributeError: + probe.text = entry.data + else: + try: + probe.text = entry.data + except: # pylint: disable=W0702 + self.logger.error("Client unable to handle unicode " + "probes. Skipping %s" % + probe.get('name')) + continue match = self.bangline.match(entry.data.split('\n')[0]) if match: probe.set('interpreter', match.group('interpreter')) @@ -209,8 +222,15 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - lxml.etree.SubElement(ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str( + self.probedata[client][probe]).decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 3bd6fd14f..3354763d4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -96,7 +96,7 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): client.hostname, cdata, lxml.etree.tostring( stats, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False)) self.debug_log("%s: Queued statistics data for %s" % (self.__class__.__name__, client.hostname)) return diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index fc07a90e9..5aa7c4d9e 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -172,7 +172,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for name in names[cmeta.hostname]: newnames.add(name.split('.')[0]) try: - newips.add(self.get_ipcache_entry(name)[0]) + newips.update(self.get_ipcache_entry(name)[0]) except: # pylint: disable=W0702 continue names[cmeta.hostname].update(newnames) @@ -288,7 +288,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: # need to add entry try: - ipaddr = socket.gethostbyname(client) + ipaddr = set([addr[0] for (_, _, _, _, addr) in socket.getaddrinfo(client, None)]) self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index 7d00201da..f111ffc60 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -68,7 +68,7 @@ class SSLCACertSpec(SSLCAXMLSpec): def get_spec(self, metadata): rv = SSLCAXMLSpec.get_spec(self, metadata) rv['subjectaltname'] = [e.text for e in self.Match(metadata) - if e.tag == "SubjectAltName"] + if e.tag == "subjectAltName"] return rv diff --git a/src/lib/Bcfg2/Statistics.py b/src/lib/Bcfg2/Statistics.py index a869b03cd..3825941af 100644 --- a/src/lib/Bcfg2/Statistics.py +++ b/src/lib/Bcfg2/Statistics.py @@ -28,10 +28,10 @@ class Statistic(object): :param value: The value to add to this statistic :type value: int or float """ - self.min = min(self.min, value) - self.max = max(self.max, value) - self.ave = (((self.ave * (self.count - 1)) + value) / self.count) + self.min = min(self.min, float(value)) + self.max = max(self.max, float(value)) self.count += 1 + self.ave = (((self.ave * (self.count - 1)) + value) / self.count) def get_value(self): """ Get a tuple of all the stats tracked on this named item. @@ -46,6 +46,11 @@ class Statistic(object): """ return (self.name, (self.min, self.max, self.ave, self.count)) + def __repr__(self): + return "%s(%s, (min=%s, avg=%s, max=%s, count=%s))" % ( + self.__class__.__name__, + self.name, self.min, self.ave, self.max, self.count) + class Statistics(object): """ A collection of named :class:`Statistic` objects. """ diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 39cf5255e..581445bf4 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -108,10 +108,16 @@ class ExecutorResult(object): def __init__(self, stdout, stderr, retval): #: The output of the command - self.stdout = stdout + if isinstance(stdout, str): + self.stdout = stdout + else: + self.stdout = stdout.decode('utf-8') #: The error produced by the command - self.stderr = stderr + if isinstance(stdout, str): + self.stderr = stderr + else: + self.stderr = stderr.decode('utf-8') #: The return value of the command. self.retval = retval @@ -234,6 +240,13 @@ class Executor(object): for line in inputdata.splitlines(): self.logger.debug('> %s' % line) (stdout, stderr) = proc.communicate(input=inputdata) + + # py3k fixes + if not isinstance(stdout, str): + stdout = stdout.decode('utf-8') + if not isinstance(stderr, str): + stderr = stderr.decode('utf-8') + for line in stdout.splitlines(): # pylint: disable=E1103 self.logger.debug('< %s' % line) for line in stderr.splitlines(): # pylint: disable=E1103 diff --git a/src/lib/Bcfg2/__init__.py b/src/lib/Bcfg2/__init__.py index 41743d100..74a871f2a 100644 --- a/src/lib/Bcfg2/__init__.py +++ b/src/lib/Bcfg2/__init__.py @@ -1,4 +1 @@ """Base modules definition.""" - -from Bcfg2.Compat import walk_packages -__all__ = [m[1] for m in walk_packages(path=__path__)] diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 7d405f868..9393830a8 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -54,11 +54,11 @@ DEFAULT_CONFIG = _default_config() def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): """ read the config file and set django settings based on it """ - # pylint: disable=W0603 + # pylint: disable=W0602,W0603 global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ DATABASE_HOST, DATABASE_PORT, DEBUG, TEMPLATE_DEBUG, TIME_ZONE, \ MEDIA_URL - # pylint: enable=W0603 + # pylint: enable=W0602,W0603 if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): print("%s does not exist, using %s for database configuration" % diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 462d41398..ac4c3af13 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -120,7 +120,6 @@ class InfoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) self.prompt = '> ' self.cont = True - self.fam.handle_events_in_interval(4) def _get_client_list(self, hostglobs): """ given a host glob, get a list of clients that match it """ @@ -458,9 +457,7 @@ Bcfg2 client itself.""") def do_clients(self, _): """ clients - Print out client/profile info """ data = [('Client', 'Profile')] - clist = self.metadata.clients - clist.sort() - for client in clist: + for client in sorted(self.metadata.list_clients()): imd = self.metadata.get_initial_metadata(client) data.append((client, imd.profile)) print_tabular(data) @@ -606,7 +603,7 @@ Bcfg2 client itself.""") # Dump all mappings unless type specified data = [('Plugin', 'Type', 'Name')] arglen = len(args.split()) - for generator in self.generators: + for generator in self.plugins_by_type(Bcfg2.Server.Plugin.Generator): if arglen == 0: etypes = list(generator.Entries.keys()) else: @@ -712,6 +709,8 @@ Bcfg2 client itself.""") def run(self, args): # pylint: disable=W0221 try: + self.load_plugins() + self.fam.handle_events_in_interval(1) if args: self.onecmd(" ".join(args)) else: @@ -753,8 +752,7 @@ def main(): optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, interactive=Bcfg2.Options.INTERACTIVE, interpreter=Bcfg2.Options.INTERPRETER) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) setup.hm = "\n".join([" bcfg2-info [options] [command <command args>]", "Options:", diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 2c4a918be..bb45e0009 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -233,7 +233,8 @@ def main(): try: entries = [l.strip().split(":") for l in open(options.file)] - except IOError, err: + except IOError: + err = sys.exc_info()[1] print("Cannot read entries from %s: %s" % (options.file, err)) return 2 diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 6eaf0cc33..c33143a04 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -155,6 +155,7 @@ class ClientTest(TestCase): def get_core(setup): """ Get a server core, with events handled """ core = Bcfg2.Server.Core.BaseCore(setup) + core.load_plugins() core.fam.handle_events_in_interval(0.1) return core diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 7e5c03fd5..7dbdad16b 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -9,33 +9,13 @@ import os import sys import yum import logging +import Bcfg2.Logger from optparse import OptionParser try: import json except ImportError: import simplejson as json -LOGGER = None - - -def get_logger(verbose=0): - """ set up logging according to the verbose level given on the - command line """ - global LOGGER - if LOGGER is None: - LOGGER = logging.getLogger(sys.argv[0]) - stderr = logging.StreamHandler() - if verbose: - level = logging.DEBUG - else: - level = logging.WARNING - LOGGER.setLevel(level) - LOGGER.addHandler(stderr) - syslog = logging.handlers.SysLogHandler("/dev/log") - syslog.setFormatter(logging.Formatter("%(name)s: %(message)s")) - LOGGER.addHandler(syslog) - return LOGGER - def pkg_to_tuple(package): """ json doesn't distinguish between tuples and lists, but yum @@ -76,7 +56,7 @@ class DepSolver(object): except AttributeError: self.yumbase._getConfig(cfgfile, debuglevel=verbose) # pylint: enable=E1121,W0212 - self.logger = get_logger(verbose) + self.logger = logging.getLogger(self.__class__.__name__) self._groups = None def get_groups(self): @@ -220,7 +200,17 @@ def main(): parser.add_option("-v", "--verbose", help="Verbosity level", action="count") (options, args) = parser.parse_args() - logger = get_logger(options.verbose) + + if options.verbose: + level = logging.DEBUG + clevel = logging.DEBUG + else: + level = logging.WARNING + clevel = logging.INFO + Bcfg2.Logger.setup_logging('bcfg2-yum-helper', to_syslog=True, + to_console=clevel, level=level) + logger = logging.getLogger('bcfg2-yum-helper') + try: cmd = args[0] except IndexError: diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py index 662e0e1b6..8f933e08f 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py @@ -64,10 +64,18 @@ class TestPOSIXFile(TestPOSIXTool): self.assertEqual(ptool._get_data(entry), ("test", True)) entry = copy.deepcopy(orig_entry) + entry.set("encoding", "base64") + entry.set("empty", "true") + self.assertEqual(ptool._get_data(entry), ("", True)) + + entry = copy.deepcopy(orig_entry) entry.set("empty", "true") self.assertEqual(ptool._get_data(entry), ("", False)) entry = copy.deepcopy(orig_entry) + self.assertEqual(ptool._get_data(entry), ("", False)) + + entry = copy.deepcopy(orig_entry) entry.text = "test" self.assertEqual(ptool._get_data(entry), ("test", False)) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index 58e61e13b..94866cf39 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -410,11 +410,12 @@ class TestXMLFileBacked(TestFileBacked): should_monitor = None path = os.path.join(datastore, "test", "test1.xml") - @patch("os.makedirs", Mock()) def get_obj(self, path=None, fam=None, should_monitor=False): if path is None: path = self.path - @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock()) + + @patchIf(not isinstance(os.path.exists, Mock), + "os.path.exists", Mock()) def inner(): return self.test_obj(path, fam=fam, should_monitor=should_monitor) return inner() @@ -422,20 +423,16 @@ class TestXMLFileBacked(TestFileBacked): def test__init(self): fam = Mock() xfb = self.get_obj() - if self.should_monitor is True: + if self.should_monitor: self.assertIsNotNone(xfb.fam) + fam.reset_mock() + xfb = self.get_obj(fam=fam, should_monitor=True) + fam.AddMonitor.assert_called_with(self.path, xfb) else: self.assertIsNone(xfb.fam) - - if self.should_monitor is not True: xfb = self.get_obj(fam=fam) self.assertFalse(fam.AddMonitor.called) - if self.should_monitor is not False: - fam.reset_mock() - xfb = self.get_obj(fam=fam, should_monitor=True) - fam.AddMonitor.assert_called_with(self.path, xfb) - @patch("glob.glob") @patch("lxml.etree.parse") def test_follow_xincludes(self, mock_parse, mock_glob): @@ -623,7 +620,7 @@ class TestXMLFileBacked(TestFileBacked): def test_add_monitor(self): xfb = self.get_obj() xfb.add_monitor("/test/test2.xml") - self.assertIn("/test/test2.xml", xfb.extras) + self.assertIn("/test/test2.xml", xfb.extra_monitors) fam = Mock() if self.should_monitor is not True: @@ -632,14 +629,14 @@ class TestXMLFileBacked(TestFileBacked): fam.reset_mock() xfb.add_monitor("/test/test3.xml") self.assertFalse(fam.AddMonitor.called) - self.assertIn("/test/test3.xml", xfb.extras) + self.assertIn("/test/test3.xml", xfb.extra_monitors) if self.should_monitor is not False: fam.reset_mock() xfb = self.get_obj(fam=fam, should_monitor=True) xfb.add_monitor("/test/test4.xml") fam.AddMonitor.assert_called_with("/test/test4.xml", xfb) - self.assertIn("/test/test4.xml", xfb.extras) + self.assertIn("/test/test4.xml", xfb.extra_monitors) class TestStructFile(TestXMLFileBacked): @@ -2036,6 +2033,3 @@ class TestGroupSpool(TestPlugin, TestGenerator): gs.event_id.assert_called_with(event) self.assertNotIn("/baz/quux", gs.entries) self.assertNotIn("/baz/quux", gs.Entries[gs.entry_type]) - - - diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index 1022bdc5a..0794db62e 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -3,6 +3,7 @@ import sys import copy import time import lxml.etree +import Bcfg2.version import Bcfg2.Server import Bcfg2.Server.Plugin from mock import Mock, MagicMock, patch @@ -217,6 +218,8 @@ group-specific""" ps.get_matching.return_value = matching metadata = Mock() + metadata.version_info = \ + Bcfg2.version.Bcfg2VersionInfo(Bcfg2.version.__version__) pdata = ps.get_probe_data(metadata) ps.get_matching.assert_called_with(metadata) # we can't create a matching operator.attrgetter object, and I @@ -621,5 +624,3 @@ text metadata.hostname = "nonexistent" self.assertEqual(probes.get_additional_data(metadata), ClientProbeDataSet()) - - diff --git a/testsuite/Testsrc/Testlib/TestStatistics.py b/testsuite/Testsrc/Testlib/TestStatistics.py new file mode 100644 index 000000000..496cbac28 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestStatistics.py @@ -0,0 +1,44 @@ +import os +import sys +from mock import Mock, MagicMock, patch + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * + +from Bcfg2.Statistics import * + + +class TestStatistic(Bcfg2TestCase): + def test_stat(self): + stat = Statistic("test", 1) + self.assertEqual(stat.get_value(), ("test", (1.0, 1.0, 1.0, 1))) + stat.add_value(10) + self.assertEqual(stat.get_value(), ("test", (1.0, 10.0, 5.5, 2))) + stat.add_value(100) + self.assertEqual(stat.get_value(), ("test", (1.0, 100.0, 37.0, 3))) + stat.add_value(12.345) + self.assertEqual(stat.get_value(), ("test", (1.0, 100.0, 30.83625, 4))) + stat.add_value(0.655) + self.assertEqual(stat.get_value(), ("test", (0.655, 100.0, 24.8, 5))) + + +class TestStatistics(Bcfg2TestCase): + def test_stats(self): + stats = Statistics() + self.assertEqual(stats.display(), dict()) + stats.add_value("test1", 1) + self.assertEqual(stats.display(), dict(test1=(1.0, 1.0, 1.0, 1))) + stats.add_value("test2", 1.23) + self.assertEqual(stats.display(), dict(test1=(1.0, 1.0, 1.0, 1), + test2=(1.23, 1.23, 1.23, 1))) + stats.add_value("test1", 10) + self.assertEqual(stats.display(), dict(test1=(1.0, 10.0, 5.5, 2), + test2=(1.23, 1.23, 1.23, 1))) diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py index 3cd3786f9..93314f1e3 100755 --- a/tools/bcfg2-profile-templates.py +++ b/tools/bcfg2-profile-templates.py @@ -1,25 +1,35 @@ #!/usr/bin/python -Ott +# -*- coding: utf-8 -*- """ Benchmark template rendering times """ -import os import sys import time +import math import logging import operator import Bcfg2.Logger +import Bcfg2.Options import Bcfg2.Server.Core -LOGGER = None + +def stdev(nums): + mean = float(sum(nums)) / len(nums) + return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums))) def main(): - optinfo = \ - dict(client=Bcfg2.Options.Option("Benchmark templates for one client", - cmd="--client", - odesc="<client>", - long_arg=True, - default=None), - ) + optinfo = dict( + client=Bcfg2.Options.Option("Benchmark templates for one client", + cmd="--client", + odesc="<client>", + long_arg=True, + default=None), + runs=Bcfg2.Options.Option("Number of rendering passes per template", + cmd="--runs", + odesc="<runs>", + long_arg=True, + default=5, + cook=int)) optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) @@ -40,12 +50,11 @@ def main(): core = Bcfg2.Server.Core.BaseCore(setup) logger.info("Bcfg2 server core loaded") + core.load_plugins() + logger.debug("Plugins loaded") core.fam.handle_events_in_interval(0.1) logger.debug("Repository events processed") - # how many times to render each template for each client - runs = 5 - if setup['args']: templates = setup['args'] else: @@ -57,41 +66,57 @@ def main(): clients = [core.build_metadata(setup['client'])] times = dict() + client_count = 0 for metadata in clients: - for struct in core.GetStructures(metadata): - logger.info("Rendering templates from structure %s:%s" % - (struct.tag, struct.get("name"))) - for entry in struct.xpath("//Path"): - path = entry.get("name") - logger.info("Rendering %s..." % path) - times[path] = dict() - avg = 0.0 - for i in range(runs): + client_count += 1 + logger.info("Rendering templates for client %s (%s/%s)" % + (metadata.hostname, client_count, len(clients))) + structs = core.GetStructures(metadata) + struct_count = 0 + for struct in structs: + struct_count += 1 + logger.info("Rendering templates from structure %s:%s (%s/%s)" % + (struct.tag, struct.get("name"), struct_count, + len(structs))) + entries = struct.xpath("//Path") + entry_count = 0 + for entry in entries: + entry_count += 1 + if templates and entry.get("name") not in templates: + continue + logger.info("Rendering Path:%s (%s/%s)..." % + (entry.get("name"), entry_count, len(entries))) + ptimes = times.setdefault(entry.get("name"), []) + for i in range(setup['runs']): start = time.time() try: core.Bind(entry, metadata) - avg += (time.time() - start) / runs + ptimes.append(time.time() - start) except: break - if avg: - logger.debug(" %s: %.02f sec" % (metadata.hostname, avg)) - times[path][metadata.hostname] = avg + if ptimes: + avg = sum(ptimes) / len(ptimes) + if avg: + logger.debug(" %s: %.02f sec" % + (metadata.hostname, avg)) # print out per-file results tmpltimes = [] - for tmpl, clients in times.items(): + for tmpl, ptimes in times.items(): try: - avg = sum(clients.values()) / len(clients) + mean = float(sum(ptimes)) / len(ptimes) except ZeroDivisionError: continue - if avg > 0.01 or templates: - tmpltimes.append((tmpl, avg)) - print("%-50s %s" % ("Template", "Average Render Time")) - for tmpl, avg in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): - print("%-50s %.02f" % (tmpl, avg)) - - # TODO: complain about templates that on average were quick but - # for which some clients were slow + ptimes.sort() + median = ptimes[len(ptimes) / 2] + std = stdev(ptimes) + if mean > 0.01 or median > 0.01 or std > 1 or templates: + tmpltimes.append((tmpl, mean, median, std)) + print("%-50s %-9s %-11s %6s" % + ("Template", "Mean Time", "Median Time", "σ")) + for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): + print("%-50s %9.02f %11.02f %6.02f" % info) + core.shutdown() if __name__ == "__main__": diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index 2b9d39342..edb5a7101 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -64,6 +64,12 @@ class LocalClient(Client): def main(): optinfo = Bcfg2.Options.CLIENT_COMMON_OPTIONS optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + if 'bundle_quick' in optinfo: + # CLIENT_BUNDLEQUICK option uses -Q, just like the server repo + # option. the server repo is more important for this + # application. + optinfo['bundle_quick'] = Bcfg2.Options.Option('bundlequick', + default=False) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py index b6997bb29..507a16f43 100755 --- a/tools/selinux_baseline.py +++ b/tools/selinux_baseline.py @@ -42,7 +42,10 @@ def main(): baseline.append(lxml.etree.Comment("%s entries" % etype)) extra = handler.FindExtra() for entry in extra: - entry.tag = "BoundSELinux" + if etype != "SEModule": + entry.tag = "Bound%s" % etype + else: + entry.tag = "%s" % etype baseline.extend(extra) print(lxml.etree.tostring(baseline, pretty_print=True)) diff --git a/tools/upgrade/1.3/migrate_perms_to_mode.py b/tools/upgrade/1.3/migrate_perms_to_mode.py index e061558d3..18abffec2 100755 --- a/tools/upgrade/1.3/migrate_perms_to_mode.py +++ b/tools/upgrade/1.3/migrate_perms_to_mode.py @@ -13,6 +13,7 @@ def setmodeattr(elem): elem.set('mode', elem.get('perms')) del elem.attrib['perms'] return True + return False def writefile(f, xdata): @@ -32,7 +33,7 @@ def convertinfo(ifile): return found = False for i in xdata.findall('//Info'): - found = setmodeattr(i) + found |= setmodeattr(i) if found: writefile(ifile, xdata) @@ -47,7 +48,7 @@ def convertstructure(structfile): return found = False for path in xdata.xpath('//BoundPath|//Path'): - found = setmodeattr(path) + found |= setmodeattr(path) if found: writefile(structfile, xdata) |