diff options
103 files changed, 1840 insertions, 655 deletions
@@ -1,3 +1,7 @@ +This file contains a list of copyright holders. Anyone who +contributes more than trivial fixes (typos, etc.) to Bcfg2 should also +add themselves to this file. See LICENSE for the full license. + - Narayan Desai <desai@mcs.anl.gov> has written most of Bcfg2, including all parts not explicitly mentioned in this file. @@ -152,3 +156,6 @@ - Zach Lowry <zach@mcs.anl.gov> wrote Solaris support and general hardening. + +- Michael Fenn <fennm@deshawresearch.com> fixed various small bugs + related to bcfg2 on CentOS 5 diff --git a/debian/bcfg2-server.init b/debian/bcfg2-server.init index 04774c063..b1c3aba21 100755 --- a/debian/bcfg2-server.init +++ b/debian/bcfg2-server.init @@ -17,7 +17,8 @@ ### END INIT INFO # Include lsb functions -. /lib/lsb/init-functions +test -f "/lib/lsb/init-functions" && . /lib/lsb/init-functions # debian +test -f "/etc/init.d/functions" && . /etc/init.d/functions # redhat # Commonly used stuff DAEMON=/usr/sbin/bcfg2-server diff --git a/debian/bcfg2.cron.daily b/debian/bcfg2.cron.daily index f2d1efb9f..b872887cb 100755 --- a/debian/bcfg2.cron.daily +++ b/debian/bcfg2.cron.daily @@ -10,4 +10,4 @@ else echo "No bcfg2-cron command found" exit 1 fi -$BCFG2CRON --daily 2>&1 | logger -t bcfg2-cron -p daemon.info +$BCFG2CRON --daily 2>&1 | logger -t bcfg2-cron -p daemon.info -i diff --git a/debian/bcfg2.cron.hourly b/debian/bcfg2.cron.hourly index 73aae7606..9f666e083 100755 --- a/debian/bcfg2.cron.hourly +++ b/debian/bcfg2.cron.hourly @@ -10,4 +10,4 @@ else echo "No bcfg2-cron command found" exit 1 fi -$BCFG2CRON --hourly 2>&1 | logger -t bcfg2-cron -p daemon.info +$BCFG2CRON --hourly 2>&1 | logger -t bcfg2-cron -p daemon.info -i diff --git a/debian/control b/debian/control index 6c7278e4e..20cef93c8 100644 --- a/debian/control +++ b/debian/control @@ -4,25 +4,24 @@ Priority: optional Maintainer: Arto Jantunen <viiru@debian.org> Uploaders: Sami Haahtinen <ressu@debian.org> Build-Depends: debhelper (>= 7.0.50~), - python (>= 2.3.5-7), + python (>= 2.6), python-setuptools, python-sphinx (>= 1.0.7+dfsg) | python3-sphinx, python-lxml, python-daemon, python-cherrypy, + python-gamin, python-pyinotify, python-m2crypto, python-doc, python-mock-doc Build-Depends-Indep: python-support (>= 0.5.3) Standards-Version: 3.8.0.0 -XS-Python-Version: >= 2.3 Homepage: http://bcfg2.org/ Package: bcfg2 Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, debsums, python-apt, ucf, lsb-base (>= 3.1-9), python-m2crypto | python-ssl | python2.6 | python3.0 | python3.1 | python3.2 -XB-Python-Version: >= 2.3 +Depends: ${python:Depends}, ${misc:Depends}, debsums, python-apt, ucf, lsb-base (>= 3.1-9), python (>= 2.6) Description: Configuration management client Bcfg2 is a configuration management system that generates configuration sets for clients bound by client profiles. @@ -31,8 +30,7 @@ Description: Configuration management client Package: bcfg2-server Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, python-lxml (>= 0.9), libxml2-utils (>= 2.6.23), lsb-base (>= 3.1-9), ucf, bcfg2 (= ${binary:Version}), openssl, python-ssl | python2.6 | python3.0 | python3.1 | python3.2, python-pyinotify | python-gamin, python-daemon -XB-Python-Version: >= 2.4 +Depends: ${python:Depends}, ${misc:Depends}, python-lxml (>= 0.9), libxml2-utils (>= 2.6.23), lsb-base (>= 3.1-9), ucf, bcfg2 (= ${binary:Version}), openssl, python (>= 2.6), python-pyinotify | python-gamin, python-daemon Recommends: graphviz, patch Suggests: python-cheetah, python-genshi (>= 0.4.4), python-profiler, python-sqlalchemy (>= 0.5.0), python-django, mail-transport-agent, bcfg2-doc (= ${binary:Version}) Description: Configuration management server @@ -45,7 +43,6 @@ Package: bcfg2-web Architecture: all Depends: ${python:Depends}, ${misc:Depends}, bcfg2-server (= ${binary:Version}), python-django, Suggests: python-mysqldb, python-psycopg2, python-sqlite, libapache2-mod-wsgi -XB-Python-Version: >= 2.4 Description: Configuration management web interface Bcfg2 is a configuration management system that generates configuration sets for clients bound by client profiles. diff --git a/debian/rules b/debian/rules index fcbf6346c..5694e4e37 100755 --- a/debian/rules +++ b/debian/rules @@ -3,13 +3,6 @@ %: dh $@ --with python-support,sphinxdoc -override_dh_auto_install: - # Make the build destination dir consistent between pre-7.3 and 7.3 and - # later debhelper - see http://bcfg2.org/ticket/791 - dh_auto_install - test -d debian/tmp/usr/local && mv debian/tmp/usr/local/* debian/tmp/usr || exit 0 - test -d debian/tmp/usr/local && rmdir debian/tmp/usr/local || exit 0 - override_dh_installinit: # Install bcfg2 initscript without starting it on postinst dh_installinit --package=bcfg2 --no-start diff --git a/doc/appendix/guides/centos.txt b/doc/appendix/guides/centos.txt index 5a2d1bed0..febdf5769 100644 --- a/doc/appendix/guides/centos.txt +++ b/doc/appendix/guides/centos.txt @@ -185,21 +185,15 @@ line of ``bcfg2.conf``. Then create Packages layout (as per <Sources> <!-- CentOS (5.4) sources --> - <YUMSource> - <Group>centos-5.4</Group> - <RawURL>http://mrepo/centos5-x86_64/RPMS.os</RawURL> + <Source type="yum" rawurl="http://mrepo/centos5-x86_64/RPMS.os"> <Arch>x86_64</Arch> - </YUMSource> - <YUMSource> - <Group>centos-5.4</Group> - <RawURL>http://mrepo/centos5-x86_64/RPMS.updates</RawURL> + </Source> + <Source type="yum" rawurl="http://mrepo/centos5-x86_64/RPMS.updates"> <Arch>x86_64</Arch> - </YUMSource> - <YUMSource> - <Group>centos-5.4</Group> - <RawURL>http://mrepo/centos5-x86_64/RPMS.extras</RawURL> + </Source> + <Source type="yum" rawurl="http://mrepo/centos5-x86_64/RPMS.extras"> <Arch>x86_64</Arch> - </YUMSource> + </Source> </Sources> Due to the :ref:`server-plugins-generators-packages-magic-groups`, diff --git a/doc/development/core.txt b/doc/development/core.txt index 3607533ea..886a5538b 100644 --- a/doc/development/core.txt +++ b/doc/development/core.txt @@ -71,6 +71,11 @@ XML-RPC Server .. automodule:: Bcfg2.SSLServer +Multiprocessing Core +-------------------- + +.. automodule:: Bcfg2.Server.MultiprocessingCore + CherryPy Core ------------- diff --git a/doc/development/lint.txt b/doc/development/lint.txt new file mode 100644 index 000000000..6a4651f92 --- /dev/null +++ b/doc/development/lint.txt @@ -0,0 +1,167 @@ +.. -*- mode: rst -*- + +.. _development-lint: + +=============================== + bcfg2-lint Plugin Development +=============================== + +``bcfg2-lint``, like most parts of Bcfg2, has a pluggable backend that +lets you easily write your own plugins to verify various parts of your +Bcfg2 specification. + +Plugins are loaded in one of two ways: + +* They may be included in a module of the same name as the plugin + class in :mod:`Bcfg2.Server.Lint`, e.g., + :mod:`Bcfg2.Server.Lint.Validate`. +* They may be included directly in a Bcfg2 server plugin, called + "<plugin>Lint", e.g., + :class:`Bcfg2.Server.Plugins.Metadata.MetadataLint`. + +Plugin Types +============ + +There are two types of ``bcfg2-lint`` plugins: + +Serverless plugins +------------------ + +Serverless plugins are run before ``bcfg2-lint`` starts up a local +Bcfg2 server, so the amount of introspection they can do is fairly +limited. They can directly examine the Bcfg2 specification, of +course, but they can't examine the entries handled by a given plugin +or anything that requires a running server. + +If a serverless plugin raises a lint error, however, the server will +not be started and no `Server plugins`_ will be run. This makes them +useful to check for the sorts of errors that might prevent the Bcfg2 +server from starting properly. + +Serverless plugins must subclass +:class:`Bcfg2.Server.Lint.ServerlessPlugin`. + +:mod:`Bcfg2.Server.Lint.Validate` is an example of a serverless +plugin. + +Server plugins +-------------- + +Server plugins are run after a local Bcfg2 server has been started, +and have full access to all of the parsed data and so on. Because of +this, they tend to be easier to use than `Serverless plugins`_, and +thus are more common. + +Server plugins are only run if all `Serverless plugins`_ run +successfully (i.e., raise no errors). + +Server plugins must subclass :class:`Bcfg2.Server.Lint.ServerPlugin`. + +:mod:`Bcfg2.Server.Lint.Genshi` is an example of a server plugin. + +Error Handling +============== + +The job of a ``bcfg2-lint`` plugin is to find errors. Each error that +a plugin may produce must have a name, a short string that briefly +describes the error and will be used to configure error levels in +``bcfg2.conf``. It must also have a default reporting level. +Possible reporting levels are "error", "warning", or "silent". All of +the errors that may be produced by a plugin must be returned as a dict +by :func:`Bcfg2.Server.Lint.Plugin.Errors`. For instance, consider +:func:`Bcfg2.Server.Lint.InfoXML.InfoXML.Errors`: + +.. code-block:: python + + @classmethod + def Errors(cls): + return {"no-infoxml": "warning", + "deprecated-info-file": "warning", + "paranoid-false": "warning", + "required-infoxml-attrs-missing": "error"} + +This means that the :class:`Bcfg2.Server.Lint.InfoXML.InfoXML` lint +plugin can produce five lint errors, although four of them are just +warnings by default. + +The errors returned by each plugin's ``Errors()`` method will be +passed to :func:`Bcfg2.Server.Lint.ErrorHandler.RegisterErrors`, which +will use that information and the information in the config file to +determine how to display (or not display) each error to the end user. + +Errors are produced in a plugin with +:func:`Bcfg2.Server.Lint.Plugin.LintError`, which takes two arguments: +the name of the error, which must correspond to a key in the dict +returned by :func:`Bcfg2.Server.Lint.Plugin.Errors`, and a freeform +string that will be displayed to the end user. Note that the error +name and its display are thus only tied together when the error is +produced; that is, a single error (by name) can have two completely +different outputs. + +Basics +====== + +.. automodule:: Bcfg2.Server.Lint + +Existing ``bcfg2-lint`` Plugins +=============================== + +BundlerLint +----------- + +.. autoclass:: Bcfg2.Server.Plugins.Bundler.BundlerLint + +Comments +-------- + +.. automodule:: Bcfg2.Server.Lint.Comments + +Genshi +------ + +.. automodule:: Bcfg2.Server.Lint.Genshi + +GroupNames +---------- + +.. automodule:: Bcfg2.Server.Lint.GroupNames + +GroupPatternsLint +----------------- + +.. autoclass:: Bcfg2.Server.Plugins.GroupPatterns.GroupPatternsLint + +InfoXML +------- + +.. automodule:: Bcfg2.Server.Lint.InfoXML + +MergeFiles +---------- + +.. automodule:: Bcfg2.Server.Lint.MergeFiles + +MetadataLint +------------ + +.. autoclass:: Bcfg2.Server.Plugins.Metadata.MetadataLint + +PkgmgrLint +---------- + +.. autoclass:: Bcfg2.Server.Plugins.Pkgmgr.PkgmgrLint + +RequiredAttrs +------------- + +.. automodule:: Bcfg2.Server.Lint.RequiredAttrs + +TemplateHelperLint +------------------ + +.. autoclass:: Bcfg2.Server.Plugins.TemplateHelper.TemplateHelperLint + +Validate +-------- + +.. automodule:: Bcfg2.Server.Lint.Validate diff --git a/doc/development/plugins.txt b/doc/development/plugins.txt index 593c2f83e..3f2a888ac 100644 --- a/doc/development/plugins.txt +++ b/doc/development/plugins.txt @@ -213,4 +213,4 @@ See Also -------- * :ref:`development-compat` -* :ref:`development-utils +* :ref:`development-utils` diff --git a/doc/exts/xmlschema.py b/doc/exts/xmlschema.py index 24cbf2e2d..c26aed81e 100644 --- a/doc/exts/xmlschema.py +++ b/doc/exts/xmlschema.py @@ -115,6 +115,7 @@ class _XMLDirective(Directive): def run(self): name = self.arguments[0] env = self.state.document.settings.env + reporter = self.state.memo.reporter ns_name = self.options.get('namespace') try: ns_uri = env.xmlschema_namespaces[ns_name] @@ -129,8 +130,9 @@ class _XMLDirective(Directive): except KeyError: pass else: - env.app.error("No XML %s %s found" % - (" or ".join(self.types), name)) + reporter.error("No XML %s %s found" % + (" or ".join(self.types), name)) + return [] documentor = XMLDocumentor(entity, env, self.state, name=name, ns_uri=ns_uri, include=self.process_include(), @@ -172,6 +174,7 @@ class XMLDocumentor(object): self.include = include self.options = options self.app = self.env.app + self.reporter = self.state.memo.reporter if name is None: self.ns_uri = ns_uri @@ -312,7 +315,8 @@ class XMLDocumentor(object): rv.extend(doc.document_complexType()) return rv else: - self.app.error("Unknown element type %s" % fqtype) + self.reporter.error("Unknown element type %s" % fqtype) + return [] else: rv = [] typespec = self.entity.xpath("xs:complexType", namespaces=NSMAP)[0] diff --git a/doc/help/troubleshooting.txt b/doc/help/troubleshooting.txt index aac831ae0..72fec4c63 100644 --- a/doc/help/troubleshooting.txt +++ b/doc/help/troubleshooting.txt @@ -54,7 +54,7 @@ the debug level individually on a given plugin, e.g.:: bcfg2-admin xcmd Probes.toggle_debug Finally, the File Activity Monitor has its own analogue to these two -methods, for setting the debug level of the FAM: +methods, for setting the debug level of the FAM:: bcfg2-admin xcmd Inotify.toggle_debug bcfg2-admin xcmd Inotify.set_debug false diff --git a/doc/installation/distributions.txt b/doc/installation/distributions.txt index 3dcfd7721..9db111682 100644 --- a/doc/installation/distributions.txt +++ b/doc/installation/distributions.txt @@ -66,19 +66,7 @@ This way is not recommended on production systems. Only for testing. Gentoo ====== -Early in July 2008, Bcfg2 was added to the Gentoo portage tree. So far -it's still keyworded for all architectures, but we are actively working -to get it marked as stable. - -If you don't use portage to install Bcfg2, you'll want to make sure you -have all the prerequisites installed first. For a server, you'll need: - -* ``app-admin/gamin`` or ``app-admin/fam`` -* ``dev-python/lxml`` - -Clients will need at least: - -* ``app-portage/gentoolkit`` +Bcfg2 can be installed via portage. OS X ==== diff --git a/doc/server/configuration.txt b/doc/server/configuration.txt index 2c5879ff0..7892c2612 100644 --- a/doc/server/configuration.txt +++ b/doc/server/configuration.txt @@ -20,6 +20,9 @@ Bcfg2, although it has become easier in 1.3.0. The steps to do so are described in three sections below: Common steps for all versions; steps for older versions only; and steps for 1.3.0. +Many of the steps below may have already been performed by your OS +packages. + Common Steps ------------ @@ -131,7 +134,7 @@ is even invoked. Restart ``bcfg2-server`` and you should see it running as non-root in ``ps`` output:: - % ps -ef | grep '[b]cfg2-server' + % ps -ef | grep '[b]cfg2-server' 1000 11581 1 0 07:55 ? 00:00:15 python usr/sbin/bcfg2-server -C /etc/bcfg2.conf -D /var/run/bcfg2-server/bcfg2-server.pid Steps on Bcfg2 1.3.0 @@ -159,7 +162,7 @@ natively in 1.3. Simply add the following lines to ``bcfg2.conf``:: Restart ``bcfg2-server`` and you should see it running as non-root in ``ps`` output:: - % ps -ef | grep '[b]cfg2-server' + % ps -ef | grep '[b]cfg2-server' 1000 11581 1 0 07:55 ? 00:00:15 python usr/sbin/bcfg2-server -C /etc/bcfg2.conf -D /var/run/bcfg2-server/bcfg2-server.pid .. _server-backends: @@ -169,10 +172,11 @@ Server Backends .. versionadded:: 1.3.0 -Bcfg2 supports two different server backends: a builtin server -based on the Python SimpleXMLRPCServer object, and a server that uses -CherryPy (http://www.cherrypy.org). Each one has advantages and -disadvantages. +Bcfg2 supports three different server backends: a builtin server based +on the Python SimpleXMLRPCServer object; a server that uses CherryPy +(http://www.cherrypy.org); and a version of the builtin server that +uses the Python :mod:`multiprocessing` module. Each one has +advantages and disadvantages. The builtin server: @@ -181,27 +185,36 @@ The builtin server: * Works on Python 2.4; * Is slow with larger numbers of clients. +The multiprocessing server: + +* Leverages most of the stability and maturity of the builtin server, + but does have some new bits; +* Introduces concurrent processing to Bcfg2, which may break in + various edge cases; +* Supports certificate authentication; +* Requires Python 2.6; +* Is faster with large numbers of concurrent runs. + The CherryPy server: * Is very new and potentially buggy; * Does not support certificate authentication yet, only password authentication; -* Requires CherryPy 3.2, which requires Python 2.5; +* Requires CherryPy 3.3, which requires Python 2.5; * Is smarter about daemonization, particularly if you are :ref:`server-dropping-privs`; * Is faster with large numbers of clients. Basically, the builtin server should be used unless you have a -particular need for performance, and can sacrifice certificate -authentication. +particular need for performance. The CherryPy server is purely +experimental at this point. To select which backend to use, set the ``backend`` option in the ``[server]`` section of ``/etc/bcfg2.conf``. Options are: * ``cherrypy`` * ``builtin`` +* ``multiprocessing`` * ``best`` (the default; currently the same as ``builtin``) -If the certificate authentication issues (a limitation in CherryPy -itself) can be resolved and the CherryPy server proves to be stable, -it will likely become the default (and ``best``) in a future release. +``best`` may change in future releases. diff --git a/doc/server/encryption.txt b/doc/server/encryption.txt index b56487620..e31124d4b 100644 --- a/doc/server/encryption.txt +++ b/doc/server/encryption.txt @@ -12,7 +12,8 @@ Bcfg2 supports encrypting some data on the disk, which can help protect sensitive data from other people who need access to the Bcfg2 repository but are perhaps not authorized to see all data. It supports multiple passphrases, which can be used to enforce -separations between teams, environments, etc. +separations between teams, environments, etc. Use of the encryption +feature requires M2Crypto 0.18 or newer. .. note:: diff --git a/doc/server/plugins/connectors/grouplogic.txt b/doc/server/plugins/connectors/grouplogic.txt index b9a5b00d6..abf425202 100644 --- a/doc/server/plugins/connectors/grouplogic.txt +++ b/doc/server/plugins/connectors/grouplogic.txt @@ -110,8 +110,8 @@ individually, there's a more elegant way to accomplish the same thing: <GroupLogic xmlns:py="http://genshi.edgewall.org/"> <?python -component = metadata.group_in_category("webapp-component") -env = metadata.group_in_category("environment") + component = metadata.group_in_category("webapp-component") + env = metadata.group_in_category("environment") ?> <py:if test="component and env"> <Group name="${component}-${env}"/> diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index 606e1e128..cdc4f7282 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -252,7 +252,8 @@ something like this: <Group name="ubuntu-intrepid"> <Source type="apt" url="http://us.archive.ubuntu.com/ubuntu" - version="intrepid"> + version="intrepid" + debsrc="true"> <Component>main</Component> <Component>universe</Component> <Arch>i386</Arch> @@ -280,7 +281,7 @@ something like this: .. warning:: You must regenerate the Packages cache when adding or removing the recommended attribute (``bcfg2-admin xcmd - Packages.Refresh``). + Packages.Refresh``). .. [#f1] Bcfg2 will by default add **Essential** packages to the client specification. You can disable this behavior by diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index 845006115..2493be53f 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -395,9 +395,9 @@ For example: <POSIXUser name="daemon" home="/sbin" shell="/sbin/nologin" gecos="daemon" uid="2" group="daemon"> - <MemberOf>lp</MemberOf> - <MemberOf>adm</MemberOf> - <MemberOf>bin</MemberOf> + <MemberOf group="lp"/> + <MemberOf group="adm"/> + <MemberOf group="bin/> </POSIXUser> The group specified will automatically be created if it does not diff --git a/doc/server/plugins/grouping/grouppatterns.txt b/doc/server/plugins/grouping/grouppatterns.txt index 39c632f82..44ffa5066 100644 --- a/doc/server/plugins/grouping/grouppatterns.txt +++ b/doc/server/plugins/grouping/grouppatterns.txt @@ -8,7 +8,7 @@ GroupPatterns The GroupPatterns plugin is a connector that can assign clients group membership pased on patterns in client hostnames. Two basic -methods are supported: +methods are supported: - regular expressions (NamePatterns) - ranges (NameRange) @@ -20,7 +20,7 @@ Setup ===== #. Enable the GroupPatterns plugin -#. Create the GroupPatterns/config.xml file (similar to the example below). +#. Create the ``GroupPatterns/config.xml`` file (similar to the example below). #. Client groups will be augmented based on the specification Pattern Types @@ -52,7 +52,7 @@ Examples </GroupPattern> <GroupPattern> <NamePattern>(.*)</NamePattern> - <Group>group-$1'</Group> + <Group>group-$1</Group> </GroupPattern> <GroupPattern> <NameRange>node[[1-32]]</NameRange> @@ -82,15 +82,14 @@ GroupPatterns configuration: <GroupPatterns> <GroupPattern> - <NamePattern>^x(\w[^\d|\.]+)\d*\..*</NamePattern> + <NamePattern>x(\w[^\d\.]+)\d*\.</NamePattern> <Group>$1-server</Group> </GroupPattern> </GroupPatterns> Regex explanation: -#. !^x Match any hostname that begins with "x" -#. (\w[!^\d|\.]+) followed by one or more word characters that are not a decimal digit or "." and save the string to $1 -#. \d* followed by 0 or more decimal digit(s) -#. \..* followed by a "." -#. .* followed by 1 or more of anything else. +#. ``x`` Match any hostname that begins with "x" +#. ``(\w[!^\d|\.]+)`` followed by one or more word characters that are not a decimal digit or "." and save the string to $1 +#. ``\d*`` followed by 0 or more decimal digit(s) +#. ``\.`` followed by a literal "." diff --git a/doc/server/plugins/grouping/metadata.txt b/doc/server/plugins/grouping/metadata.txt index 32834b458..ceac5dc24 100644 --- a/doc/server/plugins/grouping/metadata.txt +++ b/doc/server/plugins/grouping/metadata.txt @@ -222,7 +222,7 @@ to include a different file, or explicit content in the case that the parent include does not exist.) Wildcard XInclude -~~~~~~~~~~~~~~~~~ +----------------- .. versionadded:: 1.3.1 diff --git a/doc/server/snapshots/index.txt b/doc/server/snapshots/index.txt index 13a9fe2c0..a7e5940ed 100644 --- a/doc/server/snapshots/index.txt +++ b/doc/server/snapshots/index.txt @@ -8,9 +8,8 @@ Bcfg2 Snapshots .. versionadded:: 1.0.0 -This page describes the Snapshots plugin. This plugin is meant to replace -the older :ref:`reports-dynamic`. It stores various aspects of a client's -state when the client checks into the server. +This page describes the Snapshots plugin. Snapshots is deprecated, and +will be removed in a future release. Before you begin ================ diff --git a/misc/bcfg2-selinux.spec b/misc/bcfg2-selinux.spec index 4c05f4959..9c5262dfd 100644 --- a/misc/bcfg2-selinux.spec +++ b/misc/bcfg2-selinux.spec @@ -3,7 +3,7 @@ %global pythonversion %{py_ver} %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %{!?_initrddir: %global _initrddir %{_sysconfdir}/rc.d/init.d} -%global selinux_policyver %(%{__sed} -e 's,.*selinux-policy-\\([^/]*\\)/.*,\\1,' /usr/share/selinux/devel/policyhelp || echo 0.0.0) +%global selinux_policyver %(%{__sed} -e 's,.*selinux-policy-\\([^/]*\\)/.*,\\1,' /usr/share/selinux/devel/policyhelp 2>/dev/null || echo 0.0.0) %global selinux_types %(%{__awk} '/^#[[:space:]]*SELINUXTYPE=/,/^[^#]/ { if ($3 == "-") printf "%s ", $2 }' /etc/selinux/config 2>/dev/null) %global selinux_variants %([ -z "%{selinux_types}" ] && echo mls strict targeted || echo %{selinux_types}) diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index d8eb8e5de..10d913d7c 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -99,12 +99,11 @@ Requires: bcfg2 = %{version} Requires: python-ssl %endif Requires: python-lxml >= 1.2.1 -%if "%{_vendor}" == "redhat" -Requires: gamin-python -%endif %if 0%{?suse_version} +Requires: python-pyinotify Requires: python-python-daemon %else +Requires: python-inotify Requires: python-daemon %endif Requires: /usr/sbin/sendmail @@ -150,7 +149,7 @@ Group: System Tools Requires: bcfg2 = %{version} Requires: bcfg2-server = %{version} -# cherrypy 3.2.3 actually doesn't exist yet, but 3.2.2 has bugs that +# cherrypy 3.3 actually doesn't exist yet, but 3.2 has bugs that # prevent it from working: # https://bitbucket.org/cherrypy/cherrypy/issue/1154/assertionerror-in-recv-when-ssl-is-enabled Requires: python-cherrypy > 3.3 @@ -190,6 +189,27 @@ Group: Documentation/HTML %else Group: Documentation %endif +%if 0%{?suse_version} +BuildRequires: python-M2Crypto +BuildRequires: python-Genshi +BuildRequires: python-gamin +BuildRequires: python-pyinotify +BuildRequires: python-python-daemon +BuildRequires: python-CherryPy >= 3 +%else +BuildRequires: m2crypto +BuildRequires: python-genshi +BuildRequires: gamin-python +BuildRequires: python-inotify +BuildRequires: python-daemon +%endif + +%if "%{_vendor}" == "redhat" && 0%{?rhel} < 6 && 0%{?fedora} == 0 +BuildRequires: python-ssl +%else +BuildRequires: python-cherrypy >= 3 +BuildRequires: python-mock +%endif %description doc Bcfg2 helps system administrators produce a consistent, reproducible, @@ -275,6 +295,8 @@ This package includes the Bcfg2 reports web frontend. %{?pythonpath: export PYTHONPATH="%{pythonpath}"} %{__python}%{pythonversion} setup.py build_sphinx +sed -i "s/apache2/httpd/g" misc/apache/bcfg2.conf + %install rm -rf %{buildroot} %{__python}%{pythonversion} setup.py install --root=%{buildroot} --record=INSTALLED_FILES --prefix=/usr @@ -311,7 +333,6 @@ cp -r tools/* %{buildroot}%{_defaultdocdir}/bcfg2-server-%{version} cp -r build/sphinx/html/* %{buildroot}%{_defaultdocdir}/bcfg2-doc-%{version} %{__install} -d %{buildroot}%{apache_conf}/conf.d -sed -i "s/apache2/httpd/g" misc/apache/bcfg2.conf %{__install} -m 644 misc/apache/bcfg2.conf %{buildroot}%{apache_conf}/conf.d/wsgi_bcfg2.conf %{__mkdir_p} %{buildroot}%{_localstatedir}/cache/%{name} @@ -334,6 +355,7 @@ touch %{buildroot}%{_sysconfdir}/bcfg2.conf \ %{python_sitelib}/Bcfg2/Logger.py* %{python_sitelib}/Bcfg2/Options.py* %{python_sitelib}/Bcfg2/Proxy.py* +%{python_sitelib}/Bcfg2/Utils.py* %{python_sitelib}/Bcfg2/version.py* %{python_sitelib}/Bcfg2/Client %{_mandir}/man1/bcfg2.1* diff --git a/schemas/packages.xsd b/schemas/packages.xsd index dbee2f31b..e538bb0e7 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -167,6 +167,15 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:boolean" name="debsrc"> + <xsd:annotation> + <xsd:documentation> + Include ``deb-src`` lines in the generated APT + configuration. This only applies to sources with + :xml:attribute:`SourceType:type` = ``apt``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attribute type="xsd:string" name="url"> <xsd:annotation> <xsd:documentation> diff --git a/schemas/servicetype.xsd b/schemas/servicetype.xsd index 4d5ac7c31..4c7e1b803 100644 --- a/schemas/servicetype.xsd +++ b/schemas/servicetype.xsd @@ -34,12 +34,21 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute name="bootstatus" type="BootStatusEnum" default="off"> + <xsd:annotation> + <xsd:documentation> + Whether the service should start at boot. The default value + corresponds to the value of the status attribute. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attribute name="status" type="StatusEnum" default="off"> <xsd:annotation> <xsd:documentation> - Whether the service should start at boot. If this is set to - "ignore", then the boot-time status of the service will not - be checked. + Whether the service should be on or off when the bcfg2 client + is run. This attribute may have different behavior depending + on the characteristics of the client tool. If set to "ignore", + then the status of the service will not be checked. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/schemas/types.xsd b/schemas/types.xsd index 31fea26a2..5f6328a62 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -53,6 +53,13 @@ </xsd:restriction> </xsd:simpleType> + <xsd:simpleType name='BootStatusEnum'> + <xsd:restriction base='xsd:string'> + <xsd:enumeration value='on'/> + <xsd:enumeration value='off'/> + </xsd:restriction> + </xsd:simpleType> + <xsd:simpleType name='StatusEnum'> <xsd:restriction base='xsd:string'> <xsd:enumeration value='on'/> @@ -381,6 +388,27 @@ </xsd:restriction> </xsd:simpleType> + <xsd:complexType name="MemberOfType"> + <xsd:annotation> + <xsd:documentation> + Specify additional supplementary groups for the POSIXUser + </xsd:documentation> + </xsd:annotation> + <xsd:simpleContent> + <xsd:extension base="xsd:token"> + <xsd:attribute name="group" type="xsd:token"> + <xsd:annotation> + <xsd:documentation> + The name of the supplementary group. This can also be + specified as content of the tag, although that is + deprecated. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:extension> + </xsd:simpleContent> + </xsd:complexType> + <xsd:complexType name="POSIXUserType"> <xsd:annotation> <xsd:documentation> @@ -388,13 +416,7 @@ </xsd:documentation> </xsd:annotation> <xsd:choice minOccurs='0' maxOccurs='unbounded'> - <xsd:element name='MemberOf' type='xsd:token'> - <xsd:annotation> - <xsd:documentation> - Specify additional supplementary groups for the POSIXUser - </xsd:documentation> - </xsd:annotation> - </xsd:element> + <xsd:element name='MemberOf' type='MemberOfType'/> </xsd:choice> <xsd:attribute type="xsd:token" name="name" use="required"> <xsd:annotation> @@ -4,26 +4,26 @@ from setuptools import setup from glob import glob import sys -vfile = 'src/lib/Bcfg2/version.py' +version_file = 'src/lib/Bcfg2/version.py' try: # python 2 - execfile(vfile) + execfile(version_file) except NameError: # py3k - exec(compile(open(vfile).read(), vfile, 'exec')) + exec(compile(open(version_file).read(), version_file, 'exec')) -# we only need m2crypto on < python2.6 -need_m2crypto = False -version = sys.version_info[:2] -if version < (2, 6): - need_m2crypto = True +inst_reqs = [ + 'lockfile', + 'lxml', + 'python-daemon', +] -inst_reqs = ['lxml'] -if need_m2crypto: +# we only need m2crypto on < python2.6 +if sys.version_info[:2] < (2, 6): inst_reqs.append('M2Crypto') setup(name="Bcfg2", - version="1.3.1", + version=__version__, # Defined in src/lib/Bcfg2/version.py description="Bcfg2 Server", author="Narayan Desai", author_email="desai@mcs.anl.gov", @@ -55,9 +55,9 @@ setup(name="Bcfg2", install_requires=inst_reqs, tests_require=['mock', 'nose', 'sqlalchemy'], package_dir={'': 'src/lib', }, - package_data={'Bcfg2.Reporting': [ 'templates/*.html', - 'templates/*/*.html', - 'templates/*/*.inc']}, + package_data={'Bcfg2.Reporting': ['templates/*.html', + 'templates/*/*.html', + 'templates/*/*.inc']}, scripts=glob('src/sbin/*'), data_files=[('share/bcfg2/schemas', glob('schemas/*.xsd')), diff --git a/solaris-ips/MANIFEST.bcfg2-server.header b/solaris-ips/MANIFEST.bcfg2-server.header new file mode 100644 index 000000000..efa11181f --- /dev/null +++ b/solaris-ips/MANIFEST.bcfg2-server.header @@ -0,0 +1,5 @@ +license ../../LICENSE license=simplified_bsd +set name=description value="Configuration management server" +set name=pkg.summary value="Configuration management server" +set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@1.3.1" + diff --git a/solaris-ips/MANIFEST.bcfg2.header b/solaris-ips/MANIFEST.bcfg2.header new file mode 100644 index 000000000..8358aafca --- /dev/null +++ b/solaris-ips/MANIFEST.bcfg2.header @@ -0,0 +1,6 @@ +license ../../LICENSE license=simplified_bsd +set name=description value="Configuration management client" +set name=pkg.summary value="Configuration management client" +set name=pkg.fmri value="pkg://bcfg2/bcfg2@1.3.1" + +file usr/bin/bcfg2 group=bin mode=0755 owner=root path=usr/bin/bcfg2 diff --git a/solaris-ips/Makefile b/solaris-ips/Makefile new file mode 100644 index 000000000..343150dc5 --- /dev/null +++ b/solaris-ips/Makefile @@ -0,0 +1,20 @@ +#!/usr/bin/gmake + +VERS=1.2.4-1 +PYVERSION := $(shell python -c "import sys; print sys.version[0:3]") + +default: clean package + +package: + -mkdir tmp tmp/bcfg2-server tmp/bcfg2 + -mkdir -p build/lib/$(PYVERSION)/site-packages + -cd ../ && PYTHONPATH=$(PYTHONPATH):$(PWD)/build/lib/python2.6/site-packages/ python setup.py install --single-version-externally-managed --record=/dev/null --prefix=$(PWD)/build/usr + #setuptools appears to use a restictive umask + -chmod -R o+r build/ + -chmod +x build/usr/bin/bcfg2 + -sh ./gen-manifests.sh + +clean: + -rm -rf tmp build + -rm -rf MANIFEST.bcfg2 + -rm -rf MANIFEST.bcfg2-server diff --git a/solaris-ips/README b/solaris-ips/README new file mode 100644 index 000000000..24021b992 --- /dev/null +++ b/solaris-ips/README @@ -0,0 +1,22 @@ +BUILDING +-------- + +Dependancies: + gmake + +Usage: + gmake + + +PUBLISHING +---------- + +Modify MANIFEST.bcfg2 and MANIFEST.bcfg2-server to set your publisher name in the fmri, e.g. Change + set name=pkg.fmri value="pkg://bcfg2/bcfg2@1.2.4" +to + set name=pkg.fmri value="pkg://example.com/bcfg2@1.2.4" + + +Then run the pkgsend publish, i.e. + pkgsend publish -s http://example.com/path/to/repo -d build MANIFEST.bcfg2 + pkgsend publish -s http://example.com/path/to/repo -d build MANIFEST.bcfg2-server diff --git a/solaris-ips/gen-manifests.sh b/solaris-ips/gen-manifests.sh new file mode 100644 index 000000000..3b4cd30df --- /dev/null +++ b/solaris-ips/gen-manifests.sh @@ -0,0 +1,15 @@ +#!/usr/bin/sh + +#bcfg2 +cat MANIFEST.bcfg2.header > MANIFEST.bcfg2 +pkgsend generate build | grep man[15] >> MANIFEST.bcfg2 +pkgsend generate build | grep Bcfg2/[^/]*.py$ >> MANIFEST.bcfg2 +pkgsend generate build | grep Bcfg2/Client/.*.py$ >> MANIFEST.bcfg2 + +#bcfg2-server +cat MANIFEST.bcfg2-server.header > MANIFEST.bcfg2-server +pkgsend generate build | grep man[8] >> MANIFEST.bcfg2-server +pkgsend generate build | grep share/bcfg2 >> MANIFEST.bcfg2-server +pkgsend generate build | grep bin/bcfg2- >> MANIFEST.bcfg2-server +pkgsend generate build | grep Bcfg2/Server/.*.py$ >> MANIFEST.bcfg2-server + diff --git a/solaris-ips/pkginfo.bcfg2 b/solaris-ips/pkginfo.bcfg2 new file mode 100644 index 000000000..90c628c53 --- /dev/null +++ b/solaris-ips/pkginfo.bcfg2 @@ -0,0 +1,10 @@ +PKG="SCbcfg2" +NAME="bcfg2" +ARCH="sparc" +VERSION="1.2.4" +CATEGORY="application" +VENDOR="Argonne National Labratory" +EMAIL="bcfg-dev@mcs.anl.gov" +PSTAMP="Bcfg2 Developers" +BASEDIR="/opt/csw" +CLASSES="none" diff --git a/solaris-ips/pkginfo.bcfg2-server b/solaris-ips/pkginfo.bcfg2-server new file mode 100644 index 000000000..0e865522c --- /dev/null +++ b/solaris-ips/pkginfo.bcfg2-server @@ -0,0 +1,10 @@ +PKG="SCbcfg2-server" +NAME="bcfg2-server" +ARCH="sparc" +VERSION="1.2.4" +CATEGORY="application" +VENDOR="Argonne National Labratory" +EMAIL="bcfg-dev@mcs.anl.gov" +PSTAMP="Bcfg2 Developers" +BASEDIR="/usr" +CLASSES="none" diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 850e58d9d..6bef77081 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -6,15 +6,7 @@ import fnmatch import logging import Bcfg2.Client.Tools from Bcfg2.Client import prompt -from Bcfg2.Compat import any, all, cmp # pylint: disable=W0622 - - -def cmpent(ent1, ent2): - """Sort entries.""" - if ent1.tag != ent2.tag: - return cmp(ent1.tag, ent2.tag) - else: - return cmp(ent1.get('name'), ent2.get('name')) +from Bcfg2.Compat import any, all # pylint: disable=W0622 def matches_entry(entryspec, entry): @@ -105,8 +97,8 @@ class Frame(object): self.logger.warning(deprecated) experimental = [tool.name for tool in self.tools if tool.experimental] if experimental: - self.logger.warning("Loaded experimental tool drivers:") - self.logger.warning(experimental) + self.logger.info("Loaded experimental tool drivers:") + self.logger.info(experimental) # find entries not handled by any tools self.unhandled = [entry for struct in config @@ -155,7 +147,7 @@ class Frame(object): def promptFilter(self, msg, entries): """Filter a supplied list based on user input.""" ret = [] - entries.sort(cmpent) + entries.sort(key=lambda e: e.tag + ":" + e.get('name')) for entry in entries[:]: if entry in self.unhandled: # don't prompt for entries that can't be installed @@ -418,10 +410,12 @@ class Frame(object): # prune out unspecified bundles when running with -b continue if bundle in mbundles: - self.logger.debug("Bundle %s was modified" % bundle) + self.logger.debug("Bundle %s was modified" % + bundle.get('name')) func = "BundleUpdated" else: - self.logger.debug("Bundle %s was not modified" % bundle) + self.logger.debug("Bundle %s was not modified" % + bundle.get('name')) func = "BundleNotUpdated" for tool in self.tools: try: diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index ec7f462b3..256c28255 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -19,25 +19,22 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): def get_svc_command(self, service, action): return "/sbin/service %s %s" % (service.get('name'), action) - def VerifyService(self, entry, _): - """Verify Service status for entry.""" - if entry.get('status') == 'ignore': - return True - + def verify_bootstatus(self, entry, bootstatus): + """Verify bootstatus for entry.""" rv = self.cmd.run("/sbin/chkconfig --list %s " % entry.get('name')) if rv.success: srvdata = rv.stdout.splitlines()[0].split() else: # service not installed - entry.set('current_status', 'off') + entry.set('current_bootstatus', 'service not installed') return False if len(srvdata) == 2: # This is an xinetd service - if entry.get('status') == srvdata[1]: + if bootstatus == srvdata[1]: return True else: - entry.set('current_status', srvdata[1]) + entry.set('current_bootstatus', srvdata[1]) return False try: @@ -46,37 +43,74 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): except IndexError: onlevels = [] - pstatus = self.check_service(entry) - if entry.get('status') == 'on': - status = (len(onlevels) > 0 and pstatus) + if bootstatus == 'on': + current_bootstatus = (len(onlevels) > 0) else: - status = (len(onlevels) == 0 and not pstatus) + current_bootstatus = (len(onlevels) == 0) + return current_bootstatus + + def VerifyService(self, entry, _): + """Verify Service status for entry.""" + entry.set('target_status', entry.get('status')) # for reporting + bootstatus = self.get_bootstatus(entry) + if bootstatus is None: + return True + current_bootstatus = self.verify_bootstatus(entry, bootstatus) - if not status: + if entry.get('status') == 'ignore': + # 'ignore' should verify + current_svcstatus = True + else: + svcstatus = self.check_service(entry) if entry.get('status') == 'on': - entry.set('current_status', 'off') - else: - entry.set('current_status', 'on') - return status + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True + + if svcstatus: + entry.set('current_status', 'on') + else: + entry.set('current_status', 'off') + + return current_bootstatus and current_svcstatus def InstallService(self, entry): """Install Service entry.""" - rcmd = "/sbin/chkconfig %s %s" - self.cmd.run("/sbin/chkconfig --add %s" % (entry.attrib['name'])) + self.cmd.run("/sbin/chkconfig --add %s" % (entry.get('name'))) self.logger.info("Installing Service %s" % (entry.get('name'))) - rv = True - if entry.get('status') == 'off': - rv &= self.cmd.run((rcmd + " --level 0123456") % - (entry.get('name'), - entry.get('status'))).success - if entry.get("current_status") == "on": - rv &= self.stop_service(entry).success + bootstatus = entry.get('bootstatus') + if bootstatus is not None: + if bootstatus == 'on': + # make sure service is enabled on boot + bootcmd = '/sbin/chkconfig %s %s --level 0123456' % \ + (entry.get('name'), entry.get('bootstatus')) + elif bootstatus == 'off': + # make sure service is disabled on boot + bootcmd = '/sbin/chkconfig %s %s' % (entry.get('name'), + entry.get('bootstatus')) + bootcmdrv = self.cmd.run(bootcmd).success + if self.setup['servicemode'] == 'disabled': + # 'disabled' means we don't attempt to modify running svcs + return bootcmdrv + buildmode = self.setup['servicemode'] == 'build' + if (entry.get('status') == 'on' and not buildmode) and \ + entry.get('current_status') == 'off': + svccmdrv = self.start_service(entry) + elif (entry.get('status') == 'off' or buildmode) and \ + entry.get('current_status') == 'on': + svccmdrv = self.stop_service(entry) + else: + svccmdrv = True # ignore status attribute + return bootcmdrv and svccmdrv else: - rv &= self.cmd.run(rcmd % (entry.get('name'), - entry.get('status'))).success - if entry.get("current_status") == "off": - rv &= self.start_service(entry).success - return rv + # when bootstatus is 'None', status == 'ignore' + return True def FindExtra(self): """Locate extra chkconfig Services.""" diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index d916b1662..761c51db7 100644 --- a/src/lib/Bcfg2/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -18,13 +18,11 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): svcre = \ re.compile(r'/etc/.*/(?P<action>[SK])(?P<sequence>\d+)(?P<name>\S+)') - # implement entry (Verify|Install) ops - def VerifyService(self, entry, _): - """Verify Service status for entry.""" - - if entry.get('status') == 'ignore': - return True + def get_svc_command(self, service, action): + return '/usr/sbin/invoke-rc.d %s %s' % (service.get('name'), action) + def verify_bootstatus(self, entry, bootstatus): + """Verify bootstatus for entry.""" rawfiles = glob.glob("/etc/rc*.d/[SK]*%s" % (entry.get('name'))) files = [] @@ -54,9 +52,9 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): continue if match.group('name') == entry.get('name'): files.append(filename) - if entry.get('status') == 'off': + if bootstatus == 'off': if files: - entry.set('current_status', 'on') + entry.set('current_bootstatus', 'on') return False else: return True @@ -72,12 +70,47 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): return False return True else: - entry.set('current_status', 'off') + entry.set('current_bootstatus', 'off') return False + def VerifyService(self, entry, _): + """Verify Service status for entry.""" + entry.set('target_status', entry.get('status')) # for reporting + bootstatus = self.get_bootstatus(entry) + if bootstatus is None: + return True + current_bootstatus = self.verify_bootstatus(entry, bootstatus) + + if entry.get('status') == 'ignore': + # 'ignore' should verify + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True + + if svcstatus: + entry.set('current_status', 'on') + else: + entry.set('current_status', 'off') + + return current_bootstatus and current_svcstatus + def InstallService(self, entry): - """Install Service for entry.""" + """Install Service entry.""" self.logger.info("Installing Service %s" % (entry.get('name'))) + bootstatus = entry.get('bootstatus') + + # check if init script exists try: os.stat('/etc/init.d/%s' % entry.get('name')) except OSError: @@ -85,20 +118,41 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): entry.get('name')) return False - if entry.get('status') == 'off': - self.cmd.run("/usr/sbin/invoke-rc.d %s stop" % (entry.get('name'))) - return self.cmd.run("/usr/sbin/update-rc.d -f %s remove" % - entry.get('name')).success + if bootstatus is not None: + seqcmdrv = True + if bootstatus == 'on': + # make sure service is enabled on boot + bootcmd = '/usr/sbin/update-rc.d %s defaults' % \ + entry.get('name') + if entry.get('sequence'): + seqcmd = '/usr/sbin/update-rc.d -f %s remove' % \ + entry.get('name') + seqcmdrv = self.cmd.run(seqcmd) + start_sequence = int(entry.get('sequence')) + kill_sequence = 100 - start_sequence + bootcmd = '%s %d %d' % (bootcmd, start_sequence, + kill_sequence) + elif bootstatus == 'off': + # make sure service is disabled on boot + bootcmd = '/usr/sbin/update-rc.d -f %s remove' % \ + entry.get('name') + bootcmdrv = self.cmd.run(bootcmd) + if self.setup['servicemode'] == 'disabled': + # 'disabled' means we don't attempt to modify running svcs + return bootcmdrv and seqcmdrv + buildmode = self.setup['servicemode'] == 'build' + if (entry.get('status') == 'on' and not buildmode) and \ + entry.get('current_status') == 'off': + svccmdrv = self.start_service(entry) + elif (entry.get('status') == 'off' or buildmode) and \ + entry.get('current_status') == 'on': + svccmdrv = self.stop_service(entry) + else: + svccmdrv = True # ignore status attribute + return bootcmdrv and svccmdrv and seqcmdrv else: - command = "/usr/sbin/update-rc.d %s defaults" % (entry.get('name')) - if entry.get('sequence'): - if not self.cmd.run("/usr/sbin/update-rc.d -f %s remove" % - entry.get('name')).success: - return False - start_sequence = int(entry.get('sequence')) - kill_sequence = 100 - start_sequence - command = "%s %d %d" % (command, start_sequence, kill_sequence) - return self.cmd.run(command).success + # when bootstatus is 'None', status == 'ignore' + return True def FindExtra(self): """Find Extra Debian Service entries.""" @@ -116,6 +170,3 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): # Extra service removal is nonsensical # Extra services need to be reflected in the config return - - def get_svc_command(self, service, action): - return '/usr/sbin/invoke-rc.d %s %s' % (service.get('name'), action) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py index 9d0fe05e0..675a4461a 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py @@ -36,14 +36,14 @@ class POSIXDirectory(POSIXTool): self.logger.info("POSIX: " + msg) entry.set('qtext', entry.get('qtext', '') + '\n' + msg) for extra in extras: - Bcfg2.Client.XML.SubElement(entry, 'Prune', path=extra) + Bcfg2.Client.XML.SubElement(entry, 'Prune', name=extra) except OSError: prune = True return POSIXTool.verify(self, entry, modlist) and prune def install(self, entry): - """Install device entries.""" + """Install directory entries.""" fmode = self._exists(entry) if fmode and not stat.S_ISDIR(fmode[stat.ST_MODE]): @@ -67,7 +67,7 @@ class POSIXDirectory(POSIXTool): if entry.get('prune', 'false') == 'true': for pent in entry.findall('Prune'): - pname = pent.get('path') + pname = pent.get('name') try: self.logger.debug("POSIX: Removing %s" % pname) self._remove(pent) diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 99ed3c7d9..8226392f9 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -154,7 +154,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get("current_exists", "true") == "true": # verify supplemental groups actual = [g[0] for g in self.user_supplementary_groups(entry)] - expected = [e.text for e in entry.findall("MemberOf")] + expected = [e.get("group", e.text).strip() + for e in entry.findall("MemberOf")] if set(expected) != set(actual): entry.set('qtext', "\n".join([entry.get('qtext', '')] + @@ -252,7 +253,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get('uid'): cmd.extend(['-u', entry.get('uid')]) cmd.extend(['-g', entry.get('group')]) - extras = [e.text for e in entry.findall("MemberOf")] + extras = [e.get("group", e.text).strip() + for e in entry.findall("MemberOf")] if extras: cmd.extend(['-G', ",".join(extras)]) cmd.extend(['-d', entry.get('home')]) diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py index 17e7755a9..2d8b66ce5 100644 --- a/src/lib/Bcfg2/Client/Tools/Portage.py +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -74,10 +74,10 @@ class Portage(Bcfg2.Client.Tools.PkgTool): self.logger.debug('Running equery check on %s' % entry.get('name')) - for line in self.cmd.run(["/usr/bin/equery", "-N", "check", - '=%s-%s' % - (entry.get('name'), - version)]).stdout.splitlines(): + for line in self.cmd.run( + ["/usr/bin/equery", "-N", "check", + '=%s-%s' % (entry.get('name'), + entry.get('version'))]).stdout.splitlines(): if '!!!' in line and line.split()[1] not in modlist: return False diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index 4b78581f7..8e9626521 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -21,21 +21,38 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): '-s']).stdout.splitlines() if 'started' in line] + def get_default_svcs(self): + """Return a list of services in the 'default' runlevel.""" + return [line.split()[0] + for line in self.cmd.run(['/sbin/rc-update', + 'show']).stdout.splitlines() + if 'default' in line] + + def verify_bootstatus(self, entry, bootstatus): + """Verify bootstatus for entry.""" + # get a list of all started services + allsrv = self.get_default_svcs() + # set current_bootstatus attribute + if entry.get('name') in allsrv: + entry.set('current_bootstatus', 'on') + else: + entry.set('current_bootstatus', 'off') + if bootstatus == 'on': + return entry.get('name') in allsrv + else: + return entry.get('name') not in allsrv + def VerifyService(self, entry, _): """ Verify Service status for entry. Assumes we run in the "default" runlevel. """ - if entry.get('status') == 'ignore': + entry.set('target_status', entry.get('status')) # for reporting + bootstatus = self.get_bootstatus(entry) + if bootstatus is None: return True - - # get a list of all started services - allsrv = self.get_enabled_svcs() - - # check if service is enabled - result = self.cmd.run(["/sbin/rc-update", "show", "default"]).stdout - is_enabled = entry.get("name") in result + current_bootstatus = self.verify_bootstatus(entry, bootstatus) # check if init script exists try: @@ -45,39 +62,58 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): entry.get('name')) return False - # check if service is enabled - is_running = entry.get('name') in allsrv - - if entry.get('status') == 'on' and not (is_enabled and is_running): - entry.set('current_status', 'off') - return False - - elif entry.get('status') == 'off' and (is_enabled or is_running): + if entry.get('status') == 'ignore': + # 'ignore' should verify + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True + + if svcstatus: entry.set('current_status', 'on') - return False + else: + entry.set('current_status', 'off') - return True + return current_bootstatus and current_svcstatus def InstallService(self, entry): - """ - Install Service entry - - """ + """Install Service entry.""" self.logger.info('Installing Service %s' % entry.get('name')) - if entry.get('status') == 'on': - if entry.get('current_status') == 'off': - self.start_service(entry) - # make sure it's enabled - cmd = '/sbin/rc-update add %s default' - return self.cmd.run(cmd % entry.get('name')).success - elif entry.get('status') == 'off': - if entry.get('current_status') == 'on': - self.stop_service(entry) - # make sure it's disabled - cmd = '/sbin/rc-update del %s default' - return self.cmd.run(cmd % entry.get('name')).success - - return False + bootstatus = entry.get('bootstatus') + if bootstatus is not None: + if bootstatus == 'on': + # make sure service is enabled on boot + bootcmd = '/sbin/rc-update add %s default' + elif bootstatus == 'off': + # make sure service is disabled on boot + bootcmd = '/sbin/rc-update del %s default' + bootcmdrv = self.cmd.run(bootcmd % entry.get('name')).success + if self.setup['servicemode'] == 'disabled': + # 'disabled' means we don't attempt to modify running svcs + return bootcmdrv + buildmode = self.setup['servicemode'] == 'build' + if (entry.get('status') == 'on' and not buildmode) and \ + entry.get('current_status') == 'off': + svccmdrv = self.start_service(entry) + elif (entry.get('status') == 'off' or buildmode) and \ + entry.get('current_status') == 'on': + svccmdrv = self.stop_service(entry) + else: + svccmdrv = True # ignore status attribute + return bootcmdrv and svccmdrv + else: + # when bootstatus is 'None', status == 'ignore' + return True def FindExtra(self): """Locate extra rc-update services.""" diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 027d91c71..20a172d3d 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,6 +13,8 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} + conflicts = ['Chkconfig'] + def get_svc_command(self, service, action): return "/bin/systemctl %s %s.service" % (action, service.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index c5a5ee4d6..11fe55bd6 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -519,6 +519,22 @@ class SvcTool(Tool): """ return '/etc/init.d/%s %s' % (service.get('name'), action) + def get_bootstatus(self, service): + """ Return the bootstatus attribute if it exists. + + :param service: The service entry + :type service: lxml.etree._Element + :returns: string or None - Value of bootstatus if it exists. If + bootstatus is unspecified and status is not *ignore*, + return value of status. If bootstatus is unspecified + and status is *ignore*, return None. + """ + if service.get('bootstatus') is not None: + return service.get('bootstatus') + elif service.get('status') != 'ignore': + return service.get('status') + return None + def start_service(self, service): """ Start a service. diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index e40ef750b..3bc261f2f 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -19,7 +19,7 @@ def prompt(msg): while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: os.read(sys.stdin.fileno(), 4096) try: - ans = input(msg.encode(sys.stdout.encoding, 'replace')) + ans = input(msg) return ans in ['y', 'Y'] except EOFError: # python 2.4.3 on CentOS doesn't like ^C for some reason diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 4e9239e26..d034c0777 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -89,11 +89,28 @@ def u_str(string, encoding=None): else: return unicode(string) +try: + from functools import wraps +except ImportError: + def wraps(wrapped): # pylint: disable=W0613 + """ implementation of functools.wraps() for python 2.4 """ + return lambda f: f + + # base64 compat if sys.hexversion >= 0x03000000: from base64 import b64encode as _b64encode, b64decode as _b64decode - b64encode = lambda s: _b64encode(s.encode('UTF-8')).decode('UTF-8') - b64decode = lambda s: _b64decode(s.encode('UTF-8')).decode('UTF-8') + + @wraps(_b64encode) + def b64encode(val, **kwargs): # pylint: disable=C0111 + try: + return _b64encode(val, **kwargs) + except TypeError: + return _b64encode(val.encode('UTF-8'), **kwargs).decode('UTF-8') + + @wraps(_b64decode) + def b64decode(val, **kwargs): # pylint: disable=C0111 + return _b64decode(val.encode('UTF-8'), **kwargs).decode('UTF-8') else: from base64 import b64encode, b64decode @@ -242,14 +259,6 @@ except ImportError: from md5 import md5 -try: - from functools import wraps -except ImportError: - def wraps(wrapped): # pylint: disable=W0613 - """ implementation of functools.wraps() for python 2.4 """ - return lambda f: f - - def oct_mode(mode): """ Convert a decimal number describing a POSIX permissions mode to a string giving the octal mode. In Python 2, this is a synonym diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 55f799a29..243c4ed2a 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -535,6 +535,11 @@ SERVER_FAM_IGNORE = \ 'SCCS', '.svn', '4913', '.gitignore'], cf=('server', 'ignore_files'), cook=list_split) +SERVER_FAM_BLOCK = \ + Option('FAM blocks on startup until all events are processed', + default=False, + cook=get_bool, + cf=('server', 'fam_blocking')) SERVER_LISTEN_ALL = \ Option('Listen on all interfaces', default=False, @@ -574,7 +579,7 @@ SERVER_PASSWORD = \ SERVER_PROTOCOL = \ Option('Server Protocol', default='xmlrpc/ssl', - cf=('communication', 'procotol')) + cf=('communication', 'protocol')) SERVER_BACKEND = \ Option('Server Backend', default='best', @@ -604,6 +609,16 @@ SERVER_AUTHENTICATION = \ default='cert+password', odesc='{cert|bootstrap|cert+password}', cf=('communication', 'authentication')) +SERVER_CHILDREN = \ + Option('Spawn this number of children for the multiprocessing core. ' + 'By default spawns children equivalent to the number of processors ' + 'in the machine.', + default=None, + cmd='--children', + odesc='<children>', + cf=('server', 'children'), + cook=get_int, + long_arg=True) # database options DB_ENGINE = \ @@ -1166,6 +1181,7 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, password=SERVER_PASSWORD, filemonitor=SERVER_FILEMONITOR, ignore=SERVER_FAM_IGNORE, + fam_blocking=SERVER_FAM_BLOCK, location=SERVER_LOCATION, key=SERVER_KEY, cert=SERVER_CERT, @@ -1176,7 +1192,8 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, vcs_root=SERVER_VCS_ROOT, authentication=SERVER_AUTHENTICATION, perflog=LOG_PERFORMANCE, - perflog_interval=PERFLOG_INTERVAL) + perflog_interval=PERFLOG_INTERVAL, + children=SERVER_CHILDREN) CRYPT_OPTIONS = dict(encrypt=ENCRYPT, decrypt=DECRYPT, diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index b2b9fcc2e..f6db66a93 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -24,6 +24,7 @@ from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus version = sys.version_info[:2] has_py26 = version >= (2, 6) +has_py32 = version >= (3, 2) __all__ = ["ComponentProxy", "RetryMethod", @@ -173,8 +174,12 @@ class SSLHTTPConnection(httplib.HTTPConnection): """ if not has_py26: httplib.HTTPConnection.__init__(self, host, port, strict) - else: + elif not has_py32: httplib.HTTPConnection.__init__(self, host, port, strict, timeout) + else: + # the strict parameter is deprecated. + # HTTP 0.9-style "Simple Responses" are not supported anymore. + httplib.HTTPConnection.__init__(self, host, port, timeout=timeout) self.key = key self.cert = cert self.ca = ca @@ -309,7 +314,7 @@ class XMLRPCTransport(xmlrpclib.Transport): errcode = response.status errmsg = response.reason headers = response.msg - except (socket.error, SSL_ERROR): + except (socket.error, SSL_ERROR, httplib.BadStatusLine): err = sys.exc_info()[1] raise ProxyError(xmlrpclib.ProtocolError(host + handler, 408, diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 3b2c0ccfa..aea5e9d4b 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -197,7 +197,8 @@ class DjangoORM(StorageBase): def _import_Service(self, entry, state): return self._import_default(entry, state, defaults=dict(status='', - current_status=''), + current_status='', + target_status=''), mapping=dict(status='target_status')) def _import_SEBoolean(self, entry, state): diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index e63c180a8..598e1c6ec 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -3,7 +3,7 @@ import sys from django.core.exceptions import ImproperlyConfigured try: - from django.db import models + from django.db import models, backend, connection except ImproperlyConfigured: e = sys.exc_info()[1] print("Reports: unable to import django models: %s" % e) @@ -26,6 +26,8 @@ TYPE_CHOICES = ( (TYPE_EXTRA, 'Extra'), ) +_our_backend = None + def convert_entry_type_to_id(type_name): """Convert a entry type to its entry id""" @@ -49,6 +51,22 @@ def hash_entry(entry_dict): return hash(cPickle.dumps(dataset)) +def _quote(value): + """ + Quote a string to use as a table name or column + + Newer versions and various drivers require an argument + https://code.djangoproject.com/ticket/13630 + """ + global _our_backend + if not _our_backend: + try: + _our_backend = backend.DatabaseOperations(connection) + except TypeError: + _our_backend = backend.DatabaseOperations() + return _our_backend.quote_name(value) + + class Client(models.Model): """Object representing every client we have seen stats for.""" creation = models.DateTimeField(auto_now_add=True) @@ -77,16 +95,20 @@ class InteractionManager(models.Manager): cursor = connection.cursor() cfilter = "expiration is null" - sql = 'select ri.id, x.client_id from (select client_id, MAX(timestamp) ' + \ - 'as timer from Reporting_interaction' + sql = 'select ri.id, x.client_id from ' + \ + '(select client_id, MAX(timestamp) as timer from ' + \ + _quote('Reporting_interaction') if maxdate: if not isinstance(maxdate, datetime): raise ValueError('Expected a datetime object') sql = sql + " where timestamp <= '%s' " % maxdate cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate) - sql = sql + ' GROUP BY client_id) x, Reporting_interaction ri where ' + \ - 'ri.client_id = x.client_id AND ri.timestamp = x.timer' - sql = sql + " and x.client_id in (select id from Reporting_client where %s)" % cfilter + sql = sql + ' GROUP BY client_id) x, ' + \ + _quote('Reporting_interaction') + \ + ' ri where ri.client_id = x.client_id AND' + \ + ' ri.timestamp = x.timer and x.client_id in' + \ + ' (select id from %s where %s)' % \ + (_quote('Reporting_client'), cfilter) try: cursor.execute(sql) return [item[0] for item in cursor.fetchall()] @@ -95,7 +117,6 @@ class InteractionManager(models.Manager): pass return [] - def recent(self, maxdate=None): """ Returns the most recent interactions for clients as of a date @@ -115,7 +136,7 @@ class Interaction(models.Model): timestamp = models.DateTimeField(db_index=True) # Timestamp for this record state = models.CharField(max_length=32) # good/bad/modified/etc repo_rev_code = models.CharField(max_length=64) # repo revision at time of interaction - server = models.CharField(max_length=256) # Name of the server used for the interaction + server = models.CharField(max_length=256) # server used for interaction good_count = models.IntegerField() # of good config-items total_count = models.IntegerField() # of total config-items bad_count = models.IntegerField(default=0) @@ -211,18 +232,24 @@ class Interaction(models.Model): def bad(self): rv = [] for entry in self.entry_types: + if entry == 'failures': + continue rv.extend(getattr(self, entry).filter(state=TYPE_BAD)) return rv def modified(self): rv = [] for entry in self.entry_types: + if entry == 'failures': + continue rv.extend(getattr(self, entry).filter(state=TYPE_MODIFIED)) return rv def extra(self): rv = [] for entry in self.entry_types: + if entry == 'failures': + continue rv.extend(getattr(self, entry).filter(state=TYPE_EXTRA)) return rv @@ -234,7 +261,8 @@ class Interaction(models.Model): class Performance(models.Model): """Object representing performance data for any interaction.""" - interaction = models.ForeignKey(Interaction, related_name="performance_items") + interaction = models.ForeignKey(Interaction, + related_name="performance_items") metric = models.CharField(max_length=128) value = models.DecimalField(max_digits=32, decimal_places=16) @@ -267,11 +295,11 @@ class Group(models.Model): class Meta: ordering = ('name',) - @staticmethod def prune_orphans(): '''Prune unused groups''' - Group.objects.filter(interaction__isnull=True, group__isnull=True).delete() + Group.objects.filter(interaction__isnull=True, + group__isnull=True).delete() class Bundle(models.Model): @@ -289,11 +317,11 @@ class Bundle(models.Model): class Meta: ordering = ('name',) - @staticmethod def prune_orphans(): '''Prune unused bundles''' - Bundle.objects.filter(interaction__isnull=True, group__isnull=True).delete() + Bundle.objects.filter(interaction__isnull=True, + group__isnull=True).delete() # new interaction models @@ -396,7 +424,7 @@ class BaseEntry(models.Model): def prune_orphans(cls): '''Remove unused entries''' # yeat another sqlite hack - cls_orphans = [x['id'] \ + cls_orphans = [x['id'] for x in cls.objects.filter(interaction__isnull=True).values("id")] i = 0 while i < len(cls_orphans): @@ -671,7 +699,7 @@ class PathEntry(SuccessEntry): acls = models.ManyToManyField(FileAcl) detail_type = models.IntegerField(default=0, - choices=DETAIL_CHOICES) + choices=DETAIL_CHOICES) details = models.TextField(default='') ENTRY_TYPE = r"Path" diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index 990bcd512..316c2f86c 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -281,7 +281,10 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): raise except socket.error: err = sys.exc_info()[1] - if err[0] == 32: + if isinstance(err, socket.timeout): + self.logger.warning("Connection timed out for %s" % + self.client_address[0]) + elif err[0] == 32: self.logger.warning("Connection dropped from %s" % self.client_address[0]) elif err[0] == 104: @@ -337,7 +340,7 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, :param register: Presence should be reported to service-location :type register: bool :param allow_none: Allow None values in XML-RPC - :type allow_non: bool + :type allow_none: bool :param encoding: Encoding to use for XML-RPC """ @@ -414,7 +417,9 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, def serve_forever(self): """Serve single requests until (self.serve == False).""" self.serve = True - self.task_thread = threading.Thread(target=self._tasks_thread) + self.task_thread = \ + threading.Thread(name="%sThread" % self.__class__.__name__, + target=self._tasks_thread) self.task_thread.start() self.logger.info("serve_forever() [start]") signal.signal(signal.SIGINT, self._handle_shutdown_signal) diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py index b7916fab9..187ccfd71 100644 --- a/src/lib/Bcfg2/Server/Admin/Client.py +++ b/src/lib/Bcfg2/Server/Admin/Client.py @@ -8,6 +8,7 @@ from Bcfg2.Server.Plugin import MetadataConsistencyError class Client(Bcfg2.Server.Admin.MetadataCore): """ Create, delete, or list client entries """ __usage__ = "[options] [add|del|list] [attr=val]" + __plugin_whitelist__ = ["Metadata"] def __call__(self, args): if len(args) == 0: @@ -17,19 +18,15 @@ class Client(Bcfg2.Server.Admin.MetadataCore): try: self.metadata.add_client(args[1]) except MetadataConsistencyError: - err = sys.exc_info()[1] - print("Error in adding client: %s" % err) - raise SystemExit(1) + self.errExit("Error in adding client: %s" % sys.exc_info()[1]) elif args[0] in ['delete', 'remove', 'del', 'rm']: try: self.metadata.remove_client(args[1]) except MetadataConsistencyError: - err = sys.exc_info()[1] - print("Error in deleting client: %s" % err) - raise SystemExit(1) + self.errExit("Error in deleting client: %s" % + sys.exc_info()[1]) elif args[0] in ['list', 'ls']: for client in self.metadata.list_clients(): print(client) else: - print("No command specified") - raise SystemExit(1) + self.errExit("No command specified") diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py index c56dd0a8f..e3648a6d0 100644 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ b/src/lib/Bcfg2/Server/Admin/Compare.py @@ -145,5 +145,4 @@ class Compare(Bcfg2.Server.Admin.Mode): (old, new) = args return self.compareSpecifications(new, old) except IndexError: - print(self.__call__.__doc__) - raise SystemExit(1) + self.errExit(self.__call__.__doc__) diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index 93e42305c..37ca74894 100644 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -20,9 +20,8 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): "Please see bcfg2-admin minestruct help for usage.") try: (opts, args) = getopt.getopt(args, 'f:g:h') - except: - self.log.error(self.__doc__) - raise SystemExit(1) + except getopt.GetoptError: + self.errExit(self.__doc__) client = args[0] output = sys.stdout @@ -33,8 +32,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): try: output = open(optarg, 'w') except IOError: - self.log.error("Failed to open file: %s" % (optarg)) - raise SystemExit(1) + self.errExit("Failed to open file: %s" % (optarg)) elif opt == '-g': groups = optarg.split(':') @@ -43,10 +41,9 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): for source in self.bcore.plugins_by_type(PullSource): for item in source.GetExtra(client): extra.add(item) - except: - self.log.error("Failed to find extra entry info for client %s" % - client) - raise SystemExit(1) + except: # pylint: disable=W0702 + self.errExit("Failed to find extra entry info for client %s" % + client) root = lxml.etree.Element("Base") self.log.info("Found %d extra entries" % (len(extra))) add_point = root diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index 8001425df..459fcec65 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -32,9 +32,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): use_stdin = False try: opts, gargs = getopt.getopt(args, 'vfIs') - except: - print(self.__doc__) - raise SystemExit(1) + except getopt.GetoptError: + self.errExit(self.__doc__) for opt in opts: if opt[0] == '-v': self.log = True diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index 6e313e84b..849df8025 100644 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -79,8 +79,7 @@ class Reports(Bcfg2.Server.Admin.Mode): def __call__(self, args): if len(args) == 0 or args[0] == '-h': - print(self.__usage__) - raise SystemExit(0) + self.errExit(self.__usage__) # FIXME - dry run @@ -101,9 +100,7 @@ class Reports(Bcfg2.Server.Admin.Mode): management.call_command("syncdb", verbosity=vrb) management.call_command("migrate", verbosity=vrb) except: - print("Update failed: %s" % - traceback.format_exc().splitlines()[-1]) - raise SystemExit(1) + self.errExit("Update failed: %s" % sys.exc_info()[1]) elif args[0] == 'purge': expired = False client = None @@ -124,22 +121,20 @@ class Reports(Bcfg2.Server.Admin.Mode): maxdate = datetime.datetime.now() - \ datetime.timedelta(days=int(args[i + 1])) except: - self.log.error("Invalid number of days: %s" % - args[i + 1]) - raise SystemExit(-1) + self.errExit("Invalid number of days: %s" % + args[i + 1]) i = i + 1 elif args[i] == '--expired': expired = True i = i + 1 if expired: if state: - self.log.error("--state is not valid with --expired") - raise SystemExit(-1) + self.errExit("--state is not valid with --expired") self.purge_expired(maxdate) else: self.purge(client, maxdate, state) else: - print("Unknown command: %s" % args[0]) + self.errExit("Unknown command: %s" % args[0]) @transaction.commit_on_success def scrub(self): @@ -155,8 +150,7 @@ class Reports(Bcfg2.Server.Admin.Mode): (start_count - cls.objects.count(), cls.__class__.__name__)) except: print("Failed to prune %s: %s" % - (cls.__class__.__name__, - traceback.format_exc().splitlines()[-1])) + (cls.__class__.__name__, sys.exc_info()[1])) def django_command_proxy(self, command): '''Call a django command''' @@ -180,8 +174,7 @@ class Reports(Bcfg2.Server.Admin.Mode): cobj = Client.objects.get(name=client) ipurge = ipurge.filter(client=cobj) except Client.DoesNotExist: - self.log.error("Client %s not in database" % client) - raise SystemExit(-1) + self.errExit("Client %s not in database" % client) self.log.debug("Filtering by client: %s" % client) if maxdate: diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py index 4ba840b86..53cfd1bec 100644 --- a/src/lib/Bcfg2/Server/Admin/Syncdb.py +++ b/src/lib/Bcfg2/Server/Admin/Syncdb.py @@ -22,10 +22,7 @@ class Syncdb(Bcfg2.Server.Admin.Mode): call_command("syncdb", interactive=False, verbosity=0) self._database_available = True except ImproperlyConfigured: - err = sys.exc_info()[1] - self.log.error("Django configuration problem: %s" % err) - raise SystemExit(1) + self.errExit("Django configuration problem: %s" % + sys.exc_info()[1]) except: - err = sys.exc_info()[1] - self.log.error("Database update failed: %s" % err) - raise SystemExit(1) + self.errExit("Database update failed: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py index be556bed4..036129a1b 100644 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py @@ -4,7 +4,6 @@ import sys import Bcfg2.Options import Bcfg2.Proxy import Bcfg2.Server.Admin -from Bcfg2.Compat import xmlrpclib class Xcmd(Bcfg2.Server.Admin.Mode): @@ -31,27 +30,15 @@ class Xcmd(Bcfg2.Server.Admin.Mode): ca=setup['ca'], timeout=setup['timeout']) if len(setup['args']) == 0: - print("Usage: xcmd <xmlrpc method> <optional arguments>") - return + self.errExit("Usage: xcmd <xmlrpc method> <optional arguments>") cmd = setup['args'][0] args = () if len(setup['args']) > 1: args = tuple(setup['args'][1:]) try: data = getattr(proxy, cmd)(*args) - except xmlrpclib.Fault: - flt = sys.exc_info()[1] - if flt.faultCode == 7: - print("Unknown method %s" % cmd) - return - elif flt.faultCode == 20: - return - else: - raise except Bcfg2.Proxy.ProxyError: - err = sys.exc_info()[1] - print("Proxy Error: %s" % err) - return + self.errExit("Proxy Error: %s" % sys.exc_info()[1]) if data is not None: print(data) diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py index 7bba05eb3..8f12a940e 100644 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -128,6 +128,7 @@ class MetadataCore(Mode): except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] self.errExit("Core load failed: %s" % msg) + self.bcore.load_plugins() self.bcore.fam.handle_event_set() self.metadata = self.bcore.metadata diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index c3302f1d0..e69a92b64 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -117,11 +117,11 @@ class Core(BaseCore): self.logger.error("Server startup failed: %s" % err) self.context.close() return False - self.server.register_instance(self) return True def _block(self): """ Enter the blocking infinite loop. """ + self.server.register_instance(self) try: self.server.serve_forever() finally: diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 59d67e566..6e0d38418 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -301,6 +301,7 @@ class BaseCore(object): self.logger.info("Performance statistics: " "%s min=%.06f, max=%.06f, average=%.06f, " "count=%d" % ((name, ) + stats)) + self.logger.debug("Performance logging thread terminated") def _file_monitor_thread(self): """ The thread that runs the @@ -321,6 +322,7 @@ class BaseCore(object): except: continue self._update_vcs_revision() + self.logger.debug("File monitor thread terminated") @track_statistics() def _update_vcs_revision(self): @@ -440,8 +442,10 @@ class BaseCore(object): if not self.terminate.isSet(): self.terminate.set() self.fam.shutdown() + self.logger.debug("FAM shut down") for plugin in list(self.plugins.values()): plugin.shutdown() + self.logger.debug("All plugins shut down") @property def metadata_cache_mode(self): @@ -776,7 +780,13 @@ class BaseCore(object): self.shutdown() raise - self.set_debug(None, self.debug_flag) + if self.setup['fam_blocking']: + time.sleep(1) + while self.fam.pending() != 0: + time.sleep(1) + + if self.debug_flag: + self.set_debug(None, self.debug_flag) self._block() def _daemonize(self): diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index 8bfb76461..7c3b2d9cc 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -1,8 +1,9 @@ -""" check files for various required comments """ +""" Check files for various required comments. """ import os import lxml.etree import Bcfg2.Server.Lint +from Bcfg2.Server import XI_NAMESPACE from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ import CfgPlaintextGenerator from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator @@ -11,7 +12,10 @@ from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML class Comments(Bcfg2.Server.Lint.ServerPlugin): - """ check files for various required headers """ + """ The Comments lint plugin checks files for header comments that + give information about the files. For instance, you can require + SVN keywords in a comment, or require the name of the maintainer + of a Genshi template, and so on. """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.config_cache = {} @@ -27,21 +31,43 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): def Errors(cls): return {"unexpanded-keywords": "warning", "keywords-not-found": "warning", - "comments-not-found": "warning"} + "comments-not-found": "warning", + "broken-xinclude-chain": "warning"} def required_keywords(self, rtype): - """ given a file type, fetch the list of required VCS keywords - from the bcfg2-lint config """ + """ Given a file type, fetch the list of required VCS keywords + from the bcfg2-lint config. Valid file types are documented + in :manpage:`bcfg2-lint.conf(5)`. + + :param rtype: The file type + :type rtype: string + :returns: list - the required items + """ return self.required_items(rtype, "keyword") def required_comments(self, rtype): - """ given a file type, fetch the list of required comments - from the bcfg2-lint config """ + """ Given a file type, fetch the list of required comments + from the bcfg2-lint config. Valid file types are documented + in :manpage:`bcfg2-lint.conf(5)`. + + :param rtype: The file type + :type rtype: string + :returns: list - the required items + """ return self.required_items(rtype, "comment") def required_items(self, rtype, itype): - """ given a file type and item type (comment or keyword), - fetch the list of required items from the bcfg2-lint config """ + """ Given a file type and item type (``comment`` or + ``keyword``), fetch the list of required items from the + bcfg2-lint config. Valid file types are documented in + :manpage:`bcfg2-lint.conf(5)`. + + :param rtype: The file type + :type rtype: string + :param itype: The item type (``comment`` or ``keyword``) + :type itype: string + :returns: list - the required items + """ if itype not in self.config_cache: self.config_cache[itype] = {} @@ -62,7 +88,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): return self.config_cache[itype][rtype] def check_bundles(self): - """ check bundle files for required headers """ + """ Check bundle files for required comments. """ if 'Bundler' in self.core.plugins: for bundle in self.core.plugins['Bundler'].entries.values(): xdata = None @@ -78,15 +104,41 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_xml(bundle.name, xdata, rtype) def check_properties(self): - """ check properties files for required headers """ + """ Check Properties files for required comments. """ if 'Properties' in self.core.plugins: props = self.core.plugins['Properties'] - for propfile, pdata in props.store.entries.items(): + for propfile, pdata in props.entries.items(): if os.path.splitext(propfile)[1] == ".xml": self.check_xml(pdata.name, pdata.xdata, 'properties') + def has_all_xincludes(self, mfile): + """ Return True if :attr:`Bcfg2.Server.Lint.Plugin.files` + includes all XIncludes listed in the specified metadata type, + false otherwise. In other words, this returns True if + bcfg2-lint is dealing with complete metadata. + + :param mfile: The metadata file ("clients.xml" or + "groups.xml") to check for XIncludes + :type mfile: string + :returns: bool + """ + if self.files is None: + return True + else: + path = os.path.join(self.metadata.data, mfile) + if path in self.files: + xdata = lxml.etree.parse(path) + for el in xdata.findall('./%sinclude' % XI_NAMESPACE): + if not self.has_all_xincludes(el.get('href')): + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: could not " + "include %s" % path) + return False + + return True + def check_metadata(self): - """ check metadata files for required headers """ + """ Check Metadata files for required comments. """ if self.has_all_xincludes("groups.xml"): self.check_xml(os.path.join(self.metadata.data, "groups.xml"), self.metadata.groups_xml.data, @@ -97,7 +149,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): "metadata") def check_cfg(self): - """ check Cfg files and info.xml files for required headers """ + """ Check Cfg files and ``info.xml`` files for required + comments. """ if 'Cfg' in self.core.plugins: for entryset in self.core.plugins['Cfg'].entries.values(): for entry in entryset.entries.values(): @@ -117,29 +170,57 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_plaintext(entry.name, entry.data, rtype) def check_probes(self): - """ check probes for required headers """ + """ Check Probes for required comments """ if 'Probes' in self.core.plugins: for probe in self.core.plugins['Probes'].probes.entries.values(): self.check_plaintext(probe.name, probe.data, "probes") def check_xml(self, filename, xdata, rtype): - """ check generic XML files for required headers """ + """ Generic check to check an XML file for required comments. + + :param filename: The filename + :type filename: string + :param xdata: The file data + :type xdata: lxml.etree._Element + :param rtype: The type of file. Available types are + documented in :manpage:`bcfg2-lint.conf(5)`. + :type rtype: string + """ self.check_lines(filename, [str(el) for el in xdata.getiterator(lxml.etree.Comment)], rtype) def check_plaintext(self, filename, data, rtype): - """ check generic plaintext files for required headers """ + """ Generic check to check a plain text file for required + comments. + + :param filename: The filename + :type filename: string + :param data: The file data + :type data: string + :param rtype: The type of file. Available types are + documented in :manpage:`bcfg2-lint.conf(5)`. + :type rtype: string + """ self.check_lines(filename, data.splitlines(), rtype) def check_lines(self, filename, lines, rtype): - """ generic header check for a set of lines """ + """ Generic header check for a set of lines. + + :param filename: The filename + :type filename: string + :param lines: The data to check + :type lines: list of strings + :param rtype: The type of file. Available types are + documented in :manpage:`bcfg2-lint.conf(5)`. + :type rtype: string + """ if self.HandlesFile(filename): # found is trivalent: - # False == not found - # None == found but not expanded - # True == found and expanded + # False == keyword not found + # None == keyword found but not expanded + # True == keyword found and expanded found = dict((k, False) for k in self.required_keywords(rtype)) for line in lines: diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index c045c2ca2..1ecb6da42 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -1,4 +1,4 @@ -""" Check Genshi templates for syntax errors """ +""" Check Genshi templates for syntax errors. """ import sys import Bcfg2.Server.Lint @@ -9,10 +9,9 @@ from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator class Genshi(Bcfg2.Server.Lint.ServerPlugin): - """ Check Genshi templates for syntax errors """ + """ Check Genshi templates for syntax errors. """ def Run(self): - """ run plugin """ if 'Cfg' in self.core.plugins: self.check_cfg() if 'TGenshi' in self.core.plugins: @@ -25,7 +24,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): return {"genshi-syntax-error": "error"} def check_cfg(self): - """ Check genshi templates in Cfg for syntax errors """ + """ Check genshi templates in Cfg for syntax errors. """ for entryset in self.core.plugins['Cfg'].entries.values(): for entry in entryset.entries.values(): if (self.HandlesFile(entry.name) and @@ -38,9 +37,15 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): err = sys.exc_info()[1] self.LintError("genshi-syntax-error", "Genshi syntax error: %s" % err) + except: + etype, err = sys.exc_info()[:2] + self.LintError( + "genshi-syntax-error", + "Unexpected Genshi error on %s: %s: %s" % + (entry.name, etype.__name__, err)) def check_tgenshi(self): - """ Check templates in TGenshi for syntax errors """ + """ Check templates in TGenshi for syntax errors. """ loader = TemplateLoader() for eset in self.core.plugins['TGenshi'].entries.values(): @@ -54,7 +59,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): "Genshi syntax error: %s" % err) def check_bundler(self): - """ Check templates in Bundler for syntax errors """ + """ Check templates in Bundler for syntax errors. """ loader = TemplateLoader() for entry in self.core.plugins['Bundler'].entries.values(): diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py index 52e42aa7b..b180083d5 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupNames.py +++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py @@ -1,4 +1,4 @@ -""" ensure that all named groups are valid group names """ +""" Ensure that all named groups are valid group names. """ import os import re @@ -11,8 +11,15 @@ except ImportError: class GroupNames(Bcfg2.Server.Lint.ServerPlugin): - """ ensure that all named groups are valid group names """ + """ Ensure that all named groups are valid group names. """ + + #: A string regex that matches only valid group names. Currently, + #: a group name is considered valid if it contains only + #: non-whitespace characters. pattern = r'\S+$' + + #: A compiled regex for + #: :attr:`Bcfg2.Server.Lint.GroupNames.GroupNames.pattern` valid = re.compile(r'^' + pattern) def Run(self): @@ -31,7 +38,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): return {"invalid-group-name": "error"} def check_rules(self): - """ Check groups used in the Rules plugin for validity """ + """ Check groups used in the Rules plugin for validity. """ for rules in self.core.plugins['Rules'].entries.values(): if not self.HandlesFile(rules.name): continue @@ -40,7 +47,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): os.path.join(self.config['repo'], rules.name)) def check_bundles(self): - """ Check groups used in the Bundler plugin for validity """ + """ Check groups used in the Bundler plugin for validity. """ for bundle in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(bundle.name) and (not HAS_GENSHI or @@ -50,7 +57,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): def check_metadata(self): """ Check groups used or declared in the Metadata plugin for - validity """ + validity. """ self.check_entries(self.metadata.groups_xml.xdata.xpath("//Group"), os.path.join(self.config['repo'], self.metadata.groups_xml.name)) @@ -68,7 +75,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): def check_cfg(self): """ Check groups used in group-specific files in the Cfg - plugin for validity """ + plugin for validity. """ for root, _, files in os.walk(self.core.plugins['Cfg'].data): for fname in files: basename = os.path.basename(root) @@ -81,7 +88,14 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): def check_entries(self, entries, fname): """ Check a generic list of XML entries for <Group> tags with - invalid name attributes """ + invalid name attributes. + + :param entries: A list of XML <Group> tags whose ``name`` + attributes will be validated. + :type entries: list of lxml.etree._Element + :param fname: The filename the entry list came from + :type fname: string + """ for grp in entries: if not self.valid.search(grp.get("name")): self.LintError("invalid-group-name", diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index e34f387ff..95657317e 100644 --- a/src/lib/Bcfg2/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -1,4 +1,4 @@ -""" ensure that all config files have an info.xml file""" +""" Ensure that all config files have a valid info.xml file. """ import os import Bcfg2.Options @@ -8,7 +8,14 @@ from Bcfg2.Server.Plugins.Cfg.CfgLegacyInfo import CfgLegacyInfo class InfoXML(Bcfg2.Server.Lint.ServerPlugin): - """ ensure that all config files have an info.xml file""" + """ Ensure that all config files have a valid info.xml file. This + plugin can check for: + + * Missing ``info.xml`` files; + * Use of deprecated ``info``/``:info`` files; + * Paranoid mode disabled in an ``info.xml`` file; + * Required attributes missing from ``info.xml`` + """ def Run(self): if 'Cfg' not in self.core.plugins: return @@ -40,11 +47,10 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): return {"no-infoxml": "warning", "deprecated-info-file": "warning", "paranoid-false": "warning", - "broken-xinclude-chain": "warning", "required-infoxml-attrs-missing": "error"} def check_infoxml(self, fname, xdata): - """ verify that info.xml contains everything it should """ + """ Verify that info.xml contains everything it should. """ for info in xdata.getroottree().findall("//Info"): required = [] if "required_attrs" in self.config: diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index 44d02c2ff..2419c3d43 100644 --- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -57,7 +57,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): else: threshold = 0.75 rv = [] - elist = entries.items() + elist = list(entries.items()) while elist: result = self._find_similar(elist.pop(0), copy.copy(elist), threshold) diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 40ff71dbd..6ffdd33a0 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -1,5 +1,5 @@ -""" verify attributes for configuration entries that cannot be -verified with an XML schema alone""" +""" Verify attributes for configuration entries that cannot be +verified with an XML schema alone. """ import os import re @@ -15,7 +15,8 @@ except ImportError: HAS_GENSHI = False -# format verifying functions +# format verifying functions. TODO: These should be moved into XML +# schemas where possible. def is_filename(val): """ Return True if val is a string describing a valid full path """ @@ -53,8 +54,8 @@ def is_device_mode(val): class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): - """ verify attributes for configuration entries that cannot be - verified with an XML schema alone """ + """ Verify attributes for configuration entries that cannot be + verified with an XML schema alone. """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.required_attrs = dict( @@ -115,8 +116,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): SEInterface={None: dict(name=None, selinuxtype=is_selinux_type)}, SEPermissive={None: dict(name=is_selinux_type)}, POSIXGroup={None: dict(name=is_username)}, - POSIXUser={None: dict(name=is_username)}, - MemberOf={None: dict(__text__=is_username)}) + POSIXUser={None: dict(name=is_username)}) def Run(self): self.check_packages() @@ -136,7 +136,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): "extra-attrs": "warning"} def check_packages(self): - """ check package sources for Source entries with missing attrs """ + """ Check Packages sources for Source entries with missing + attributes. """ if 'Packages' not in self.core.plugins: return @@ -176,7 +177,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): rules.name)) def check_bundles(self): - """ check bundles for BoundPath entries with missing attrs """ + """ Check bundles for BoundPath entries with missing + attrs. """ if 'Bundler' not in self.core.plugins: return @@ -195,7 +197,13 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): self.check_entry(path, bundle.name) def check_entry(self, entry, filename): - """ generic entry check """ + """ Generic entry check. + + :param entry: The XML entry to check for missing attributes. + :type entry: lxml.etree._Element + :param filename: The filename the entry came from + :type filename: string + """ if self.HandlesFile(filename): name = entry.get('name') tag = entry.tag diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index ae7c75804..09f3f3d25 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -1,4 +1,5 @@ -""" Ensure that the repo validates """ +""" Ensure that all XML files in the Bcfg2 repository validate +according to their respective schemas. """ import os import sys @@ -10,10 +11,19 @@ import Bcfg2.Server.Lint class Validate(Bcfg2.Server.Lint.ServerlessPlugin): - """ Ensure that the repo validates """ + """ Ensure that all XML files in the Bcfg2 repository validate + according to their respective schemas. """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + + #: A dict of <file glob>: <schema file> that maps files in the + #: Bcfg2 specification to their schemas. The globs are + #: extended :mod:`fnmatch` globs that also support ``**``, + #: which matches any number of any characters, including + #: forward slashes. The schema files are relative to the + #: schema directory, which can be controlled by the + #: ``bcfg2-lint --schema`` option. self.filesets = \ {"Metadata/groups.xml": "metadata.xsd", "Metadata/clients.xml": "clients.xsd", @@ -76,7 +86,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "input-output-error": "error"} def check_properties(self): - """ check Properties files against their schemas """ + """ Check Properties files against their schemas. """ for filename in self.filelists['props']: schemafile = "%s.xsd" % os.path.splitext(filename)[0] if os.path.exists(schemafile): @@ -90,7 +100,11 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): def parse(self, filename): """ Parse an XML file, raising the appropriate LintErrors if it can't be parsed or read. Return the - lxml.etree._ElementTree parsed from the file. """ + lxml.etree._ElementTree parsed from the file. + + :param filename: The full path to the file to parse + :type filename: string + :returns: lxml.etree._ElementTree - the parsed data""" try: return lxml.etree.parse(filename) except SyntaxError: @@ -106,8 +120,20 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): return False def validate(self, filename, schemafile, schema=None): - """validate a file against the given lxml.etree.Schema. - return True on success, False on failure """ + """ Validate a file against the given schema. + + :param filename: The full path to the file to validate + :type filename: string + :param schemafile: The full path to the schema file to + validate against + :type schemafile: string + :param schema: The loaded schema to validate against. This + can be used to avoid parsing a single schema + file for every file that needs to be validate + against it. + :type schema: lxml.etree.Schema + :returns: bool - True if the file validates, false otherwise + """ if schema is None: # if no schema object was provided, instantiate one schema = self._load_schema(schemafile) @@ -121,6 +147,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): cmd.extend(["--noout", "--schema", schemafile, filename]) lint = Popen(cmd, stdout=PIPE, stderr=STDOUT) output = lint.communicate()[0] + # py3k fix + if not isinstance(output, str): + output = output.decode('utf-8') if lint.wait(): self.LintError("xml-failed-to-verify", "%s fails to verify:\n%s" % (filename, output)) @@ -128,7 +157,14 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): return True def get_filelists(self): - """ get lists of different kinds of files to validate """ + """ Get lists of different kinds of files to validate. This + doesn't return anything, but it sets + :attr:`Bcfg2.Server.Lint.Validate.Validate.filelists` to a + dict whose keys are path globs given in + :attr:`Bcfg2.Server.Lint.Validate.Validate.filesets` and whose + values are lists of the full paths to all files in the Bcfg2 + repository (or given with ``bcfg2-lint --stdin``) that match + the glob.""" if self.files is not None: listfiles = lambda p: fnmatch.filter(self.files, os.path.join('*', p)) @@ -155,7 +191,13 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.filelists['props'] = listfiles("Properties/*.xml") def _load_schema(self, filename): - """ load an XML schema document, returning the Schema object """ + """ Load an XML schema document, returning the Schema object + and raising appropriate lint errors on failure. + + :param filename: The full path to the schema file to load. + :type filename: string + :returns: lxml.etree.Schema - The loaded schema data + """ try: return lxml.etree.XMLSchema(lxml.etree.parse(filename)) except IOError: diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 11afdd75d..28644263f 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -9,10 +9,9 @@ import lxml.etree import fcntl import termios import struct -from Bcfg2.Server import XI_NAMESPACE from Bcfg2.Compat import walk_packages -__all__ = [m[1] for m in walk_packages(path=__path__)] +plugins = [m[1] for m in walk_packages(path=__path__)] # pylint: disable=C0103 def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -45,30 +44,56 @@ def get_termsize(): class Plugin(object): - """ base class for ServerlessPlugin and ServerPlugin """ + """ Base class for all bcfg2-lint plugins """ def __init__(self, config, errorhandler=None, files=None): + """ + :param config: A :mod:`Bcfg2.Options` setup dict + :type config: dict + :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` + that will be used to handle lint errors. + If one is not provided, a new one will be + instantiated. + :type errorhandler: Bcfg2.Server.Lint.ErrorHandler + :param files: A list of files to run bcfg2-lint against. (See + the bcfg2-lint ``--stdin`` option.) + :type files: list of strings + """ + + #: The list of files that bcfg2-lint should be run against self.files = files + + #: The Bcfg2.Options setup dict self.config = config + self.logger = logging.getLogger('bcfg2-lint') if errorhandler is None: + #: The error handler self.errorhandler = ErrorHandler() else: self.errorhandler = errorhandler self.errorhandler.RegisterErrors(self.Errors()) def Run(self): - """ run the plugin. must be overloaded by child classes """ - pass + """ Run the plugin. Must be overloaded by child classes. """ + raise NotImplementedError @classmethod def Errors(cls): - """ returns a dict of errors the plugin supplies. must be - overloaded by child classes """ + """ Returns a dict of errors the plugin supplies, in a format + suitable for passing to + :func:`Bcfg2.Server.Lint.ErrorHandler.RegisterErrors`. + + Must be overloaded by child classes. + + :returns: dict + """ + raise NotImplementedError def HandlesFile(self, fname): - """ returns true if the given file should be handled by the - plugin according to the files list, false otherwise """ + """ Returns True if the given file should be handled by the + plugin according to :attr:`Bcfg2.Server.Lint.Plugin.files`, + False otherwise. """ return (self.files is None or fname in self.files or os.path.join(self.config['repo'], fname) in self.files or @@ -77,12 +102,27 @@ class Plugin(object): fname)) in self.files) def LintError(self, err, msg): - """ record an error in the lint process """ + """ Raise an error from the lint process. + + :param err: The name of the error being raised. This name + must be a key in the dict returned by + :func:`Bcfg2.Server.Lint.Plugin.Errors`. + :type err: string + :param msg: The freeform message to display to the end user. + :type msg: string + """ self.errorhandler.dispatch(err, msg) def RenderXML(self, element, keep_text=False): - """render an XML element for error output -- line number - prefixed, no children""" + """ Render an XML element for error output. This prefixes the + line number and removes children for nicer display. + + :param element: The element to render + :type element: lxml.etree._Element + :param keep_text: Do not discard text content from the element + for display + :type keep_text: boolean + """ xml = None if len(element) or element.text: el = copy(element) @@ -100,11 +140,18 @@ class Plugin(object): return " line %s: %s" % (element.sourceline, xml) -class ErrorHandler (object): - """ a class to handle errors for bcfg2-lint plugins """ +class ErrorHandler(object): + """ A class to handle errors for bcfg2-lint plugins """ - def __init__(self, config=None): + def __init__(self, errors=None): + """ + :param config: An initial dict of errors to register + :type config: dict + """ + #: The number of errors passed to this error handler self.errors = 0 + + #: The number of warnings passed to this error handler self.warnings = 0 self.logger = logging.getLogger('bcfg2-lint') @@ -114,17 +161,25 @@ class ErrorHandler (object): twrap = textwrap.TextWrapper(initial_indent=" ", subsequent_indent=" ", width=termsize[0]) + #: A function to wrap text to the width of the terminal self._wrapper = twrap.wrap else: self._wrapper = lambda s: [s] + #: A dict of registered errors self.errortypes = dict() - if config is not None: - self.RegisterErrors(dict(config.items())) + if errors is not None: + self.RegisterErrors(dict(errors.items())) def RegisterErrors(self, errors): - """ Register a dict of errors (name: default level) that a - plugin may raise """ + """ Register a dict of errors that a plugin may raise. The + keys of the dict are short strings that describe each error; + the values are the default error handling for that error + ("error", "warning", or "silent"). + + :param errors: The error dict + :type errors: dict + """ for err, action in errors.items(): if err not in self.errortypes: if "warn" in action: @@ -135,7 +190,16 @@ class ErrorHandler (object): self.errortypes[err] = self.debug def dispatch(self, err, msg): - """ Dispatch an error to the correct handler """ + """ Dispatch an error to the correct handler. + + :param err: The name of the error being raised. This name + must be a key in + :attr:`Bcfg2.Server.Lint.ErrorHandler.errortypes`, + the dict of registered errors. + :type err: string + :param msg: The freeform message to display to the end user. + :type msg: string + """ if err in self.errortypes: self.errortypes[err](msg) self.logger.debug(" (%s)" % err) @@ -145,22 +209,34 @@ class ErrorHandler (object): self.logger.warning("Unknown error %s" % err) def error(self, msg): - """ log an error condition """ + """ Log an error condition. + + :param msg: The freeform message to display to the end user. + :type msg: string + """ self.errors += 1 self._log(msg, self.logger.error, prefix="ERROR: ") def warn(self, msg): - """ log a warning condition """ + """ Log a warning condition. + + :param msg: The freeform message to display to the end user. + :type msg: string + """ self.warnings += 1 self._log(msg, self.logger.warning, prefix="WARNING: ") def debug(self, msg): - """ log a silent/debug condition """ + """ Log a silent/debug condition. + + :param msg: The freeform message to display to the end user. + :type msg: string + """ self._log(msg, self.logger.debug) def _log(self, msg, logfunc, prefix=""): """ Generic log function that logs a message with the given - function after wrapping it for the terminal width """ + function after wrapping it for the terminal width. """ # a message may itself consist of multiple lines. wrap() will # elide them all into a single paragraph, which we don't want. # so we split the message into its paragraphs and wrap each @@ -180,37 +256,37 @@ class ErrorHandler (object): logfunc(line) -class ServerlessPlugin (Plugin): - """ base class for plugins that are run before the server starts - up (i.e., plugins that check things that may prevent the server - from starting up) """ +class ServerlessPlugin(Plugin): # pylint: disable=W0223 + """ Base class for bcfg2-lint plugins that are run before the + server starts up (i.e., plugins that check things that may prevent + the server from starting up). """ pass -class ServerPlugin (Plugin): - """ base class for plugins that check things that require the - running Bcfg2 server """ - def __init__(self, core, config, **kwargs): - Plugin.__init__(self, config, **kwargs) +class ServerPlugin(Plugin): # pylint: disable=W0223 + """ Base class for bcfg2-lint plugins that check things that + require the running Bcfg2 server. """ + + def __init__(self, core, config, errorhandler=None, files=None): + """ + :param core: The Bcfg2 server core + :type core: Bcfg2.Server.Core.BaseCore + :param config: A :mod:`Bcfg2.Options` setup dict + :type config: dict + :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` + that will be used to handle lint errors. + If one is not provided, a new one will be + instantiated. + :type errorhandler: Bcfg2.Server.Lint.ErrorHandler + :param files: A list of files to run bcfg2-lint against. (See + the bcfg2-lint ``--stdin`` option.) + :type files: list of strings + """ + Plugin.__init__(self, config, errorhandler=errorhandler, files=files) + + #: The server core self.core = core self.logger = self.core.logger - self.metadata = self.core.metadata - self.errorhandler.RegisterErrors({"broken-xinclude-chain": "warning"}) - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./%sinclude' % XI_NAMESPACE): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not " - "include %s" % path) - return False - - return True + #: The metadata plugin + self.metadata = self.core.metadata diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py new file mode 100644 index 000000000..81fba7092 --- /dev/null +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -0,0 +1,204 @@ +""" The multiprocessing server core is a reimplementation of the +:mod:`Bcfg2.Server.BuiltinCore` that uses the Python +:mod:`multiprocessing` library to offload work to multiple child +processes. As such, it requires Python 2.6+. +""" + +import threading +import lxml.etree +import multiprocessing +from Bcfg2.Compat import Queue +from Bcfg2.Server.Core import BaseCore, exposed +from Bcfg2.Server.BuiltinCore import Core as BuiltinCore + + +class DualEvent(object): + """ DualEvent is a clone of :class:`threading.Event` that + internally implements both :class:`threading.Event` and + :class:`multiprocessing.Event`. """ + + def __init__(self, threading_event=None, multiprocessing_event=None): + self._threading_event = threading_event or threading.Event() + self._multiproc_event = multiprocessing_event or \ + multiprocessing.Event() + if threading_event or multiprocessing_event: + # initialize internal flag to false, regardless of the + # state of either object passed in + self.clear() + + def is_set(self): + """ Return true if and only if the internal flag is true. """ + return self._threading_event.is_set() + + isSet = is_set + + def set(self): + """ Set the internal flag to true. """ + self._threading_event.set() + self._multiproc_event.set() + + def clear(self): + """ Reset the internal flag to false. """ + self._threading_event.clear() + self._multiproc_event.clear() + + def wait(self, timeout=None): + """ Block until the internal flag is true, or until the + optional timeout occurs. """ + return self._threading_event.wait(timeout=timeout) + + +class ChildCore(BaseCore): + """ A child process for :class:`Bcfg2.MultiprocessingCore.Core`. + This core builds configurations from a given + :class:`multiprocessing.Pipe`. Note that this is a full-fledged + server core; the only input it gets from the parent process is the + hostnames of clients to render. All other state comes from the + FAM. However, this core only is used to render configs; it doesn't + handle anything else (authentication, probes, etc.) because those + are all much faster. There's no reason that it couldn't handle + those, though, if the pipe communication "protocol" were made more + robust. """ + + #: How long to wait while polling for new clients to build. This + #: doesn't affect the speed with which a client is built, but + #: setting it too high will result in longer shutdown times, since + #: we only check for the termination event from the main process + #: every ``poll_wait`` seconds. + poll_wait = 5.0 + + def __init__(self, setup, pipe, terminate): + """ + :param setup: A Bcfg2 options dict + :type setup: Bcfg2.Options.OptionParser + :param pipe: The pipe to which client hostnames are added for + ChildCore objects to build configurations, and to + which client configurations are added after + having been built by ChildCore objects. + :type pipe: multiprocessing.Pipe + :param terminate: An event that flags ChildCore objects to shut + themselves down. + :type terminate: multiprocessing.Event + """ + BaseCore.__init__(self, setup) + + #: The pipe to which client hostnames are added for ChildCore + #: objects to build configurations, and to which client + #: configurations are added after having been built by + #: ChildCore objects. + self.pipe = pipe + + #: The :class:`multiprocessing.Event` that will be monitored + #: to determine when this child should shut down. + self.terminate = terminate + + def _daemonize(self): + return True + + def _run(self): + return True + + def _block(self): + while not self.terminate.isSet(): + try: + if self.pipe.poll(self.poll_wait): + if not self.metadata.use_database: + # handle FAM events, in case (for instance) the + # client has just been added to clients.xml, or a + # profile has just been asserted. but really, you + # should be using the metadata database if you're + # using this core. + self.fam.handle_events_in_interval(0.1) + client = self.pipe.recv() + self.logger.debug("Building configuration for %s" % client) + config = \ + lxml.etree.tostring(self.BuildConfiguration(client)) + self.logger.debug("Returning configuration for %s to main " + "process" % client) + self.pipe.send(config) + self.logger.debug("Returned configuration for %s to main " + "process" % client) + except KeyboardInterrupt: + break + self.shutdown() + + +class Core(BuiltinCore): + """ A multiprocessing core that delegates building the actual + client configurations to + :class:`Bcfg2.Server.MultiprocessingCore.ChildCore` objects. The + parent process doesn't build any children itself; all calls to + :func:`GetConfig` are delegated to children. All other calls are + handled by the parent process. """ + + #: How long to wait for a child process to shut down cleanly + #: before it is terminated. + shutdown_timeout = 10.0 + + def __init__(self, setup): + BuiltinCore.__init__(self, setup) + if setup['children'] is None: + setup['children'] = multiprocessing.cpu_count() + + #: A dict of child name -> one end of the + #: :class:`multiprocessing.Pipe` object used to communicate + #: with that child. (The child is given the other end of the + #: Pipe.) + self.pipes = dict() + + #: A queue that keeps track of which children are available to + #: render a configuration. A child is popped from the queue + #: when it starts to render a config, then it's pushed back on + #: when it's done. This lets us use a blocking call to + #: :func:`Queue.Queue.get` when waiting for an available + #: child. + self.available_children = Queue(maxsize=self.setup['children']) + + # sigh. multiprocessing was added in py2.6, which is when the + # camelCase methods for threading objects were deprecated in + # favor of the Pythonic under_score methods. So + # multiprocessing.Event *only* has is_set(), while + # threading.Event has *both* isSet() and is_set(). In order + # to make the core work with Python 2.4+, and with both + # multiprocessing and threading Event objects, we just + # monkeypatch self.terminate to have isSet(). + self.terminate = DualEvent(threading_event=self.terminate) + + def _run(self): + for cnum in range(self.setup['children']): + name = "Child-%s" % cnum + (mainpipe, childpipe) = multiprocessing.Pipe() + self.pipes[name] = mainpipe + self.logger.debug("Starting child %s" % name) + childcore = ChildCore(self.setup, childpipe, self.terminate) + child = multiprocessing.Process(target=childcore.run, name=name) + child.start() + self.logger.debug("Child %s started with PID %s" % (name, + child.pid)) + self.available_children.put(name) + return BuiltinCore._run(self) + + def shutdown(self): + BuiltinCore.shutdown(self) + for child in multiprocessing.active_children(): + self.logger.debug("Shutting down child %s" % child.name) + child.join(self.shutdown_timeout) + if child.is_alive(): + self.logger.error("Waited %s seconds to shut down %s, " + "terminating" % (self.shutdown_timeout, + child.name)) + child.terminate() + else: + self.logger.debug("Child %s shut down" % child.name) + self.logger.debug("All children shut down") + + @exposed + def GetConfig(self, address): + client = self.resolve_client(address)[0] + childname = self.available_children.get() + self.logger.debug("Building configuration on child %s" % childname) + pipe = self.pipes[childname] + pipe.send(client) + config = pipe.recv() + self.available_children.put_nowait(childname) + return config diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index b14968d77..81dc1d736 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -41,15 +41,15 @@ del DEFAULT_FILE_METADATA['configfile'] LOGGER = logging.getLogger(__name__) #: a compiled regular expression for parsing info and :info files -INFO_REGEX = re.compile(r'owner:(\s)*(?P<owner>\S+)|' + - r'group:(\s)*(?P<group>\S+)|' + - r'mode:(\s)*(?P<mode>\w+)|' + - r'secontext:(\s)*(?P<secontext>\S+)|' + - r'paranoid:(\s)*(?P<paranoid>\S+)|' + - r'sensitive:(\s)*(?P<sensitive>\S+)|' + - r'encoding:(\s)*(?P<encoding>\S+)|' + - r'important:(\s)*(?P<important>\S+)|' + - r'mtime:(\s)*(?P<mtime>\w+)|') +INFO_REGEX = re.compile(r'owner:\s*(?P<owner>\S+)|' + + r'group:\s*(?P<group>\S+)|' + + r'mode:\s*(?P<mode>\w+)|' + + r'secontext:\s*(?P<secontext>\S+)|' + + r'paranoid:\s*(?P<paranoid>\S+)|' + + r'sensitive:\s*(?P<sensitive>\S+)|' + + r'encoding:\s*(?P<encoding>\S+)|' + + r'important:\s*(?P<important>\S+)|' + + r'mtime:\s*(?P<mtime>\w+)') def bind_info(entry, metadata, infoxml=None, default=DEFAULT_FILE_METADATA): diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 5c5e3da0c..eef176cca 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -144,10 +144,10 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various bundle checks """ + """ Perform various :ref:`Bundler + <server-plugins-structures-bundler-index>` checks. """ def Run(self): - """ run plugin """ self.missing_bundles() for bundle in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(bundle.name) and @@ -161,7 +161,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): "inconsistent-bundle-name": "warning"} def missing_bundles(self): - """ find bundles listed in Metadata but not implemented in Bundler """ + """ Find bundles listed in Metadata but not implemented in + Bundler. """ if self.files is None: # when given a list of files on stdin, this check is # useless, so skip it @@ -180,7 +181,11 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): bundle) def bundle_names(self, bundle): - """ verify bundle name attribute matches filename """ + """ Verify bundle name attribute matches filename. + + :param bundle: The bundle to verify + :type bundle: Bcfg2.Server.Plugins.Bundler.BundleFile + """ try: xdata = lxml.etree.XML(bundle.data) except AttributeError: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index ffe93c25b..842202a9c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -10,6 +10,7 @@ import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.Lint +from itertools import chain from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ @@ -35,6 +36,24 @@ SETUP = None #: facility for passing it otherwise. CFG = None +_HANDLERS = [] + + +def handlers(): + """ A list of Cfg handler classes. Loading the handlers must + be done at run-time, not at compile-time, or it causes a + circular import and Bad Things Happen.""" + if not _HANDLERS: + for submodule in walk_packages(path=__path__, prefix=__name__ + "."): + mname = submodule[1].rsplit('.', 1)[-1] + module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, + mname) + hdlr = getattr(module, mname) + if issubclass(hdlr, CfgBaseFileMatcher): + _HANDLERS.append(hdlr) + _HANDLERS.sort(key=operator.attrgetter("__priority__")) + return _HANDLERS + class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, Bcfg2.Server.Plugin.Debuggable): @@ -459,7 +478,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, entry_type, encoding) Bcfg2.Server.Plugin.Debuggable.__init__(self) self.specific = None - self._handlers = None __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ def set_debug(self, debug): @@ -468,24 +486,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, entry.set_debug(debug) return rv - @property - def handlers(self): - """ A list of Cfg handler classes. Loading the handlers must - be done at run-time, not at compile-time, or it causes a - circular import and Bad Things Happen.""" - if self._handlers is None: - self._handlers = [] - for submodule in walk_packages(path=__path__, - prefix=__name__ + "."): - mname = submodule[1].rsplit('.', 1)[-1] - module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, - mname) - hdlr = getattr(module, mname) - if CfgBaseFileMatcher in hdlr.__mro__: - self._handlers.append(hdlr) - self._handlers.sort(key=operator.attrgetter("__priority__")) - return self._handlers - def handle_event(self, event): """ Dispatch a FAM event to :func:`entry_init` or the appropriate child handler object. @@ -502,7 +502,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # process a bogus changed event like a created return - for hdlr in self.handlers: + for hdlr in handlers(): if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -520,7 +520,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, return elif hdlr.ignore(event, basename=self.path): return - elif action == 'changed': + # we only get here if event.filename in self.entries, so handle + # created event like changed + elif action == 'changed' or action == 'created': self.entries[event.filename].handle_event(event) return elif action == 'deleted': @@ -886,12 +888,15 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): for basename, entry in list(self.core.plugins['Cfg'].entries.items()): self.check_delta(basename, entry) self.check_pubkey(basename, entry) + self.check_missing_files() @classmethod def Errors(cls): return {"cat-file-used": "warning", "diff-file-used": "warning", - "no-pubkey-xml": "warning"} + "no-pubkey-xml": "warning", + "unknown-cfg-files": "error", + "extra-cfg-files": "error"} def check_delta(self, basename, entry): """ check that no .cat or .diff files are in use """ @@ -925,3 +930,41 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): self.LintError("no-pubkey-xml", "%s has no corresponding pubkey.xml at %s" % (basename, pubkey)) + + def check_missing_files(self): + """ check that all files on the filesystem are known to Cfg """ + cfg = self.core.plugins['Cfg'] + + # first, collect ignore patterns from handlers + ignore = [] + for hdlr in handlers(): + ignore.extend(hdlr.__ignore__) + + # next, get a list of all non-ignored files on the filesystem + all_files = set() + for root, _, files in os.walk(cfg.data): + all_files.update(os.path.join(root, fname) + for fname in files + if not any(fname.endswith("." + i) + for i in ignore)) + + # next, get a list of all files known to Cfg + cfg_files = set() + for root, eset in cfg.entries.items(): + cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) + for fname in eset.entries.keys()) + + # finally, compare the two + unknown_files = all_files - cfg_files + extra_files = cfg_files - all_files + if unknown_files: + self.LintError( + "unknown-cfg-files", + "Files on the filesystem could not be understood by Cfg: %s" % + "; ".join(unknown_files)) + if extra_files: + self.LintError( + "extra-cfg-files", + "Cfg has entries for files that do not exist on the " + "filesystem: %s\nThis is probably a bug." % + "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index d816192aa..8e074118f 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -24,7 +24,11 @@ import sys import pwd import grp import Bcfg2.Client.XML -from Bcfg2.Compat import b64encode, oct_mode +try: + from Bcfg2.Compat import b64encode, oct_mode +except ImportError: + from base64 import b64encode + oct_mode = oct path = "%s" diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 8d1e50526..09685d972 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -129,7 +129,12 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): - """ bcfg2-lint plugin for GroupPatterns """ + """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns + <server-plugins-grouping-grouppatterns>` patterns for validity. + This is simply done by trying to create a + :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for + each pattern, and catching exceptions and presenting them as + ``bcfg2-lint`` errors.""" def Run(self): cfg = self.core.plugins['GroupPatterns'].config diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index bdf3b87fe..3b8361c76 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -557,6 +557,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, open(os.path.join(repo, cls.name, fname), "w").write(kwargs[aname]) + @property + def use_database(self): + """ Expose self._use_db publicly for use in + :class:`Bcfg2.Server.MultiprocessingCore.ChildCore` """ + return self._use_db + def _handle_file(self, fname): """ set up the necessary magic for handling a metadata file (clients.xml or groups.xml, e.g.) """ @@ -945,7 +951,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Client %s set as nonexistent group %s" % (client, group)) - def set_profile(self, client, profile, addresspair): + def set_profile(self, client, profile, # pylint: disable=W0221 + addresspair, require_public=True): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) @@ -957,7 +964,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] - if not group.is_public: + if require_public and not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) @@ -1128,7 +1135,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = self.default if pgroup: - self.set_profile(client, pgroup, (None, None)) + self.set_profile(client, pgroup, (None, None), + require_public=False) profile = _add_group(pgroup) else: msg = "Cannot add new client %s; no default group set" % client @@ -1477,7 +1485,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): - """ bcfg2-lint plugin for Metadata """ + """ ``bcfg2-lint`` plugin for :ref:`Metadata + <server-plugins-grouping-metadata>`. This checks for several things: + + * ``<Client>`` tags nested inside other ``<Client>`` tags; + * Deprecated options (like ``location="floating"``); + * Profiles that don't exist, or that aren't profile groups; + * Groups or clients that are defined multiple times; + * Multiple default groups or a default group that isn't a profile + group. + """ def Run(self): self.nested_clients() @@ -1500,8 +1517,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "default-is-not-profile": "error"} def deprecated_options(self): - """ check for the location='floating' option, which has been - deprecated in favor of floating='true' """ + """ Check for the ``location='floating'`` option, which has + been deprecated in favor of ``floating='true'``. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1519,8 +1536,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (loc, floating, self.RenderXML(el))) def nested_clients(self): - """ check for a Client tag inside a Client tag, which doesn't - make any sense """ + """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, + which is either redundant or will never match. """ groupdata = self.metadata.groups_xml.xdata for el in groupdata.xpath("//Client//Client"): self.LintError("nested-client-tags", @@ -1528,8 +1545,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (el.get("name"), self.RenderXML(el))) def bogus_profiles(self): - """ check for clients that have profiles that are either not - flagged as public groups in groups.xml, or don't exist """ + """ Check for clients that have profiles that are either not + flagged as profile groups in ``groups.xml``, or don't exist. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1547,20 +1564,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (profile, client.get("name"), profile, self.RenderXML(client))) - def duplicate_groups(self): - """ check for groups that are defined twice. We count a group - tag as a definition if it a) has profile or public set; or b) - has any children. """ - self.duplicate_entries( - self.metadata.groups_xml.xdata.xpath("//Groups/Group") + - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"), - "group", - include=lambda g: (g.get("profile") or - g.get("public") or - g.getchildren())) - def duplicate_default_groups(self): - """ check for multiple default groups """ + """ Check for multiple default groups. """ defaults = [] for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): @@ -1572,7 +1577,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "\n".join(defaults)) def duplicate_clients(self): - """ check for clients that are defined twice. """ + """ Check for clients that are defined more than once. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1580,17 +1585,34 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): self.metadata.clients_xml.xdata.xpath("//Client"), "client") - def duplicate_entries(self, allentries, etype, include=None): - """ generic duplicate entry finder """ - if include is None: - include = lambda e: True + def duplicate_groups(self): + """ Check for groups that are defined more than once. We + count a group tag as a definition if it a) has profile or + public set; or b) has any children.""" + allgroups = [ + g + for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") + if g.get("profile") or g.get("public") or g.getchildren()] + self.duplicate_entries(allgroups, "group") + + def duplicate_entries(self, allentries, etype): + """ Generic duplicate entry finder. + + :param allentries: A list of all entries to check for + duplicates. + :type allentries: list of lxml.etree._Element + :param etype: The entry type. This will be used to determine + the error name (``duplicate-<etype>``) and for + display to the end user. + :type etype: string + """ entries = dict() for el in allentries: - if include(el): - if el.get("name") in entries: - entries[el.get("name")].append(self.RenderXML(el)) - else: - entries[el.get("name")] = [self.RenderXML(el)] + if el.get("name") in entries: + entries[el.get("name")].append(self.RenderXML(el)) + else: + entries[el.get("name")] = [self.RenderXML(el)] for ename, els in entries.items(): if len(els) > 1: self.LintError("duplicate-%s" % etype, @@ -1598,7 +1620,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (etype.title(), ename, "\n".join(els))) def default_is_profile(self): - """ ensure that the default group is a profile group """ + """ Ensure that the default group is a profile group. """ if (self.metadata.default and not self.metadata.groups[self.metadata.default].is_profile): xdata = \ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index bc2928fa6..a82a183d8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -40,6 +40,11 @@ class AptCollection(Collection): else: lines.append("deb %s %s %s" % (source.url, source.version, " ".join(source.components))) + if source.debsrc: + lines.append("deb-src %s %s %s" % + (source.url, + source.version, + " ".join(source.components))) lines.append("") return "\n".join(lines) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 7ba374dd3..22073493c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -158,6 +158,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: this source self.whitelist = [item.text for item in xsource.findall('Whitelist')] + #: Whether or not to include deb-src lines in the generated APT + #: configuration + self.debsrc = xsource.get('debsrc', 'false') == 'true' + #: A dict of repository options that will be included in the #: configuration generated on the server side (if such is #: applicable; most backends do not generate any sort of diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 7438c633b..20b2c9500 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -675,7 +675,10 @@ class YumCollection(Collection): gdicts.append(dict(group=group, type=ptype)) if self.use_yum: - return self.call_helper("get_groups", inputdata=gdicts) + try: + return self.call_helper("get_groups", inputdata=gdicts) + except ValueError: + return dict() else: pkgs = dict() for gdict in gdicts: @@ -838,12 +841,13 @@ class YumCollection(Collection): return Collection.complete(self, packagelist) if packagelist: - result = \ - self.call_helper("complete", - dict(packages=list(packagelist), - groups=list(self.get_relevant_groups()))) - if not result: - # some sort of error, reported by call_helper() + try: + result = self.call_helper( + "complete", + dict(packages=list(packagelist), + groups=list(self.get_relevant_groups()))) + except ValueError: + # error reported by call_helper() return set(), packagelist # json doesn't understand sets or tuples, so we get back a # lists of lists (packages) and a list of unicode strings @@ -905,7 +909,7 @@ class YumCollection(Collection): err = sys.exc_info()[1] self.logger.error("Packages: error reading bcfg2-yum-helper " "output: %s" % err) - return None + raise def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called @@ -931,13 +935,21 @@ class YumCollection(Collection): if force_update: # we call this twice: one to clean up data from the old # config, and once to clean up data from the new config - self.call_helper("clean") + try: + self.call_helper("clean") + except ValueError: + # error reported by call_helper + pass os.unlink(self.cfgfile) self.write_config() if force_update: - self.call_helper("clean") + try: + self.call_helper("clean") + except ValueError: + # error reported by call_helper + pass class YumSource(Source): @@ -1120,9 +1132,9 @@ class YumSource(Source): self.packages['global'] = copy.deepcopy(sdata.pop()) except IndexError: self.logger.error("Packages: No packages in repo") + self.packages['global'] = set() while sdata: - self.packages['global'] = \ - self.packages['global'].intersection(sdata.pop()) + self.packages['global'].update(sdata.pop()) for key in self.packages: if key == 'global': diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index efbca28cd..f82b8a392 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,7 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -459,7 +459,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, try: open(localfile, 'w').write(urlopen(key).read()) keys.append(key) - except HTTPError: + except (URLError, HTTPError): err = sys.exc_info()[1] self.logger.error("Packages: Error downloading %s: %s" % (key, err)) @@ -527,8 +527,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, collection = cclass(metadata, relevant, self.cachepath, self.data, self.core.fam, debug=self.debug_flag) ckey = collection.cachekey - self.clients[metadata.hostname] = ckey - self.collections[ckey] = collection + if cclass != Collection: + self.clients[metadata.hostname] = ckey + self.collections[ckey] = collection return collection def get_additional_data(self, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index 7dac907e1..a1dcb575f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -177,7 +177,10 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): - """ find duplicate Pkgmgr entries with the same priority """ + """ Find duplicate :ref:`Pkgmgr + <server-plugins-generators-pkgmgr>` entries with the same + priority. """ + def Run(self): pset = set() for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr', @@ -202,12 +205,13 @@ class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): # check if package is already listed with same # priority, type, grp if ptuple in pset: - self.LintError("duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) + self.LintError( + "duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) else: pset.add(ptuple) - + @classmethod def Errors(cls): - return {"duplicate-packages":"error"} + return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 309b96475..f8baddb4b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -63,7 +63,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data.encode('utf-8')) + return str.__new__(cls, data) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -222,15 +222,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - try: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str( - self.probedata[client][probe]).decode('utf-8')) - except AttributeError: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self.probedata[client][probe]) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index fc07a90e9..d8b3104b7 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -172,7 +172,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for name in names[cmeta.hostname]: newnames.add(name.split('.')[0]) try: - newips.add(self.get_ipcache_entry(name)[0]) + newips.update(self.get_ipcache_entry(name)[0]) except: # pylint: disable=W0702 continue names[cmeta.hostname].update(newnames) @@ -288,7 +288,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: # need to add entry try: - ipaddr = socket.gethostbyname(client) + ipaddr = set([info[4][0] + for info in socket.getaddrinfo(client, None)]) self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 7dd15f7b5..fcd73bae2 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -97,7 +97,18 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): - """ find duplicate Pkgmgr entries with the same priority """ + """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper + <server-plugins-connectors-templatehelper>` modules are valid. + This can check for: + + * A TemplateHelper module that cannot be imported due to syntax or + other compile-time errors; + * A TemplateHelper module that does not have an ``__export__`` + attribute, or whose ``__export__`` is not a list; + * Bogus symbols listed in ``__export__``, including symbols that + don't exist, that are reserved, or that start with underscores. + """ + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.reserved_keywords = dir(HelperModule("foo.py")) @@ -108,7 +119,11 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): self.check_helper(helper.name) def check_helper(self, helper): - """ check a helper module for export errors """ + """ Check a single helper module. + + :param helper: The filename of the helper module + :type helper: string + """ module_name = MODULE_RE.search(helper).group(1) try: diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 581445bf4..1c2dceed2 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -2,6 +2,7 @@ used by both client and server. Stuff that doesn't fit anywhere else. """ +import shlex import fcntl import logging import threading @@ -218,6 +219,7 @@ class Executor(object): """ if isinstance(command, str): cmdstr = command + command = shlex.split(cmdstr) else: cmdstr = " ".join(command) self.logger.debug("Running: %s" % cmdstr) diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 31e49c00b..14d188342 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -83,7 +83,7 @@ def main(): raise SystemExit(1) mode = mode_cls(setup) try: - mode(setup['args'][1:]) + return mode(setup['args'][1:]) finally: mode.shutdown() else: @@ -93,6 +93,6 @@ def main(): if __name__ == '__main__': try: - main() + sys.exit(main()) except KeyboardInterrupt: raise SystemExit(1) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index ac4c3af13..4e71ba35a 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -231,10 +231,14 @@ class InfoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): print("Refusing to write files outside of /tmp without -f " "option") return - lxml.etree.ElementTree(self.BuildConfiguration(client)).write( - ofile, - encoding='UTF-8', xml_declaration=True, - pretty_print=True) + try: + lxml.etree.ElementTree(self.BuildConfiguration(client)).write( + ofile, + encoding='UTF-8', xml_declaration=True, + pretty_print=True) + except IOError: + err = sys.exc_info()[1] + print("Failed to write File %s: %s" % (ofile, err)) else: print(self._get_usage(self.do_build)) @@ -627,30 +631,34 @@ Bcfg2 client itself.""") self.fam.debug = True def do_packageresolve(self, args): - """ packageresolve <hostname> <package> [<package>...] - - Resolve the specified set of packages """ + """ packageresolve <hostname> [<package> [<package>...]] - + Resolve packages for the given host, optionally specifying a + set of packages """ arglist = args.split(" ") - if len(arglist) < 2: + if len(arglist) < 1: print(self._get_usage(self.do_packageresolve)) return - if 'Packages' not in self.plugins: + try: + pkgs = self.plugins['Packages'] + except KeyError: print("Packages plugin not enabled") return - self.plugins['Packages'].toggle_debug() - - indep = lxml.etree.Element("Independent") - structures = [lxml.etree.Element("Bundle", name="packages")] - for arg in arglist[1:]: - lxml.etree.SubElement(structures[0], "Package", name=arg) + pkgs.toggle_debug() hostname = arglist[0] metadata = self.build_metadata(hostname) - # pylint: disable=W0212 - self.plugins['Packages']._build_packages(metadata, indep, structures) - # pylint: enable=W0212 + indep = lxml.etree.Element("Independent") + if len(arglist) > 1: + structures = [lxml.etree.Element("Bundle", name="packages")] + for arg in arglist[1:]: + lxml.etree.SubElement(structures[0], "Package", name=arg) + else: + structures = self.GetStructures(metadata) + pkgs._build_packages(metadata, indep, # pylint: disable=W0212 + structures) print("%d new packages added" % len(indep.getchildren())) if len(indep.getchildren()): print(" %s" % "\n ".join(lxml.etree.tostring(p) diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 430c4c54f..ab3b6450f 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -3,6 +3,7 @@ """This tool examines your Bcfg2 specifications for errors.""" import sys +import time import inspect import logging import Bcfg2.Logger @@ -52,22 +53,27 @@ def run_plugin(plugin, plugin_name, setup=None, errorhandler=None, args.append(setup) # python 2.5 doesn't support mixing *magic and keyword arguments - return plugin(*args, **dict(files=files, errorhandler=errorhandler)).Run() + start = time.time() + rv = plugin(*args, **dict(files=files, errorhandler=errorhandler)).Run() + LOGGER.debug(" Ran %s in %0.2f seconds" % (plugin_name, + time.time() - start)) + return rv def get_errorhandler(setup): """ get a Bcfg2.Server.Lint.ErrorHandler object """ if setup.cfp.has_section("errors"): - conf = dict(setup.cfp.items("errors")) + errors = dict(setup.cfp.items("errors")) else: - conf = None - return Bcfg2.Server.Lint.ErrorHandler(config=conf) + errors = None + return Bcfg2.Server.Lint.ErrorHandler(errors=errors) def load_server(setup): """ load server """ core = Bcfg2.Server.Core.BaseCore(setup) - core.fam.handle_events_in_interval(4) + core.load_plugins() + core.fam.handle_events_in_interval(0.1) return core @@ -92,7 +98,7 @@ def load_plugins(setup): elif setup['lint_plugins']: plugin_list = setup['lint_plugins'] else: - plugin_list = Bcfg2.Server.Lint.__all__ + plugin_list = Bcfg2.Server.Lint.plugins allplugins = dict() for plugin in plugin_list: diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index cdca71e74..4c4a71fa7 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -24,21 +24,32 @@ def main(): print("Could not read %s" % setup['configfile']) sys.exit(1) - if setup['backend'] not in ['best', 'cherrypy', 'builtin']: + # TODO: normalize case of various core modules so we can add a new + # core without modifying this script + backends = dict(cherrypy='CherryPyCore', + builtin='BuiltinCore', + best='BuiltinCore', + multiprocessing='MultiprocessingCore') + + if setup['backend'] not in backends: print("Unknown server backend %s, using 'best'" % setup['backend']) setup['backend'] = 'best' - if setup['backend'] == 'cherrypy': - try: - from Bcfg2.Server.CherryPyCore import Core - except ImportError: - err = sys.exc_info()[1] - print("Unable to import CherryPy server core: %s" % err) - raise - elif setup['backend'] == 'builtin' or setup['backend'] == 'best': - from Bcfg2.Server.BuiltinCore import Core + + coremodule = backends[setup['backend']] + try: + corecls = getattr(__import__("Bcfg2.Server.%s" % coremodule).Server, + coremodule).Core + except ImportError: + err = sys.exc_info()[1] + print("Unable to import %s server core: %s" % (setup['backend'], err)) + raise + except AttributeError: + err = sys.exc_info()[1] + print("Unable to load %s server core: %s" % (setup['backend'], err)) + raise try: - core = Core(setup) + core = corecls(setup) core.run() except CoreInitError: msg = sys.exc_info()[1] diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 6eaf0cc33..c33143a04 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -155,6 +155,7 @@ class ClientTest(TestCase): def get_core(setup): """ Get a server core, with events handled """ core = Bcfg2.Server.Core.BaseCore(setup) + core.load_plugins() core.fam.handle_events_in_interval(0.1) return core diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py index d9431dc63..d14696b68 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py @@ -70,7 +70,7 @@ class TestPOSIXDirectory(TestPOSIXTool): expected = [os.path.join(entry.get("name"), e) for e in entries if os.path.join(entry.get("name"), e) not in modlist] - actual = [e.get("path") for e in entry.findall("Prune")] + actual = [e.get("name") for e in entry.findall("Prune")] self.assertItemsEqual(expected, actual) mock_verify.reset_mock() @@ -137,7 +137,7 @@ class TestPOSIXDirectory(TestPOSIXTool): entry.set("prune", "true") prune = ["/test/foo/bar/prune1", "/test/foo/bar/prune2"] for path in prune: - lxml.etree.SubElement(entry, "Prune", path=path) + lxml.etree.SubElement(entry, "Prune", name=path) reset() mock_install.return_value = True @@ -145,6 +145,6 @@ class TestPOSIXDirectory(TestPOSIXTool): self.assertTrue(ptool.install(entry)) ptool._exists.assert_called_with(entry) mock_install.assert_called_with(ptool, entry) - self.assertItemsEqual([c[0][0].get("path") + self.assertItemsEqual([c[0][0].get("name") for c in ptool._remove.call_args_list], prune) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py index 4fcd63a60..9478f7071 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py @@ -227,8 +227,7 @@ class TestPOSIXUsers(TestTool): users.user_supplementary_groups.assert_called_with(entry) reset() - m1 = lxml.etree.SubElement(entry, "MemberOf") - m1.text = "wheel" + m1 = lxml.etree.SubElement(entry, "MemberOf", group="wheel") m2 = lxml.etree.SubElement(entry, "MemberOf") m2.text = "users" self.assertTrue(users.VerifyPOSIXUser(entry, [])) @@ -237,8 +236,7 @@ class TestPOSIXUsers(TestTool): users.user_supplementary_groups.assert_called_with(entry) reset() - m3 = lxml.etree.SubElement(entry, "MemberOf") - m3.text = "extra" + m3 = lxml.etree.SubElement(entry, "MemberOf", group="extra") self.assertFalse(users.VerifyPOSIXUser(entry, [])) users.populate_user_entry.assert_called_with(entry) users._verify.assert_called_with(users.populate_user_entry.return_value) @@ -373,8 +371,7 @@ class TestPOSIXUsers(TestTool): entry = lxml.etree.Element("POSIXUser", name="test", group="test", home="/home/test", shell="/bin/zsh", gecos="Test McTest") - m1 = lxml.etree.SubElement(entry, "MemberOf") - m1.text = "wheel" + m1 = lxml.etree.SubElement(entry, "MemberOf", group="wheel") m2 = lxml.etree.SubElement(entry, "MemberOf") m2.text = "users" diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py index 2e758774e..f838030e2 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py @@ -298,21 +298,20 @@ class TestCfgEntrySet(TestEntrySet): for submodule in walk_packages(path=Bcfg2.Server.Plugins.Cfg.__path__, prefix="Bcfg2.Server.Plugins.Cfg."): expected.append(submodule[1].rsplit('.', 1)[-1]) - eset = self.get_obj() - self.assertItemsEqual(expected, - [h.__name__ for h in eset.handlers]) + self.assertItemsEqual(expected, [h.__name__ for h in handlers()]) - def test_handle_event(self): + @patch("Bcfg2.Server.Plugins.Cfg.handlers") + def test_handle_event(self, mock_handlers): eset = self.get_obj() eset.entry_init = Mock() - eset._handlers = [Mock(), Mock(), Mock()] - for hdlr in eset.handlers: + mock_handlers.return_value = [Mock(), Mock(), Mock()] + for hdlr in mock_handlers.return_value: hdlr.__name__ = "handler" eset.entries = dict() def reset(): eset.entry_init.reset_mock() - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: hdlr.reset_mock() # test that a bogus deleted event is discarded @@ -322,7 +321,7 @@ class TestCfgEntrySet(TestEntrySet): eset.handle_event(evt) self.assertFalse(eset.entry_init.called) self.assertItemsEqual(eset.entries, dict()) - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) @@ -333,7 +332,7 @@ class TestCfgEntrySet(TestEntrySet): evt.filename = os.path.join(datastore, "test.txt") # test with no handler that handles - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: hdlr.handles.return_value = False hdlr.ignore.return_value = False @@ -341,16 +340,16 @@ class TestCfgEntrySet(TestEntrySet): eset.handle_event(evt) self.assertFalse(eset.entry_init.called) self.assertItemsEqual(eset.entries, dict()) - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: hdlr.handles.assert_called_with(evt, basename=eset.path) hdlr.ignore.assert_called_with(evt, basename=eset.path) # test with a handler that handles the entry reset() - eset.handlers[-1].handles.return_value = True + mock_handlers.return_value[-1].handles.return_value = True eset.handle_event(evt) - eset.entry_init.assert_called_with(evt, eset.handlers[-1]) - for hdlr in eset.handlers: + eset.entry_init.assert_called_with(evt, mock_handlers.return_value[-1]) + for hdlr in mock_handlers.return_value: hdlr.handles.assert_called_with(evt, basename=eset.path) if not hdlr.return_value: hdlr.ignore.assert_called_with(evt, basename=eset.path) @@ -358,14 +357,14 @@ class TestCfgEntrySet(TestEntrySet): # test with a handler that ignores the entry before one # that handles it reset() - eset.handlers[0].ignore.return_value = True + mock_handlers.return_value[0].ignore.return_value = True eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - eset.handlers[0].handles.assert_called_with(evt, + mock_handlers.return_value[0].handles.assert_called_with(evt, basename=eset.path) - eset.handlers[0].ignore.assert_called_with(evt, + mock_handlers.return_value[0].ignore.assert_called_with(evt, basename=eset.path) - for hdlr in eset.handlers[1:]: + for hdlr in mock_handlers.return_value[1:]: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) @@ -377,7 +376,7 @@ class TestCfgEntrySet(TestEntrySet): eset.entries[evt.filename] = Mock() eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) eset.entries[evt.filename].handle_event.assert_called_with(evt) @@ -387,7 +386,7 @@ class TestCfgEntrySet(TestEntrySet): evt.code2str.return_value = "deleted" eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) self.assertItemsEqual(eset.entries, dict()) diff --git a/testsuite/install.sh b/testsuite/install.sh index 817ed5911..1ca89f40f 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -15,7 +15,7 @@ if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then if [[ ${PYVER:0:1} == "2" ]]; then # django supports py3k, but South doesn't, and the django bits # in bcfg2 require South - pip install cheetah 'django<1.5' South M2Crypto + pip install cheetah 'django<1.5' 'South<0.8' M2Crypto fi else # python < 2.6 requires M2Crypto for SSL communication, not just diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py index 3cd3786f9..93314f1e3 100755 --- a/tools/bcfg2-profile-templates.py +++ b/tools/bcfg2-profile-templates.py @@ -1,25 +1,35 @@ #!/usr/bin/python -Ott +# -*- coding: utf-8 -*- """ Benchmark template rendering times """ -import os import sys import time +import math import logging import operator import Bcfg2.Logger +import Bcfg2.Options import Bcfg2.Server.Core -LOGGER = None + +def stdev(nums): + mean = float(sum(nums)) / len(nums) + return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums))) def main(): - optinfo = \ - dict(client=Bcfg2.Options.Option("Benchmark templates for one client", - cmd="--client", - odesc="<client>", - long_arg=True, - default=None), - ) + optinfo = dict( + client=Bcfg2.Options.Option("Benchmark templates for one client", + cmd="--client", + odesc="<client>", + long_arg=True, + default=None), + runs=Bcfg2.Options.Option("Number of rendering passes per template", + cmd="--runs", + odesc="<runs>", + long_arg=True, + default=5, + cook=int)) optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) @@ -40,12 +50,11 @@ def main(): core = Bcfg2.Server.Core.BaseCore(setup) logger.info("Bcfg2 server core loaded") + core.load_plugins() + logger.debug("Plugins loaded") core.fam.handle_events_in_interval(0.1) logger.debug("Repository events processed") - # how many times to render each template for each client - runs = 5 - if setup['args']: templates = setup['args'] else: @@ -57,41 +66,57 @@ def main(): clients = [core.build_metadata(setup['client'])] times = dict() + client_count = 0 for metadata in clients: - for struct in core.GetStructures(metadata): - logger.info("Rendering templates from structure %s:%s" % - (struct.tag, struct.get("name"))) - for entry in struct.xpath("//Path"): - path = entry.get("name") - logger.info("Rendering %s..." % path) - times[path] = dict() - avg = 0.0 - for i in range(runs): + client_count += 1 + logger.info("Rendering templates for client %s (%s/%s)" % + (metadata.hostname, client_count, len(clients))) + structs = core.GetStructures(metadata) + struct_count = 0 + for struct in structs: + struct_count += 1 + logger.info("Rendering templates from structure %s:%s (%s/%s)" % + (struct.tag, struct.get("name"), struct_count, + len(structs))) + entries = struct.xpath("//Path") + entry_count = 0 + for entry in entries: + entry_count += 1 + if templates and entry.get("name") not in templates: + continue + logger.info("Rendering Path:%s (%s/%s)..." % + (entry.get("name"), entry_count, len(entries))) + ptimes = times.setdefault(entry.get("name"), []) + for i in range(setup['runs']): start = time.time() try: core.Bind(entry, metadata) - avg += (time.time() - start) / runs + ptimes.append(time.time() - start) except: break - if avg: - logger.debug(" %s: %.02f sec" % (metadata.hostname, avg)) - times[path][metadata.hostname] = avg + if ptimes: + avg = sum(ptimes) / len(ptimes) + if avg: + logger.debug(" %s: %.02f sec" % + (metadata.hostname, avg)) # print out per-file results tmpltimes = [] - for tmpl, clients in times.items(): + for tmpl, ptimes in times.items(): try: - avg = sum(clients.values()) / len(clients) + mean = float(sum(ptimes)) / len(ptimes) except ZeroDivisionError: continue - if avg > 0.01 or templates: - tmpltimes.append((tmpl, avg)) - print("%-50s %s" % ("Template", "Average Render Time")) - for tmpl, avg in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): - print("%-50s %.02f" % (tmpl, avg)) - - # TODO: complain about templates that on average were quick but - # for which some clients were slow + ptimes.sort() + median = ptimes[len(ptimes) / 2] + std = stdev(ptimes) + if mean > 0.01 or median > 0.01 or std > 1 or templates: + tmpltimes.append((tmpl, mean, median, std)) + print("%-50s %-9s %-11s %6s" % + ("Template", "Mean Time", "Median Time", "σ")) + for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): + print("%-50s %9.02f %11.02f %6.02f" % info) + core.shutdown() if __name__ == "__main__": diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index edb5a7101..8c164e52e 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -19,7 +19,8 @@ class LocalCore(BaseCore): setup['logging'] = None Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) setup['syslog'], setup['logging'] = saved - self.fam.handle_events_in_interval(4) + self.load_plugins() + self.fam.handle_events_in_interval(0.1) def _daemonize(self): return True diff --git a/tools/export.py b/tools/export.py index 716c831d9..5cd0d5c41 100755 --- a/tools/export.py +++ b/tools/export.py @@ -227,9 +227,15 @@ E.G. 1.2.0pre1 is a valid version. 'VERSION="%s"\n' % version, startswith=True, dryrun=options.dryrun) - # set new version in setup.py - find_and_replace('setup.py', 'version=', ' version="%s",\n' % version, - dryrun=options.dryrun) + # update solaris IPS version + find_and_replace('solaris-ips/MANIFEST.bcfg2.header', 'set name=pkg.fmri value="pkg://bcfg2/bcfg2@', + 'set name=pkg.fmri value="pkg://bcfg2/bcfg2@%s"' % version, + startswith=True, + dryrun=options.dryrun) + find_and_replace('solaris-ips/MANIFEST.bcfg2-server.header', 'set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@', + 'set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@%s"' % version, + startswith=True, + dryrun=options.dryrun) # set new version in Bcfg2/version.py find_and_replace('src/lib/Bcfg2/version.py', '__version__ =', diff --git a/tools/posixusers_baseline.py b/tools/posixusers_baseline.py index a4abca42d..c45e54f1a 100755 --- a/tools/posixusers_baseline.py +++ b/tools/posixusers_baseline.py @@ -61,8 +61,8 @@ def main(): if entry.tag == 'POSIXUser': entry.set("group", grp.getgrgid(data[3])[0]) for group in users.user_supplementary_groups(entry): - memberof = lxml.etree.SubElement(entry, "MemberOf") - memberof.text = group[0] + memberof = lxml.etree.SubElement(entry, "MemberOf", + group=group[0]) entry.tag = "Bound" + entry.tag baseline.append(entry) diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py index 06f6e6b98..507a16f43 100755 --- a/tools/selinux_baseline.py +++ b/tools/selinux_baseline.py @@ -42,7 +42,10 @@ def main(): baseline.append(lxml.etree.Comment("%s entries" % etype)) extra = handler.FindExtra() for entry in extra: - entry.tag = "Bound%s" % etype + if etype != "SEModule": + entry.tag = "Bound%s" % etype + else: + entry.tag = "%s" % etype baseline.extend(extra) print(lxml.etree.tostring(baseline, pretty_print=True)) diff --git a/tools/upgrade/1.3/README b/tools/upgrade/1.3/README index 2831d8f00..1a919f869 100644 --- a/tools/upgrade/1.3/README +++ b/tools/upgrade/1.3/README @@ -19,3 +19,8 @@ migrate_dbstats.py migrate_perms_to_mode.py - Convert perms attribute to mode (note that if you have info/:info files, you should run migrate_info.py first) + +migrate_probe_groups_to_db.py + - Migrate Probe host and group data from XML to DB backend for Metadata + and Probe plugins. Does not migrate individual probe return data. Assumes + migration to BOTH Metadata and Probe to database backends. diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py index cbd2a6099..07def2ac8 100755 --- a/tools/upgrade/1.3/migrate_dbstats.py +++ b/tools/upgrade/1.3/migrate_dbstats.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) _our_backend = None + def _quote(value): """ Quote a string to use as a table name or column @@ -44,12 +45,12 @@ def _migrate_perms(): fperms = {} logger.info("Creating FilePerms objects") - for data in ( ('owner', 'group', 'perms'), + for data in (('owner', 'group', 'perms'), ('current_owner', 'current_group', 'current_perms')): for grp in legacy_models.Reason.objects.values_list(*data).distinct(): if grp in fperms: continue - fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2]) + fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2]) fp.save() fperms[grp] = fp @@ -60,7 +61,7 @@ def _migrate_perms(): def _migrate_transaction(inter, entries, fperms): """helper""" - logger.debug("Migrating interaction %s for %s" % + logger.debug("Migrating interaction %s for %s" % (inter.id, inter.client.name)) newint = new_models.Interaction(id=inter.id, @@ -107,7 +108,7 @@ def _migrate_transaction(inter, entries, fperms): elif ent.kind == 'Package': act_dict['target_version'] = ei.reason.version act_dict['current_version'] = ei.reason.current_version - logger.debug("Adding package %s %s" % + logger.debug("Adding package %s %s" % (name, act_dict['target_version'])) updates['packages'].append(new_models.PackageEntry.entry_get_or_create(act_dict)) elif ent.kind == 'Path': @@ -116,7 +117,7 @@ def _migrate_transaction(inter, entries, fperms): act_dict['target_perms'] = fperms[( ei.reason.owner, - ei.reason.group, + ei.reason.group, ei.reason.perms )] @@ -141,7 +142,6 @@ def _migrate_transaction(inter, entries, fperms): act_dict['detail_type'] = new_models.PathEntry.DETAIL_PRUNED act_dict['details'] = ei.reason.unpruned - if ei.reason.is_sensitive: act_dict['detail_type'] = new_models.PathEntry.DETAIL_SENSITIVE elif ei.reason.is_binary: @@ -164,7 +164,7 @@ def _migrate_transaction(inter, entries, fperms): for entry_type in updates.keys(): i = 0 while(i < len(updates[entry_type])): - getattr(newint, entry_type).add(*updates[entry_type][i:i+100]) + getattr(newint, entry_type).add(*updates[entry_type][i:i + 100]) i += 100 for perf in inter.performance_items.all(): @@ -220,8 +220,8 @@ def _restructure(): # run any migrations from the previous schema try: - from Bcfg2.Server.Reports.updatefix import update_database - update_database() + from Bcfg2.Server.Reports.updatefix import update_database + update_database() except: logger.error("Failed to run legacy schema updates", exc_info=1) return False @@ -295,4 +295,3 @@ if __name__ == '__main__': Reports(setup).__call__(['update']) _restructure() - diff --git a/tools/upgrade/1.3/migrate_info.py b/tools/upgrade/1.3/migrate_info.py index e72599daf..3ccbf0285 100755 --- a/tools/upgrade/1.3/migrate_info.py +++ b/tools/upgrade/1.3/migrate_info.py @@ -1,12 +1,16 @@ #!/usr/bin/env python import os +import re import sys import lxml.etree import Bcfg2.Options from Bcfg2.Server.Plugin import INFO_REGEX +PERMS_REGEX = re.compile(r'perms:\s*(?P<perms>\w+)') + + def convert(info_file): info_xml = os.path.join(os.path.dirname(info_file), "info.xml") if os.path.exists(info_xml): @@ -16,7 +20,7 @@ def convert(info_file): fileinfo = lxml.etree.Element("FileInfo") info = lxml.etree.SubElement(fileinfo, "Info") for line in open(info_file).readlines(): - match = INFO_REGEX.match(line) + match = INFO_REGEX.match(line) or PERMS_REGEX.match(line) if match: mgd = match.groupdict() for key, value in list(mgd.items()): diff --git a/tools/upgrade/1.3/migrate_probe_groups_to_db.py b/tools/upgrade/1.3/migrate_probe_groups_to_db.py new file mode 100755 index 000000000..73339e787 --- /dev/null +++ b/tools/upgrade/1.3/migrate_probe_groups_to_db.py @@ -0,0 +1,68 @@ +#!/bin/env python +""" Migrate Probe host and group data from XML to DB backend for Metadata +and Probe plugins. Does not migrate individual probe return data. Assumes +migration to BOTH Metadata and Probe to database backends. """ + +import os +os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' + +import lxml.etree +import sys +import Bcfg2.Options + +from Bcfg2.Server.Plugins.Metadata import MetadataClientModel +from Bcfg2.Server.Plugins.Probes import ProbesGroupsModel + +def migrate(xclient): + """ Helper to do the migration given a <Client/> XML element """ + client_name = xclient.get('name') + try: + try: + client = MetadataClientModel.objects.get(hostname=client_name) + except MetadataClientModel.DoesNotExist: + client = MetadataClientModel(hostname=client_name) + client.save() + except: + print("Failed to migrate client %s" % (client)) + return False + + try: + cgroups = [] + for xgroup in xclient.findall('Group'): + group_name = xgroup.get('name') + cgroups.append(group_name) + try: + group = ProbesGroupsModel.objects.get(hostname=client_name, group=group_name) + except ProbesGroupsModel.DoesNotExist: + group = ProbesGroupsModel(hostname=client_name, group=group_name) + group.save() + + ProbesGroupsModel.objects.filter( + hostname=client.hostname).exclude( + group__in=cgroups).delete() + + except: + print("Failed to migrate groups") + return False + return True + +def main(): + """ Main """ + opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY) + setup = Bcfg2.Options.OptionParser(opts) + setup.parse(sys.argv[1:]) + + probefile = os.path.join(setup['repo'], 'Probes', "probed.xml") + + try: + xdata = lxml.etree.parse(probefile) + except lxml.etree.XMLSyntaxError: + err = sys.exc_info()[1] + print("Could not parse %s, skipping: %s" % (probefile, err)) + + for xclient in xdata.findall('Client'): + print "Migrating Metadata and Probe groups for %s" % xclient.get('name') + migrate(xclient) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/yum-listpkgs-xml.py b/tools/yum-listpkgs-xml.py index a052e75af..b4c5f6589 100755 --- a/tools/yum-listpkgs-xml.py +++ b/tools/yum-listpkgs-xml.py @@ -39,5 +39,5 @@ try: sys.argv = [sys.argv[0], '-d', '0', 'list'] yummain.main(sys.argv[1:]) except KeyboardInterrupt: - print("\n\nExiting on user cancel.", file=sys.stderr) + sys.stderr.write("\n\nExiting on user cancel.") sys.exit(1) |