diff options
59 files changed, 942 insertions, 350 deletions
diff --git a/.travis.yml b/.travis.yml index 73b8a9594..655d9fad5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - "2.5" - "2.6" - "2.7" + - "3.2" env: - WITH_OPTIONAL_DEPS=yes - WITH_OPTIONAL_DEPS=no @@ -1,3 +1,7 @@ +This file contains a list of copyright holders. Anyone who +contributes more than trivial fixes (typos, etc.) to Bcfg2 should also +add themselves to this file. See LICENSE for the full license. + - Narayan Desai <desai@mcs.anl.gov> has written most of Bcfg2, including all parts not explicitly mentioned in this file. diff --git a/doc/_templates/indexsidebar.html b/doc/_templates/indexsidebar.html index 39916315d..2133cdcc5 100644 --- a/doc/_templates/indexsidebar.html +++ b/doc/_templates/indexsidebar.html @@ -7,5 +7,6 @@ <ul> <li><a href="http://docs.bcfg2.org/1.1/">Bcfg2 1.1 (stable)</a></li> <li><a href="http://docs.bcfg2.org/1.2/">Bcfg2 1.2 (stable)</a></li> + <li><a href="http://docs.bcfg2.org/1.3/">Bcfg2 1.3 (stable)</a></li> <li><a href="http://docs.bcfg2.org/dev/">Bcfg2 development documentation</a></li> </ul> diff --git a/doc/development/documentation.txt b/doc/development/documentation.txt index 2a3cf46d1..4d8a7c9f8 100644 --- a/doc/development/documentation.txt +++ b/doc/development/documentation.txt @@ -8,13 +8,14 @@ There are two parts of documentation in the Bcfg2 project: -* The wiki -* The manual +* The Wiki_ +* The Manual_ The wiki ======== .. _Wiki: http://bcfg2.org +.. _Manual: http://docs.bcfg2.org .. _Trac: http://trac.edgewall.org/ .. _OpenID: https://openid.org/ .. _MCS: http://www.mcs.anl.gov/ @@ -31,9 +32,9 @@ The manual ========== .. _rst: http://en.wikipedia.org/wiki/ReStructuredText .. _Sphinx: http://sphinx.pocoo.org -.. _Docutils: +.. _Docutils: http://docutils.sourceforge.net -The source for the manual is located in the ``doc/`` directory in the +The source for the Manual_ is located in the ``doc/`` directory in the git repository or in the source tarball. All files are written in rst_ (ReStructuredText) format. Sphinx_ is used to build the documentation from the restructured text sources. @@ -49,11 +50,20 @@ Building the Manual apt-get -t lenny-backports install python-sphinx - * The needed tools for Fedora based systems are in the `Fedora + * The tools for Fedora based systems are in the `Fedora Package Collection <https://admin.fedoraproject.org/pkgdb>`_; installation can be done easily with Yum:: yum -y install python-sphinx python-docutils + + * The tools for RHEL6-based systems are in the base distribution; you can install them with Yum:: + + yum -y install python-sphinx python-docutils + + * The tools for RHEL5-based systems are in the `Extra Packages for Enterprise Linux(EPEL) <https://fedoraproject.org/wiki/EPEL>`_ repository; if your system is configured for EPEL, you can install them with Yum:: + + yum -y install python-sphinx python-docutils + * Additionally, to build the PDF version: @@ -80,14 +90,14 @@ Documentation Style Guide for Bcfg2 =================================== This is a style guide to use when creating documentation for Bcfg2. It -is meant to be helpful, not a hinderence. +is meant to be helpful, not a hindrance. Basics ------ **Bcfg2** - When referring to project, Bcfg2 is the preferred use of cases. + When referring to project, Bcfg2 is the preferred use of case. **Monospace fonts** @@ -97,8 +107,7 @@ Basics **Repository** When used alone this refers to a Bcfg2 :term:`repository`. When there - is a chance for confusion, for instance in documents also talking - about :term:`VCS`, be sure to use the longer Bcfg2 :term:`repository`. + is a chance for confusion, for instance in documents that also discuss :term:`VCS`, be sure to use the longer phrase "Bcfg2 :term:`repository`". Sections -------- diff --git a/doc/development/lint.txt b/doc/development/lint.txt new file mode 100644 index 000000000..6a4651f92 --- /dev/null +++ b/doc/development/lint.txt @@ -0,0 +1,167 @@ +.. -*- mode: rst -*- + +.. _development-lint: + +=============================== + bcfg2-lint Plugin Development +=============================== + +``bcfg2-lint``, like most parts of Bcfg2, has a pluggable backend that +lets you easily write your own plugins to verify various parts of your +Bcfg2 specification. + +Plugins are loaded in one of two ways: + +* They may be included in a module of the same name as the plugin + class in :mod:`Bcfg2.Server.Lint`, e.g., + :mod:`Bcfg2.Server.Lint.Validate`. +* They may be included directly in a Bcfg2 server plugin, called + "<plugin>Lint", e.g., + :class:`Bcfg2.Server.Plugins.Metadata.MetadataLint`. + +Plugin Types +============ + +There are two types of ``bcfg2-lint`` plugins: + +Serverless plugins +------------------ + +Serverless plugins are run before ``bcfg2-lint`` starts up a local +Bcfg2 server, so the amount of introspection they can do is fairly +limited. They can directly examine the Bcfg2 specification, of +course, but they can't examine the entries handled by a given plugin +or anything that requires a running server. + +If a serverless plugin raises a lint error, however, the server will +not be started and no `Server plugins`_ will be run. This makes them +useful to check for the sorts of errors that might prevent the Bcfg2 +server from starting properly. + +Serverless plugins must subclass +:class:`Bcfg2.Server.Lint.ServerlessPlugin`. + +:mod:`Bcfg2.Server.Lint.Validate` is an example of a serverless +plugin. + +Server plugins +-------------- + +Server plugins are run after a local Bcfg2 server has been started, +and have full access to all of the parsed data and so on. Because of +this, they tend to be easier to use than `Serverless plugins`_, and +thus are more common. + +Server plugins are only run if all `Serverless plugins`_ run +successfully (i.e., raise no errors). + +Server plugins must subclass :class:`Bcfg2.Server.Lint.ServerPlugin`. + +:mod:`Bcfg2.Server.Lint.Genshi` is an example of a server plugin. + +Error Handling +============== + +The job of a ``bcfg2-lint`` plugin is to find errors. Each error that +a plugin may produce must have a name, a short string that briefly +describes the error and will be used to configure error levels in +``bcfg2.conf``. It must also have a default reporting level. +Possible reporting levels are "error", "warning", or "silent". All of +the errors that may be produced by a plugin must be returned as a dict +by :func:`Bcfg2.Server.Lint.Plugin.Errors`. For instance, consider +:func:`Bcfg2.Server.Lint.InfoXML.InfoXML.Errors`: + +.. code-block:: python + + @classmethod + def Errors(cls): + return {"no-infoxml": "warning", + "deprecated-info-file": "warning", + "paranoid-false": "warning", + "required-infoxml-attrs-missing": "error"} + +This means that the :class:`Bcfg2.Server.Lint.InfoXML.InfoXML` lint +plugin can produce five lint errors, although four of them are just +warnings by default. + +The errors returned by each plugin's ``Errors()`` method will be +passed to :func:`Bcfg2.Server.Lint.ErrorHandler.RegisterErrors`, which +will use that information and the information in the config file to +determine how to display (or not display) each error to the end user. + +Errors are produced in a plugin with +:func:`Bcfg2.Server.Lint.Plugin.LintError`, which takes two arguments: +the name of the error, which must correspond to a key in the dict +returned by :func:`Bcfg2.Server.Lint.Plugin.Errors`, and a freeform +string that will be displayed to the end user. Note that the error +name and its display are thus only tied together when the error is +produced; that is, a single error (by name) can have two completely +different outputs. + +Basics +====== + +.. automodule:: Bcfg2.Server.Lint + +Existing ``bcfg2-lint`` Plugins +=============================== + +BundlerLint +----------- + +.. autoclass:: Bcfg2.Server.Plugins.Bundler.BundlerLint + +Comments +-------- + +.. automodule:: Bcfg2.Server.Lint.Comments + +Genshi +------ + +.. automodule:: Bcfg2.Server.Lint.Genshi + +GroupNames +---------- + +.. automodule:: Bcfg2.Server.Lint.GroupNames + +GroupPatternsLint +----------------- + +.. autoclass:: Bcfg2.Server.Plugins.GroupPatterns.GroupPatternsLint + +InfoXML +------- + +.. automodule:: Bcfg2.Server.Lint.InfoXML + +MergeFiles +---------- + +.. automodule:: Bcfg2.Server.Lint.MergeFiles + +MetadataLint +------------ + +.. autoclass:: Bcfg2.Server.Plugins.Metadata.MetadataLint + +PkgmgrLint +---------- + +.. autoclass:: Bcfg2.Server.Plugins.Pkgmgr.PkgmgrLint + +RequiredAttrs +------------- + +.. automodule:: Bcfg2.Server.Lint.RequiredAttrs + +TemplateHelperLint +------------------ + +.. autoclass:: Bcfg2.Server.Plugins.TemplateHelper.TemplateHelperLint + +Validate +-------- + +.. automodule:: Bcfg2.Server.Lint.Validate diff --git a/doc/server/plugins/connectors/grouplogic.txt b/doc/server/plugins/connectors/grouplogic.txt index b9a5b00d6..abf425202 100644 --- a/doc/server/plugins/connectors/grouplogic.txt +++ b/doc/server/plugins/connectors/grouplogic.txt @@ -110,8 +110,8 @@ individually, there's a more elegant way to accomplish the same thing: <GroupLogic xmlns:py="http://genshi.edgewall.org/"> <?python -component = metadata.group_in_category("webapp-component") -env = metadata.group_in_category("environment") + component = metadata.group_in_category("webapp-component") + env = metadata.group_in_category("environment") ?> <py:if test="component and env"> <Group name="${component}-${env}"/> diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index 092cff1ae..a7cdfad2d 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -190,7 +190,8 @@ something like this: <Group name="ubuntu-intrepid"> <Source type="apt" url="http://us.archive.ubuntu.com/ubuntu" - version="intrepid"> + version="intrepid" + debsrc="true"> <Component>main</Component> <Component>universe</Component> <Arch>i386</Arch> @@ -218,7 +219,7 @@ something like this: .. warning:: You must regenerate the Packages cache when adding or removing the recommended attribute (``bcfg2-admin xcmd - Packages.Refresh``). + Packages.Refresh``). .. [#f1] Bcfg2 will by default add **Essential** packages to the client specification. You can disable this behavior by diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index ccde65eb0..a85cd3fc9 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -377,9 +377,9 @@ For example: <POSIXUser name="daemon" home="/sbin" shell="/sbin/nologin" gecos="daemon" uid="2" group="daemon"> - <MemberOf>lp</MemberOf> - <MemberOf>adm</MemberOf> - <MemberOf>bin</MemberOf> + <MemberOf group="lp"/> + <MemberOf group="adm"/> + <MemberOf group="bin/> </POSIXUser> The group specified will automatically be created if it does not diff --git a/doc/server/plugins/grouping/grouppatterns.txt b/doc/server/plugins/grouping/grouppatterns.txt index 39c632f82..44ffa5066 100644 --- a/doc/server/plugins/grouping/grouppatterns.txt +++ b/doc/server/plugins/grouping/grouppatterns.txt @@ -8,7 +8,7 @@ GroupPatterns The GroupPatterns plugin is a connector that can assign clients group membership pased on patterns in client hostnames. Two basic -methods are supported: +methods are supported: - regular expressions (NamePatterns) - ranges (NameRange) @@ -20,7 +20,7 @@ Setup ===== #. Enable the GroupPatterns plugin -#. Create the GroupPatterns/config.xml file (similar to the example below). +#. Create the ``GroupPatterns/config.xml`` file (similar to the example below). #. Client groups will be augmented based on the specification Pattern Types @@ -52,7 +52,7 @@ Examples </GroupPattern> <GroupPattern> <NamePattern>(.*)</NamePattern> - <Group>group-$1'</Group> + <Group>group-$1</Group> </GroupPattern> <GroupPattern> <NameRange>node[[1-32]]</NameRange> @@ -82,15 +82,14 @@ GroupPatterns configuration: <GroupPatterns> <GroupPattern> - <NamePattern>^x(\w[^\d|\.]+)\d*\..*</NamePattern> + <NamePattern>x(\w[^\d\.]+)\d*\.</NamePattern> <Group>$1-server</Group> </GroupPattern> </GroupPatterns> Regex explanation: -#. !^x Match any hostname that begins with "x" -#. (\w[!^\d|\.]+) followed by one or more word characters that are not a decimal digit or "." and save the string to $1 -#. \d* followed by 0 or more decimal digit(s) -#. \..* followed by a "." -#. .* followed by 1 or more of anything else. +#. ``x`` Match any hostname that begins with "x" +#. ``(\w[!^\d|\.]+)`` followed by one or more word characters that are not a decimal digit or "." and save the string to $1 +#. ``\d*`` followed by 0 or more decimal digit(s) +#. ``\.`` followed by a literal "." diff --git a/gentoo/bcfg2-1.3.0.ebuild b/gentoo/bcfg2-1.3.0.ebuild index e600448d9..4d8530b02 100644 --- a/gentoo/bcfg2-1.3.0.ebuild +++ b/gentoo/bcfg2-1.3.0.ebuild @@ -1,13 +1,13 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Header: $ -EAPI="4" +EAPI=4 -PYTHON_DEPEND="2:2.6" +PYTHON_DEPEND="*:2.6" SUPPORT_PYTHON_ABIS="1" # ssl module required. -RESTRICT_PYTHON_ABIS="2.4 2.5 3.*" +RESTRICT_PYTHON_ABIS="2.5" inherit distutils eutils @@ -15,7 +15,7 @@ DESCRIPTION="configuration management tool" HOMEPAGE="http://bcfg2.org" SRC_URI="ftp://ftp.mcs.anl.gov/pub/bcfg/${P}.tar.gz" -LICENSE="BSD" +LICENSE="BSD-2" SLOT="0" KEYWORDS="~amd64 ~x86 ~amd64-linux ~x86-linux ~x64-solaris" IUSE="doc cheetah genshi server" diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 0b975cb37..5feef1d74 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -334,6 +334,7 @@ touch %{buildroot}%{_sysconfdir}/bcfg2.conf \ %{python_sitelib}/Bcfg2/Logger.py* %{python_sitelib}/Bcfg2/Options.py* %{python_sitelib}/Bcfg2/Proxy.py* +%{python_sitelib}/Bcfg2/Utils.py* %{python_sitelib}/Bcfg2/version.py* %{python_sitelib}/Bcfg2/Client %{_mandir}/man1/bcfg2.1* diff --git a/schemas/packages.xsd b/schemas/packages.xsd index 2645a8be0..e01093c56 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -172,6 +172,15 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:boolean" name="debsrc"> + <xsd:annotation> + <xsd:documentation> + Include ``deb-src`` lines in the generated APT + configuration. This only applies to sources with + :xml:attribute:`SourceType:type` = ``apt``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attribute type="xsd:string" name="url"> <xsd:annotation> <xsd:documentation> diff --git a/schemas/selinux.xsd b/schemas/selinux.xsd index 760953e34..3651549f5 100644 --- a/schemas/selinux.xsd +++ b/schemas/selinux.xsd @@ -80,6 +80,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to this port + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -127,6 +134,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to files matching this specification + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -157,6 +171,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to this node + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -205,6 +226,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to this user + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -235,6 +263,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to this user + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> @@ -258,6 +293,13 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:token" name="mlsrange"> + <xsd:annotation> + <xsd:documentation> + SELinux MLS range to apply to this interface + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/types.xsd b/schemas/types.xsd index 9c4a0a48e..fe9b5c7d4 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -389,6 +389,27 @@ </xsd:restriction> </xsd:simpleType> + <xsd:complexType name="MemberOfType"> + <xsd:annotation> + <xsd:documentation> + Specify additional supplementary groups for the POSIXUser + </xsd:documentation> + </xsd:annotation> + <xsd:simpleContent> + <xsd:extension base="xsd:token"> + <xsd:attribute name="group" type="xsd:token"> + <xsd:annotation> + <xsd:documentation> + The name of the supplementary group. This can also be + specified as content of the tag, although that is + deprecated. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:extension> + </xsd:simpleContent> + </xsd:complexType> + <xsd:complexType name="POSIXUserType"> <xsd:annotation> <xsd:documentation> @@ -396,13 +417,7 @@ </xsd:documentation> </xsd:annotation> <xsd:choice minOccurs='0' maxOccurs='unbounded'> - <xsd:element name='MemberOf' type='xsd:token'> - <xsd:annotation> - <xsd:documentation> - Specify additional supplementary groups for the POSIXUser - </xsd:documentation> - </xsd:annotation> - </xsd:element> + <xsd:element name='MemberOf' type='MemberOfType'/> </xsd:choice> <xsd:attribute type="xsd:token" name="name" use="required"> <xsd:annotation> @@ -4,26 +4,26 @@ from setuptools import setup from glob import glob import sys -vfile = 'src/lib/Bcfg2/version.py' +version_file = 'src/lib/Bcfg2/version.py' try: # python 2 - execfile(vfile) + execfile(version_file) except NameError: # py3k - exec(compile(open(vfile).read(), vfile, 'exec')) + exec(compile(open(version_file).read(), version_file, 'exec')) -# we only need m2crypto on < python2.6 -need_m2crypto = False -version = sys.version_info[:2] -if version < (2, 6): - need_m2crypto = True +inst_reqs = [ + 'lockfile', + 'lxml', + 'python-daemon', +] -inst_reqs = ['lxml', 'genshi'] -if need_m2crypto: +# we only need m2crypto on < python2.6 +if sys.version_info[:2] < (2, 6): inst_reqs.append('M2Crypto') setup(name="Bcfg2", - version="1.3.1", + version=__version__, # Defined in src/lib/Bcfg2/version.py description="Bcfg2 Server", author="Narayan Desai", author_email="desai@mcs.anl.gov", @@ -52,9 +52,9 @@ setup(name="Bcfg2", install_requires=inst_reqs, tests_require=['mock', 'nose', 'sqlalchemy'], package_dir={'': 'src/lib', }, - package_data={'Bcfg2.Reporting': [ 'templates/*.html', - 'templates/*/*.html', - 'templates/*/*.inc']}, + package_data={'Bcfg2.Reporting': ['templates/*.html', + 'templates/*/*.html', + 'templates/*/*.inc']}, scripts=glob('src/sbin/*'), data_files=[('share/bcfg2/schemas', glob('schemas/*.xsd')), diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 11a82fcc0..b24b46dbc 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -161,7 +161,7 @@ class Frame(object): def promptFilter(self, msg, entries): """Filter a supplied list based on user input.""" ret = [] - entries.sort(cmpent) + entries.sort(key=lambda e: e.tag + ":" + e.get('name')) for entry in entries[:]: if entry in self.unhandled: # don't prompt for entries that can't be installed @@ -427,10 +427,12 @@ class Frame(object): # prune out unspecified bundles when running with -b continue if bundle in mbundles: - self.logger.debug("Bundle %s was modified" % bundle) + self.logger.debug("Bundle %s was modified" % + bundle.get('name')) func = "BundleUpdated" else: - self.logger.debug("Bundle %s was not modified" % bundle) + self.logger.debug("Bundle %s was not modified" % + bundle.get('name')) func = "BundleNotUpdated" for tool in self.tools: try: @@ -477,8 +479,8 @@ class Frame(object): self.logger.info("%s:%s:%s" % (entry.tag, etype, entry.get('name'))) else: - self.logger.info(" %s:%s" % (entry.tag, - entry.get('name'))) + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) self.logger.info('Total managed entries: %d' % len(list(self.states.values()))) self.logger.info('Unmanaged entries: %d' % len(self.extra)) @@ -490,8 +492,8 @@ class Frame(object): self.logger.info("%s:%s:%s" % (entry.tag, etype, entry.get('name'))) else: - self.logger.info(" %s:%s" % (entry.tag, - entry.get('name'))) + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) if ((list(self.states.values()).count(False) == 0) and not self.extra): self.logger.info('All entries correct.') diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py index 57be34369..eda3a7fce 100644 --- a/src/lib/Bcfg2/Client/Proxy.py +++ b/src/lib/Bcfg2/Client/Proxy.py @@ -24,6 +24,7 @@ from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus version = sys.version_info[:2] has_py26 = version >= (2, 6) +has_py32 = version >= (3, 2) __all__ = ["ComponentProxy", "RetryMethod", @@ -171,11 +172,14 @@ class SSLHTTPConnection(httplib.HTTPConnection): """ if not has_py26: httplib.HTTPConnection.__init__(self, host, port, strict) - else: + elif not has_py32: httplib.HTTPConnection.__init__(self, host, port, strict, timeout) + else: + # the strict parameter is deprecated. + # HTTP 0.9-style "Simple Responses" are not supported anymore. + httplib.HTTPConnection.__init__(self, host, port, timeout=timeout) self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__, self.__class__.__name__)) - self.key = key self.cert = cert self.ca = ca diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index ec7f462b3..c3dcf7796 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -65,16 +65,19 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): self.cmd.run("/sbin/chkconfig --add %s" % (entry.attrib['name'])) self.logger.info("Installing Service %s" % (entry.get('name'))) rv = True - if entry.get('status') == 'off': + if (entry.get('status') == 'off' or + self.setup["servicemode"] == "build"): rv &= self.cmd.run((rcmd + " --level 0123456") % (entry.get('name'), entry.get('status'))).success - if entry.get("current_status") == "on": + if entry.get("current_status") == "on" and \ + self.setup["servicemode"] != "disabled": rv &= self.stop_service(entry).success else: rv &= self.cmd.run(rcmd % (entry.get('name'), entry.get('status'))).success - if entry.get("current_status") == "off": + if entry.get("current_status") == "off" and \ + self.setup["servicemode"] != "disabled": rv &= self.start_service(entry).success return rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 8ba1944d8..8f6bc5f37 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -154,7 +154,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get("current_exists", "true") == "true": # verify supplemental groups actual = [g[0] for g in self.user_supplementary_groups(entry)] - expected = [e.text for e in entry.findall("MemberOf")] + expected = [e.get("group", e.text).strip() + for e in entry.findall("MemberOf")] if set(expected) != set(actual): entry.set('qtext', "\n".join([entry.get('qtext', '')] + @@ -254,7 +255,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get('uid'): cmd.extend(['-u', entry.get('uid')]) cmd.extend(['-g', entry.get('group')]) - extras = [e.text for e in entry.findall("MemberOf")] + extras = [e.get("group", e.text).strip() + for e in entry.findall("MemberOf")] if extras: cmd.extend(['-G', ",".join(extras)]) cmd.extend(['-d', entry.get('home')]) diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py index 5e319a414..e52da081b 100644 --- a/src/lib/Bcfg2/Client/Tools/Portage.py +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -73,10 +73,10 @@ class Portage(Bcfg2.Client.Tools.PkgTool): self.logger.debug('Running equery check on %s' % entry.get('name')) - for line in self.cmd.run(["/usr/bin/equery", "-N", "check", - '=%s-%s' % - (entry.get('name'), - version)]).stdout.splitlines(): + for line in self.cmd.run( + ["/usr/bin/equery", "-N", "check", + '=%s-%s' % (entry.get('name'), + entry.get('version'))]).stdout.splitlines(): if '!!!' in line and line.split()[1] not in modlist: return False diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index f38615062..92572ef1d 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -492,7 +492,8 @@ class SELinuxSeportHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding and modifying entries """ (port, proto) = entry.get("name").split("/") - return (port, proto, '', entry.get("selinuxtype")) + return (port, proto, entry.get("mlsrange", ""), + entry.get("selinuxtype")) def _deleteargs(self, entry): return tuple(entry.get("name").split("/")) @@ -565,7 +566,7 @@ class SELinuxSefcontextHandler(SELinuxEntryHandler): """ argument list for adding, modifying, and deleting entries """ return (entry.get("name"), entry.get("selinuxtype"), self.filetypeargs[entry.get("filetype", "all")], - '', '') + entry.get("mlsrange", ""), '') def primarykey(self, entry): return ":".join([entry.tag, entry.get("name"), @@ -600,7 +601,7 @@ class SELinuxSenodeHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ (addr, netmask) = entry.get("name").split("/") - return (addr, netmask, entry.get("proto"), "", + return (addr, netmask, entry.get("proto"), entry.get("mlsrange", ""), entry.get("selinuxtype")) @@ -612,7 +613,8 @@ class SELinuxSeloginHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ - return (entry.get("name"), entry.get("selinuxuser"), "") + return (entry.get("name"), entry.get("selinuxuser"), + entry.get("mlsrange", "")) class SELinuxSeuserHandler(SELinuxEntryHandler): @@ -652,15 +654,16 @@ class SELinuxSeuserHandler(SELinuxEntryHandler): # prefix. see the comment in Install() above for more # details. rv = [entry.get("name"), - entry.get("roles", "").replace(" ", ",").split(",")] + entry.get("roles", "").replace(" ", ",").split(","), + '', entry.get("mlsrange", "")] if self.needs_prefix: - rv.extend(['', '', entry.get("prefix")]) + rv.append(entry.get("prefix")) else: key = self._key(entry) if key in self.all_records: attrs = self._key2attrs(key) if attrs['prefix'] != entry.get("prefix"): - rv.extend(['', '', entry.get("prefix")]) + rv.append(entry.get("prefix")) return tuple(rv) @@ -672,7 +675,8 @@ class SELinuxSeinterfaceHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ - return (entry.get("name"), '', entry.get("selinuxtype")) + return (entry.get("name"), entry.get("mlsrange", ""), + entry.get("selinuxtype")) class SELinuxSepermissiveHandler(SELinuxEntryHandler): diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 027d91c71..20a172d3d 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,6 +13,8 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} + conflicts = ['Chkconfig'] + def get_svc_command(self, service, action): return "/bin/systemctl %s %s.service" % (action, service.get('name')) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index e40ef750b..3bc261f2f 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -19,7 +19,7 @@ def prompt(msg): while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: os.read(sys.stdin.fileno(), 4096) try: - ans = input(msg.encode(sys.stdout.encoding, 'replace')) + ans = input(msg) return ans in ['y', 'Y'] except EOFError: # python 2.4.3 on CentOS doesn't like ^C for some reason diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 4e9239e26..d034c0777 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -89,11 +89,28 @@ def u_str(string, encoding=None): else: return unicode(string) +try: + from functools import wraps +except ImportError: + def wraps(wrapped): # pylint: disable=W0613 + """ implementation of functools.wraps() for python 2.4 """ + return lambda f: f + + # base64 compat if sys.hexversion >= 0x03000000: from base64 import b64encode as _b64encode, b64decode as _b64decode - b64encode = lambda s: _b64encode(s.encode('UTF-8')).decode('UTF-8') - b64decode = lambda s: _b64decode(s.encode('UTF-8')).decode('UTF-8') + + @wraps(_b64encode) + def b64encode(val, **kwargs): # pylint: disable=C0111 + try: + return _b64encode(val, **kwargs) + except TypeError: + return _b64encode(val.encode('UTF-8'), **kwargs).decode('UTF-8') + + @wraps(_b64decode) + def b64decode(val, **kwargs): # pylint: disable=C0111 + return _b64decode(val.encode('UTF-8'), **kwargs).decode('UTF-8') else: from base64 import b64encode, b64decode @@ -242,14 +259,6 @@ except ImportError: from md5 import md5 -try: - from functools import wraps -except ImportError: - def wraps(wrapped): # pylint: disable=W0613 - """ implementation of functools.wraps() for python 2.4 """ - return lambda f: f - - def oct_mode(mode): """ Convert a decimal number describing a POSIX permissions mode to a string giving the octal mode. In Python 2, this is a synonym diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py index 5c51dad1e..73bdd0b3a 100644 --- a/src/lib/Bcfg2/Reporting/Transport/__init__.py +++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py @@ -2,11 +2,11 @@ Public transport routines """ -import traceback - +import sys from Bcfg2.Reporting.Transport.base import TransportError, \ TransportImportError + def load_transport(transport_name, setup): """ Try to load the transport. Raise TransportImportError on failure @@ -18,13 +18,14 @@ def load_transport(transport_name, setup): try: mod = __import__(transport_name) except: - raise TransportImportError("Unavailable") + raise TransportImportError("Error importing transport %s: %s" % + (transport_name, sys.exc_info()[1])) try: - cls = getattr(mod, transport_name) - return cls(setup) + return getattr(mod, transport_name)(setup) except: - raise TransportImportError("Transport unavailable: %s" % - traceback.format_exc().splitlines()[-1]) + raise TransportImportError("Error instantiating transport %s: %s" % + (transport_name, sys.exc_info()[1])) + def load_transport_from_config(setup): """Load the transport in the config... eventually""" @@ -32,4 +33,3 @@ def load_transport_from_config(setup): return load_transport(setup['reporting_transport'], setup) except KeyError: raise TransportImportError('Transport missing in config') - diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index e63c180a8..2e75c1d1a 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -3,7 +3,7 @@ import sys from django.core.exceptions import ImproperlyConfigured try: - from django.db import models + from django.db import models, backend, connection except ImproperlyConfigured: e = sys.exc_info()[1] print("Reports: unable to import django models: %s" % e) @@ -49,6 +49,20 @@ def hash_entry(entry_dict): return hash(cPickle.dumps(dataset)) +_our_backend = None +def _quote(value): + """ + Quote a string to use as a table name or column + + Newer versions and various drivers require an argument + https://code.djangoproject.com/ticket/13630 + """ + global _our_backend + if not _our_backend: + _our_backend = backend.DatabaseOperations(connection) + return _our_backend.quote_name(value) + + class Client(models.Model): """Object representing every client we have seen stats for.""" creation = models.DateTimeField(auto_now_add=True) @@ -77,16 +91,20 @@ class InteractionManager(models.Manager): cursor = connection.cursor() cfilter = "expiration is null" - sql = 'select ri.id, x.client_id from (select client_id, MAX(timestamp) ' + \ - 'as timer from Reporting_interaction' + sql = 'select ri.id, x.client_id from ' + \ + '(select client_id, MAX(timestamp) as timer from ' + \ + _quote('Reporting_interaction') if maxdate: if not isinstance(maxdate, datetime): raise ValueError('Expected a datetime object') sql = sql + " where timestamp <= '%s' " % maxdate cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate) - sql = sql + ' GROUP BY client_id) x, Reporting_interaction ri where ' + \ - 'ri.client_id = x.client_id AND ri.timestamp = x.timer' - sql = sql + " and x.client_id in (select id from Reporting_client where %s)" % cfilter + sql = sql + ' GROUP BY client_id) x, ' + \ + _quote('Reporting_interaction') + \ + ' ri where ri.client_id = x.client_id AND' + \ + ' ri.timestamp = x.timer and x.client_id in' + \ + ' (select id from %s where %s)' % \ + (_quote('Reporting_client'), cfilter) try: cursor.execute(sql) return [item[0] for item in cursor.fetchall()] diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py index d317cc03b..06a419354 100644 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -128,6 +128,7 @@ class MetadataCore(Mode): except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] self.errExit("Core load failed: %s" % msg) + self.bcore.load_plugins() self.bcore.fam.handle_event_set() self.metadata = self.bcore.metadata diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 48455819d..b05ad9d41 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -118,11 +118,11 @@ class Core(BaseCore): self.logger.error("Server startup failed: %s" % err) self.context.close() return False - self.server.register_instance(self) return True def _block(self): """ Enter the blocking infinite loop. """ + self.server.register_instance(self) try: self.server.serve_forever() finally: diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index 8bfb76461..7c3b2d9cc 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -1,8 +1,9 @@ -""" check files for various required comments """ +""" Check files for various required comments. """ import os import lxml.etree import Bcfg2.Server.Lint +from Bcfg2.Server import XI_NAMESPACE from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ import CfgPlaintextGenerator from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator @@ -11,7 +12,10 @@ from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML class Comments(Bcfg2.Server.Lint.ServerPlugin): - """ check files for various required headers """ + """ The Comments lint plugin checks files for header comments that + give information about the files. For instance, you can require + SVN keywords in a comment, or require the name of the maintainer + of a Genshi template, and so on. """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.config_cache = {} @@ -27,21 +31,43 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): def Errors(cls): return {"unexpanded-keywords": "warning", "keywords-not-found": "warning", - "comments-not-found": "warning"} + "comments-not-found": "warning", + "broken-xinclude-chain": "warning"} def required_keywords(self, rtype): - """ given a file type, fetch the list of required VCS keywords - from the bcfg2-lint config """ + """ Given a file type, fetch the list of required VCS keywords + from the bcfg2-lint config. Valid file types are documented + in :manpage:`bcfg2-lint.conf(5)`. + + :param rtype: The file type + :type rtype: string + :returns: list - the required items + """ return self.required_items(rtype, "keyword") def required_comments(self, rtype): - """ given a file type, fetch the list of required comments - from the bcfg2-lint config """ + """ Given a file type, fetch the list of required comments + from the bcfg2-lint config. Valid file types are documented + in :manpage:`bcfg2-lint.conf(5)`. + + :param rtype: The file type + :type rtype: string + :returns: list - the required items + """ return self.required_items(rtype, "comment") def required_items(self, rtype, itype): - """ given a file type and item type (comment or keyword), - fetch the list of required items from the bcfg2-lint config """ + """ Given a file type and item type (``comment`` or + ``keyword``), fetch the list of required items from the + bcfg2-lint config. Valid file types are documented in + :manpage:`bcfg2-lint.conf(5)`. + + :param rtype: The file type + :type rtype: string + :param itype: The item type (``comment`` or ``keyword``) + :type itype: string + :returns: list - the required items + """ if itype not in self.config_cache: self.config_cache[itype] = {} @@ -62,7 +88,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): return self.config_cache[itype][rtype] def check_bundles(self): - """ check bundle files for required headers """ + """ Check bundle files for required comments. """ if 'Bundler' in self.core.plugins: for bundle in self.core.plugins['Bundler'].entries.values(): xdata = None @@ -78,15 +104,41 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_xml(bundle.name, xdata, rtype) def check_properties(self): - """ check properties files for required headers """ + """ Check Properties files for required comments. """ if 'Properties' in self.core.plugins: props = self.core.plugins['Properties'] - for propfile, pdata in props.store.entries.items(): + for propfile, pdata in props.entries.items(): if os.path.splitext(propfile)[1] == ".xml": self.check_xml(pdata.name, pdata.xdata, 'properties') + def has_all_xincludes(self, mfile): + """ Return True if :attr:`Bcfg2.Server.Lint.Plugin.files` + includes all XIncludes listed in the specified metadata type, + false otherwise. In other words, this returns True if + bcfg2-lint is dealing with complete metadata. + + :param mfile: The metadata file ("clients.xml" or + "groups.xml") to check for XIncludes + :type mfile: string + :returns: bool + """ + if self.files is None: + return True + else: + path = os.path.join(self.metadata.data, mfile) + if path in self.files: + xdata = lxml.etree.parse(path) + for el in xdata.findall('./%sinclude' % XI_NAMESPACE): + if not self.has_all_xincludes(el.get('href')): + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: could not " + "include %s" % path) + return False + + return True + def check_metadata(self): - """ check metadata files for required headers """ + """ Check Metadata files for required comments. """ if self.has_all_xincludes("groups.xml"): self.check_xml(os.path.join(self.metadata.data, "groups.xml"), self.metadata.groups_xml.data, @@ -97,7 +149,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): "metadata") def check_cfg(self): - """ check Cfg files and info.xml files for required headers """ + """ Check Cfg files and ``info.xml`` files for required + comments. """ if 'Cfg' in self.core.plugins: for entryset in self.core.plugins['Cfg'].entries.values(): for entry in entryset.entries.values(): @@ -117,29 +170,57 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_plaintext(entry.name, entry.data, rtype) def check_probes(self): - """ check probes for required headers """ + """ Check Probes for required comments """ if 'Probes' in self.core.plugins: for probe in self.core.plugins['Probes'].probes.entries.values(): self.check_plaintext(probe.name, probe.data, "probes") def check_xml(self, filename, xdata, rtype): - """ check generic XML files for required headers """ + """ Generic check to check an XML file for required comments. + + :param filename: The filename + :type filename: string + :param xdata: The file data + :type xdata: lxml.etree._Element + :param rtype: The type of file. Available types are + documented in :manpage:`bcfg2-lint.conf(5)`. + :type rtype: string + """ self.check_lines(filename, [str(el) for el in xdata.getiterator(lxml.etree.Comment)], rtype) def check_plaintext(self, filename, data, rtype): - """ check generic plaintext files for required headers """ + """ Generic check to check a plain text file for required + comments. + + :param filename: The filename + :type filename: string + :param data: The file data + :type data: string + :param rtype: The type of file. Available types are + documented in :manpage:`bcfg2-lint.conf(5)`. + :type rtype: string + """ self.check_lines(filename, data.splitlines(), rtype) def check_lines(self, filename, lines, rtype): - """ generic header check for a set of lines """ + """ Generic header check for a set of lines. + + :param filename: The filename + :type filename: string + :param lines: The data to check + :type lines: list of strings + :param rtype: The type of file. Available types are + documented in :manpage:`bcfg2-lint.conf(5)`. + :type rtype: string + """ if self.HandlesFile(filename): # found is trivalent: - # False == not found - # None == found but not expanded - # True == found and expanded + # False == keyword not found + # None == keyword found but not expanded + # True == keyword found and expanded found = dict((k, False) for k in self.required_keywords(rtype)) for line in lines: diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index caee238bc..a1d0b7fa1 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -1,4 +1,4 @@ -""" Check Genshi templates for syntax errors """ +""" Check Genshi templates for syntax errors. """ import sys import Bcfg2.Server.Lint @@ -8,10 +8,9 @@ from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator class Genshi(Bcfg2.Server.Lint.ServerPlugin): - """ Check Genshi templates for syntax errors """ + """ Check Genshi templates for syntax errors. """ def Run(self): - """ run plugin """ if 'Cfg' in self.core.plugins: self.check_cfg() if 'Bundler' in self.core.plugins: @@ -22,7 +21,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): return {"genshi-syntax-error": "error"} def check_cfg(self): - """ Check genshi templates in Cfg for syntax errors """ + """ Check genshi templates in Cfg for syntax errors. """ for entryset in self.core.plugins['Cfg'].entries.values(): for entry in entryset.entries.values(): if (self.HandlesFile(entry.name) and @@ -37,7 +36,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): "Genshi syntax error: %s" % err) def check_bundler(self): - """ Check templates in Bundler for syntax errors """ + """ Check templates in Bundler for syntax errors. """ loader = TemplateLoader() for entry in self.core.plugins['Bundler'].entries.values(): diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py index e41ed867e..730f32750 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupNames.py +++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py @@ -1,4 +1,4 @@ -""" ensure that all named groups are valid group names """ +""" Ensure that all named groups are valid group names. """ import os import re @@ -6,8 +6,15 @@ import Bcfg2.Server.Lint class GroupNames(Bcfg2.Server.Lint.ServerPlugin): - """ ensure that all named groups are valid group names """ + """ Ensure that all named groups are valid group names. """ + + #: A string regex that matches only valid group names. Currently, + #: a group name is considered valid if it contains only + #: non-whitespace characters. pattern = r'\S+$' + + #: A compiled regex for + #: :attr:`Bcfg2.Server.Lint.GroupNames.GroupNames.pattern` valid = re.compile(r'^' + pattern) def Run(self): @@ -26,7 +33,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): return {"invalid-group-name": "error"} def check_rules(self): - """ Check groups used in the Rules plugin for validity """ + """ Check groups used in the Rules plugin for validity. """ for rules in self.core.plugins['Rules'].entries.values(): if not self.HandlesFile(rules.name): continue @@ -35,7 +42,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): os.path.join(self.config['repo'], rules.name)) def check_bundles(self): - """ Check groups used in the Bundler plugin for validity """ + """ Check groups used in the Bundler plugin for validity. """ for bundle in self.core.plugins['Bundler'].entries.values(): if self.HandlesFile(bundle.name) and bundle.template is None: self.check_entries(bundle.xdata.xpath("//Group"), @@ -43,7 +50,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): def check_metadata(self): """ Check groups used or declared in the Metadata plugin for - validity """ + validity. """ self.check_entries(self.metadata.groups_xml.xdata.xpath("//Group"), os.path.join(self.config['repo'], self.metadata.groups_xml.name)) @@ -61,7 +68,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): def check_cfg(self): """ Check groups used in group-specific files in the Cfg - plugin for validity """ + plugin for validity. """ for root, _, files in os.walk(self.core.plugins['Cfg'].data): for fname in files: basename = os.path.basename(root) @@ -74,7 +81,14 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): def check_entries(self, entries, fname): """ Check a generic list of XML entries for <Group> tags with - invalid name attributes """ + invalid name attributes. + + :param entries: A list of XML <Group> tags whose ``name`` + attributes will be validated. + :type entries: list of lxml.etree._Element + :param fname: The filename the entry list came from + :type fname: string + """ for grp in entries: if not self.valid.search(grp.get("name")): self.LintError("invalid-group-name", diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index f2349f861..184f657b7 100644 --- a/src/lib/Bcfg2/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -1,4 +1,4 @@ -""" ensure that all config files have an info.xml file""" +""" Ensure that all config files have a valid info.xml file. """ import os import Bcfg2.Options @@ -7,7 +7,14 @@ from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML class InfoXML(Bcfg2.Server.Lint.ServerPlugin): - """ ensure that all config files have an info.xml file""" + """ Ensure that all config files have a valid info.xml file. This + plugin can check for: + + * Missing ``info.xml`` files; + * Use of deprecated ``info``/``:info`` files; + * Paranoid mode disabled in an ``info.xml`` file; + * Required attributes missing from ``info.xml`` + """ def Run(self): if 'Cfg' not in self.core.plugins: return @@ -29,11 +36,10 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): def Errors(cls): return {"no-infoxml": "warning", "paranoid-false": "warning", - "broken-xinclude-chain": "warning", "required-infoxml-attrs-missing": "error"} def check_infoxml(self, fname, xdata): - """ verify that info.xml contains everything it should """ + """ Verify that info.xml contains everything it should. """ for info in xdata.getroottree().findall("//Info"): required = [] if "required_attrs" in self.config: diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index 44d02c2ff..2419c3d43 100644 --- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -57,7 +57,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): else: threshold = 0.75 rv = [] - elist = entries.items() + elist = list(entries.items()) while elist: result = self._find_similar(elist.pop(0), copy.copy(elist), threshold) diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 7a2fd3fe9..f2464b585 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -1,5 +1,5 @@ -""" verify attributes for configuration entries that cannot be -verified with an XML schema alone""" +""" Verify attributes for configuration entries that cannot be +verified with an XML schema alone. """ import os import re @@ -14,7 +14,8 @@ except ImportError: HAS_GENSHI = False -# format verifying functions +# format verifying functions. TODO: These should be moved into XML +# schemas where possible. def is_filename(val): """ Return True if val is a string describing a valid full path """ @@ -52,8 +53,8 @@ def is_device_mode(val): class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): - """ verify attributes for configuration entries that cannot be - verified with an XML schema alone """ + """ Verify attributes for configuration entries that cannot be + verified with an XML schema alone. """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.required_attrs = dict( @@ -114,8 +115,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): SEInterface={None: dict(name=None, selinuxtype=is_selinux_type)}, SEPermissive={None: dict(name=is_selinux_type)}, POSIXGroup={None: dict(name=is_username)}, - POSIXUser={None: dict(name=is_username)}, - MemberOf={None: dict(__text__=is_username)}) + POSIXUser={None: dict(name=is_username)}) def Run(self): self.check_packages() @@ -135,7 +135,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): "extra-attrs": "warning"} def check_packages(self): - """ check package sources for Source entries with missing attrs """ + """ Check Packages sources for Source entries with missing + attributes. """ if 'Packages' not in self.core.plugins: return @@ -175,7 +176,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): rules.name)) def check_bundles(self): - """ check bundles for BoundPath entries with missing attrs """ + """ Check bundles for BoundPath entries with missing + attrs. """ if 'Bundler' not in self.core.plugins: return @@ -186,7 +188,13 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): self.check_entry(path, bundle.name) def check_entry(self, entry, filename): - """ generic entry check """ + """ Generic entry check. + + :param entry: The XML entry to check for missing attributes. + :type entry: lxml.etree._Element + :param filename: The filename the entry came from + :type filename: string + """ if self.HandlesFile(filename): name = entry.get('name') tag = entry.tag diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 946ef8270..ca9f138ef 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -1,4 +1,5 @@ -""" Ensure that the repo validates """ +""" Ensure that all XML files in the Bcfg2 repository validate +according to their respective schemas. """ import os import sys @@ -10,10 +11,19 @@ from Bcfg2.Utils import Executor class Validate(Bcfg2.Server.Lint.ServerlessPlugin): - """ Ensure that the repo validates """ + """ Ensure that all XML files in the Bcfg2 repository validate + according to their respective schemas. """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + + #: A dict of <file glob>: <schema file> that maps files in the + #: Bcfg2 specification to their schemas. The globs are + #: extended :mod:`fnmatch` globs that also support ``**``, + #: which matches any number of any characters, including + #: forward slashes. The schema files are relative to the + #: schema directory, which can be controlled by the + #: ``bcfg2-lint --schema`` option. self.filesets = \ {"Metadata/groups.xml": "metadata.xsd", "Metadata/clients.xml": "clients.xsd", @@ -76,7 +86,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "input-output-error": "error"} def check_properties(self): - """ check Properties files against their schemas """ + """ Check Properties files against their schemas. """ for filename in self.filelists['props']: schemafile = "%s.xsd" % os.path.splitext(filename)[0] if os.path.exists(schemafile): @@ -90,7 +100,11 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): def parse(self, filename): """ Parse an XML file, raising the appropriate LintErrors if it can't be parsed or read. Return the - lxml.etree._ElementTree parsed from the file. """ + lxml.etree._ElementTree parsed from the file. + + :param filename: The full path to the file to parse + :type filename: string + :returns: lxml.etree._ElementTree - the parsed data""" try: return lxml.etree.parse(filename) except SyntaxError: @@ -105,8 +119,20 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): return False def validate(self, filename, schemafile, schema=None): - """validate a file against the given lxml.etree.Schema. - return True on success, False on failure """ + """ Validate a file against the given schema. + + :param filename: The full path to the file to validate + :type filename: string + :param schemafile: The full path to the schema file to + validate against + :type schemafile: string + :param schema: The loaded schema to validate against. This + can be used to avoid parsing a single schema + file for every file that needs to be validate + against it. + :type schema: lxml.etree.Schema + :returns: bool - True if the file validates, false otherwise + """ if schema is None: # if no schema object was provided, instantiate one schema = self._load_schema(schemafile) @@ -127,7 +153,14 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): return True def get_filelists(self): - """ get lists of different kinds of files to validate """ + """ Get lists of different kinds of files to validate. This + doesn't return anything, but it sets + :attr:`Bcfg2.Server.Lint.Validate.Validate.filelists` to a + dict whose keys are path globs given in + :attr:`Bcfg2.Server.Lint.Validate.Validate.filesets` and whose + values are lists of the full paths to all files in the Bcfg2 + repository (or given with ``bcfg2-lint --stdin``) that match + the glob.""" if self.files is not None: listfiles = lambda p: fnmatch.filter(self.files, os.path.join('*', p)) @@ -154,7 +187,13 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.filelists['props'] = listfiles("Properties/*.xml") def _load_schema(self, filename): - """ load an XML schema document, returning the Schema object """ + """ Load an XML schema document, returning the Schema object + and raising appropriate lint errors on failure. + + :param filename: The full path to the schema file to load. + :type filename: string + :returns: lxml.etree.Schema - The loaded schema data + """ try: return lxml.etree.XMLSchema(lxml.etree.parse(filename)) except IOError: diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 11afdd75d..28644263f 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -9,10 +9,9 @@ import lxml.etree import fcntl import termios import struct -from Bcfg2.Server import XI_NAMESPACE from Bcfg2.Compat import walk_packages -__all__ = [m[1] for m in walk_packages(path=__path__)] +plugins = [m[1] for m in walk_packages(path=__path__)] # pylint: disable=C0103 def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -45,30 +44,56 @@ def get_termsize(): class Plugin(object): - """ base class for ServerlessPlugin and ServerPlugin """ + """ Base class for all bcfg2-lint plugins """ def __init__(self, config, errorhandler=None, files=None): + """ + :param config: A :mod:`Bcfg2.Options` setup dict + :type config: dict + :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` + that will be used to handle lint errors. + If one is not provided, a new one will be + instantiated. + :type errorhandler: Bcfg2.Server.Lint.ErrorHandler + :param files: A list of files to run bcfg2-lint against. (See + the bcfg2-lint ``--stdin`` option.) + :type files: list of strings + """ + + #: The list of files that bcfg2-lint should be run against self.files = files + + #: The Bcfg2.Options setup dict self.config = config + self.logger = logging.getLogger('bcfg2-lint') if errorhandler is None: + #: The error handler self.errorhandler = ErrorHandler() else: self.errorhandler = errorhandler self.errorhandler.RegisterErrors(self.Errors()) def Run(self): - """ run the plugin. must be overloaded by child classes """ - pass + """ Run the plugin. Must be overloaded by child classes. """ + raise NotImplementedError @classmethod def Errors(cls): - """ returns a dict of errors the plugin supplies. must be - overloaded by child classes """ + """ Returns a dict of errors the plugin supplies, in a format + suitable for passing to + :func:`Bcfg2.Server.Lint.ErrorHandler.RegisterErrors`. + + Must be overloaded by child classes. + + :returns: dict + """ + raise NotImplementedError def HandlesFile(self, fname): - """ returns true if the given file should be handled by the - plugin according to the files list, false otherwise """ + """ Returns True if the given file should be handled by the + plugin according to :attr:`Bcfg2.Server.Lint.Plugin.files`, + False otherwise. """ return (self.files is None or fname in self.files or os.path.join(self.config['repo'], fname) in self.files or @@ -77,12 +102,27 @@ class Plugin(object): fname)) in self.files) def LintError(self, err, msg): - """ record an error in the lint process """ + """ Raise an error from the lint process. + + :param err: The name of the error being raised. This name + must be a key in the dict returned by + :func:`Bcfg2.Server.Lint.Plugin.Errors`. + :type err: string + :param msg: The freeform message to display to the end user. + :type msg: string + """ self.errorhandler.dispatch(err, msg) def RenderXML(self, element, keep_text=False): - """render an XML element for error output -- line number - prefixed, no children""" + """ Render an XML element for error output. This prefixes the + line number and removes children for nicer display. + + :param element: The element to render + :type element: lxml.etree._Element + :param keep_text: Do not discard text content from the element + for display + :type keep_text: boolean + """ xml = None if len(element) or element.text: el = copy(element) @@ -100,11 +140,18 @@ class Plugin(object): return " line %s: %s" % (element.sourceline, xml) -class ErrorHandler (object): - """ a class to handle errors for bcfg2-lint plugins """ +class ErrorHandler(object): + """ A class to handle errors for bcfg2-lint plugins """ - def __init__(self, config=None): + def __init__(self, errors=None): + """ + :param config: An initial dict of errors to register + :type config: dict + """ + #: The number of errors passed to this error handler self.errors = 0 + + #: The number of warnings passed to this error handler self.warnings = 0 self.logger = logging.getLogger('bcfg2-lint') @@ -114,17 +161,25 @@ class ErrorHandler (object): twrap = textwrap.TextWrapper(initial_indent=" ", subsequent_indent=" ", width=termsize[0]) + #: A function to wrap text to the width of the terminal self._wrapper = twrap.wrap else: self._wrapper = lambda s: [s] + #: A dict of registered errors self.errortypes = dict() - if config is not None: - self.RegisterErrors(dict(config.items())) + if errors is not None: + self.RegisterErrors(dict(errors.items())) def RegisterErrors(self, errors): - """ Register a dict of errors (name: default level) that a - plugin may raise """ + """ Register a dict of errors that a plugin may raise. The + keys of the dict are short strings that describe each error; + the values are the default error handling for that error + ("error", "warning", or "silent"). + + :param errors: The error dict + :type errors: dict + """ for err, action in errors.items(): if err not in self.errortypes: if "warn" in action: @@ -135,7 +190,16 @@ class ErrorHandler (object): self.errortypes[err] = self.debug def dispatch(self, err, msg): - """ Dispatch an error to the correct handler """ + """ Dispatch an error to the correct handler. + + :param err: The name of the error being raised. This name + must be a key in + :attr:`Bcfg2.Server.Lint.ErrorHandler.errortypes`, + the dict of registered errors. + :type err: string + :param msg: The freeform message to display to the end user. + :type msg: string + """ if err in self.errortypes: self.errortypes[err](msg) self.logger.debug(" (%s)" % err) @@ -145,22 +209,34 @@ class ErrorHandler (object): self.logger.warning("Unknown error %s" % err) def error(self, msg): - """ log an error condition """ + """ Log an error condition. + + :param msg: The freeform message to display to the end user. + :type msg: string + """ self.errors += 1 self._log(msg, self.logger.error, prefix="ERROR: ") def warn(self, msg): - """ log a warning condition """ + """ Log a warning condition. + + :param msg: The freeform message to display to the end user. + :type msg: string + """ self.warnings += 1 self._log(msg, self.logger.warning, prefix="WARNING: ") def debug(self, msg): - """ log a silent/debug condition """ + """ Log a silent/debug condition. + + :param msg: The freeform message to display to the end user. + :type msg: string + """ self._log(msg, self.logger.debug) def _log(self, msg, logfunc, prefix=""): """ Generic log function that logs a message with the given - function after wrapping it for the terminal width """ + function after wrapping it for the terminal width. """ # a message may itself consist of multiple lines. wrap() will # elide them all into a single paragraph, which we don't want. # so we split the message into its paragraphs and wrap each @@ -180,37 +256,37 @@ class ErrorHandler (object): logfunc(line) -class ServerlessPlugin (Plugin): - """ base class for plugins that are run before the server starts - up (i.e., plugins that check things that may prevent the server - from starting up) """ +class ServerlessPlugin(Plugin): # pylint: disable=W0223 + """ Base class for bcfg2-lint plugins that are run before the + server starts up (i.e., plugins that check things that may prevent + the server from starting up). """ pass -class ServerPlugin (Plugin): - """ base class for plugins that check things that require the - running Bcfg2 server """ - def __init__(self, core, config, **kwargs): - Plugin.__init__(self, config, **kwargs) +class ServerPlugin(Plugin): # pylint: disable=W0223 + """ Base class for bcfg2-lint plugins that check things that + require the running Bcfg2 server. """ + + def __init__(self, core, config, errorhandler=None, files=None): + """ + :param core: The Bcfg2 server core + :type core: Bcfg2.Server.Core.BaseCore + :param config: A :mod:`Bcfg2.Options` setup dict + :type config: dict + :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` + that will be used to handle lint errors. + If one is not provided, a new one will be + instantiated. + :type errorhandler: Bcfg2.Server.Lint.ErrorHandler + :param files: A list of files to run bcfg2-lint against. (See + the bcfg2-lint ``--stdin`` option.) + :type files: list of strings + """ + Plugin.__init__(self, config, errorhandler=errorhandler, files=files) + + #: The server core self.core = core self.logger = self.core.logger - self.metadata = self.core.metadata - self.errorhandler.RegisterErrors({"broken-xinclude-chain": "warning"}) - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./%sinclude' % XI_NAMESPACE): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not " - "include %s" % path) - return False - - return True + #: The metadata plugin + self.metadata = self.core.metadata diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index d8290d844..d114b0873 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -126,10 +126,10 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various bundle checks """ + """ Perform various :ref:`Bundler + <server-plugins-structures-bundler-index>` checks. """ def Run(self): - """ run plugin """ self.missing_bundles() for bundle in self.core.plugins['Bundler'].entries.values(): if self.HandlesFile(bundle.name): @@ -143,7 +143,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): "genshi-extension-bundle": "error"} def missing_bundles(self): - """ find bundles listed in Metadata but not implemented in Bundler """ + """ Find bundles listed in Metadata but not implemented in + Bundler. """ if self.files is None: # when given a list of files on stdin, this check is # useless, so skip it diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 9042a979e..3e5508160 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -125,7 +125,12 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): - """ bcfg2-lint plugin for GroupPatterns """ + """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns + <server-plugins-grouping-grouppatterns>` patterns for validity. + This is simply done by trying to create a + :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for + each pattern, and catching exceptions and presenting them as + ``bcfg2-lint`` errors.""" def Run(self): cfg = self.core.plugins['GroupPatterns'].config diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 49e36f72b..2bc82caa9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -945,7 +945,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Client %s set as nonexistent group %s" % (client, group)) - def set_profile(self, client, profile, addresspair): + def set_profile(self, client, profile, # pylint: disable=W0221 + addresspair, require_public=True): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) @@ -957,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] - if not group.is_public: + if require_public and not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) @@ -1128,7 +1129,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = self.default if pgroup: - self.set_profile(client, pgroup, (None, None)) + self.set_profile(client, pgroup, (None, None), + require_public=False) profile = _add_group(pgroup) else: msg = "Cannot add new client %s; no default group set" % client @@ -1477,7 +1479,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): - """ bcfg2-lint plugin for Metadata """ + """ ``bcfg2-lint`` plugin for :ref:`Metadata + <server-plugins-grouping-metadata>`. This checks for several things: + + * ``<Client>`` tags nested inside other ``<Client>`` tags; + * Deprecated options (like ``location="floating"``); + * Profiles that don't exist, or that aren't profile groups; + * Groups or clients that are defined multiple times; + * Multiple default groups or a default group that isn't a profile + group. + """ def Run(self): self.nested_clients() @@ -1500,8 +1511,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "default-is-not-profile": "error"} def deprecated_options(self): - """ check for the location='floating' option, which has been - deprecated in favor of floating='true' """ + """ Check for the ``location='floating'`` option, which has + been deprecated in favor of ``floating='true'``. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1519,8 +1530,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (loc, floating, self.RenderXML(el))) def nested_clients(self): - """ check for a Client tag inside a Client tag, which doesn't - make any sense """ + """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, + which is either redundant or will never match. """ groupdata = self.metadata.groups_xml.xdata for el in groupdata.xpath("//Client//Client"): self.LintError("nested-client-tags", @@ -1528,8 +1539,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (el.get("name"), self.RenderXML(el))) def bogus_profiles(self): - """ check for clients that have profiles that are either not - flagged as public groups in groups.xml, or don't exist """ + """ Check for clients that have profiles that are either not + flagged as profile groups in ``groups.xml``, or don't exist. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1547,20 +1558,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (profile, client.get("name"), profile, self.RenderXML(client))) - def duplicate_groups(self): - """ check for groups that are defined twice. We count a group - tag as a definition if it a) has profile or public set; or b) - has any children. """ - self.duplicate_entries( - self.metadata.groups_xml.xdata.xpath("//Groups/Group") + - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"), - "group", - include=lambda g: (g.get("profile") or - g.get("public") or - g.getchildren())) - def duplicate_default_groups(self): - """ check for multiple default groups """ + """ Check for multiple default groups. """ defaults = [] for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): @@ -1572,7 +1571,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "\n".join(defaults)) def duplicate_clients(self): - """ check for clients that are defined twice. """ + """ Check for clients that are defined more than once. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1580,17 +1579,34 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): self.metadata.clients_xml.xdata.xpath("//Client"), "client") - def duplicate_entries(self, allentries, etype, include=None): - """ generic duplicate entry finder """ - if include is None: - include = lambda e: True + def duplicate_groups(self): + """ Check for groups that are defined more than once. We + count a group tag as a definition if it a) has profile or + public set; or b) has any children.""" + allgroups = [ + g + for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") + if g.get("profile") or g.get("public") or g.getchildren()] + self.duplicate_entries(allgroups, "group") + + def duplicate_entries(self, allentries, etype): + """ Generic duplicate entry finder. + + :param allentries: A list of all entries to check for + duplicates. + :type allentries: list of lxml.etree._Element + :param etype: The entry type. This will be used to determine + the error name (``duplicate-<etype>``) and for + display to the end user. + :type etype: string + """ entries = dict() for el in allentries: - if include(el): - if el.get("name") in entries: - entries[el.get("name")].append(self.RenderXML(el)) - else: - entries[el.get("name")] = [self.RenderXML(el)] + if el.get("name") in entries: + entries[el.get("name")].append(self.RenderXML(el)) + else: + entries[el.get("name")] = [self.RenderXML(el)] for ename, els in entries.items(): if len(els) > 1: self.LintError("duplicate-%s" % etype, @@ -1598,7 +1614,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (etype.title(), ename, "\n".join(els))) def default_is_profile(self): - """ ensure that the default group is a profile group """ + """ Ensure that the default group is a profile group. """ if (self.metadata.default and not self.metadata.groups[self.metadata.default].is_profile): xdata = \ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 48c580be1..dba56eed2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -39,6 +39,11 @@ class AptCollection(Collection): else: lines.append("deb %s %s %s" % (source.url, source.version, " ".join(source.components))) + if source.debsrc: + lines.append("deb-src %s %s %s" % + (source.url, + source.version, + " ".join(source.components))) lines.append("") return "\n".join(lines) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 3319dda78..767ac13ac 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -152,6 +152,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: this source self.whitelist = [item.text for item in xsource.findall('Whitelist')] + #: Whether or not to include deb-src lines in the generated APT + #: configuration + self.debsrc = xsource.get('debsrc', 'false') == 'true' + #: A dict of repository options that will be included in the #: configuration generated on the server side (if such is #: applicable; most backends do not generate any sort of diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 052c362ab..567a16c40 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,7 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -445,7 +445,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, try: open(localfile, 'w').write(urlopen(key).read()) keys.append(key) - except HTTPError: + except (URLError, HTTPError): err = sys.exc_info()[1] self.logger.error("Packages: Error downloading %s: %s" % (key, err)) diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index 7dac907e1..a1dcb575f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -177,7 +177,10 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): - """ find duplicate Pkgmgr entries with the same priority """ + """ Find duplicate :ref:`Pkgmgr + <server-plugins-generators-pkgmgr>` entries with the same + priority. """ + def Run(self): pset = set() for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr', @@ -202,12 +205,13 @@ class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): # check if package is already listed with same # priority, type, grp if ptuple in pset: - self.LintError("duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) + self.LintError( + "duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) else: pset.add(ptuple) - + @classmethod def Errors(cls): - return {"duplicate-packages":"error"} + return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 7e4935d74..e97607093 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -65,7 +65,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data.encode('utf-8')) + return str.__new__(cls, data) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -224,15 +224,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - try: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str( - self.probedata[client][probe]).decode('utf-8')) - except AttributeError: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self.probedata[client][probe]) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 1264fd1cf..84dcf2780 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -173,8 +173,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for name in names[cmeta.hostname]: newnames.add(name.split('.')[0]) try: - newips.add(self.get_ipcache_entry(name)[0]) - except PluginExecutionError: + newips.update(self.get_ipcache_entry(name)[0]) + except: # pylint: disable=W0702 continue names[cmeta.hostname].update(newnames) names[cmeta.hostname].update(cmeta.addresses) @@ -290,7 +290,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: # need to add entry try: - ipaddr = socket.gethostbyname(client) + ipaddr = set([info[4][0] + for info in socket.getaddrinfo(client, None)]) self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index e834759c2..050ba3b3e 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -96,7 +96,18 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): - """ find duplicate Pkgmgr entries with the same priority """ + """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper + <server-plugins-connectors-templatehelper>` modules are valid. + This can check for: + + * A TemplateHelper module that cannot be imported due to syntax or + other compile-time errors; + * A TemplateHelper module that does not have an ``__export__`` + attribute, or whose ``__export__`` is not a list; + * Bogus symbols listed in ``__export__``, including symbols that + don't exist, that are reserved, or that start with underscores. + """ + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.reserved_keywords = dir(HelperModule("foo.py")) @@ -107,7 +118,11 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): self.check_helper(helper.name) def check_helper(self, helper): - """ check a helper module for export errors """ + """ Check a single helper module. + + :param helper: The filename of the helper module + :type helper: string + """ module_name = MODULE_RE.search(helper).group(1) try: diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 13c756049..eea2183f7 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -346,7 +346,7 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, :param register: Presence should be reported to service-location :type register: bool :param allow_none: Allow None values in XML-RPC - :type allow_non: bool + :type allow_none: bool :param encoding: Encoding to use for XML-RPC """ diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 68282331e..69c3264f9 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -2,6 +2,7 @@ used by both client and server. Stuff that doesn't fit anywhere else. """ +import shlex import fcntl import logging import threading @@ -217,6 +218,7 @@ class Executor(object): """ if isinstance(command, str): cmdstr = command + command = shlex.split(cmdstr) else: cmdstr = " ".join(command) self.logger.debug("Running: %s" % cmdstr) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 101530cac..4654359f7 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -626,30 +626,34 @@ Bcfg2 client itself.""") self.fam.debug = True def do_packageresolve(self, args): - """ packageresolve <hostname> <package> [<package>...] - - Resolve the specified set of packages """ + """ packageresolve <hostname> [<package> [<package>...]] - + Resolve packages for the given host, optionally specifying a + set of packages """ arglist = args.split(" ") - if len(arglist) < 2: + if len(arglist) < 1: print(self._get_usage(self.do_packageresolve)) return - if 'Packages' not in self.plugins: + try: + pkgs = self.plugins['Packages'] + except KeyError: print("Packages plugin not enabled") return - self.plugins['Packages'].toggle_debug() - - indep = lxml.etree.Element("Independent") - structures = [lxml.etree.Element("Bundle", name="packages")] - for arg in arglist[1:]: - lxml.etree.SubElement(structures[0], "Package", name=arg) + pkgs.toggle_debug() hostname = arglist[0] metadata = self.build_metadata(hostname) - # pylint: disable=W0212 - self.plugins['Packages']._build_packages(metadata, indep, structures) - # pylint: enable=W0212 + indep = lxml.etree.Element("Independent") + if len(arglist) > 1: + structures = [lxml.etree.Element("Bundle", name="packages")] + for arg in arglist[1:]: + lxml.etree.SubElement(structures[0], "Package", name=arg) + else: + structures = self.GetStructures(metadata) + pkgs._build_packages(metadata, indep, # pylint: disable=W0212 + structures) print("%d new packages added" % len(indep.getchildren())) if len(indep.getchildren()): print(" %s" % "\n ".join(lxml.etree.tostring(p) diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 4f81df89c..2ae5e02d5 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -58,16 +58,17 @@ def get_errorhandler(): """ get a Bcfg2.Server.Lint.ErrorHandler object """ setup = Bcfg2.Options.get_option_parser() if setup.cfp.has_section("errors"): - conf = dict(setup.cfp.items("errors")) + errors = dict(setup.cfp.items("errors")) else: - conf = None - return Bcfg2.Server.Lint.ErrorHandler(config=conf) + errors = None + return Bcfg2.Server.Lint.ErrorHandler(errors=errors) def load_server(): """ load server """ core = Bcfg2.Server.Core.BaseCore() - core.fam.handle_events_in_interval(4) + core.load_plugins() + core.fam.handle_events_in_interval(0.1) return core @@ -93,7 +94,7 @@ def load_plugins(): elif setup['lint_plugins']: plugin_list = setup['lint_plugins'] else: - plugin_list = Bcfg2.Server.Lint.__all__ + plugin_list = Bcfg2.Server.Lint.plugins allplugins = dict() for plugin in plugin_list: diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 510bb898b..f13240879 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -155,6 +155,7 @@ class ClientTest(TestCase): def get_core(setup): """ Get a server core, with events handled """ core = Bcfg2.Server.Core.BaseCore(setup) + core.load_plugins() core.fam.handle_events_in_interval(0.1) return core diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py index 211c39732..6d4644ea5 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py @@ -226,8 +226,7 @@ class TestPOSIXUsers(TestTool): users.user_supplementary_groups.assert_called_with(entry) reset() - m1 = lxml.etree.SubElement(entry, "MemberOf") - m1.text = "wheel" + m1 = lxml.etree.SubElement(entry, "MemberOf", group="wheel") m2 = lxml.etree.SubElement(entry, "MemberOf") m2.text = "users" self.assertTrue(users.VerifyPOSIXUser(entry, [])) @@ -236,8 +235,7 @@ class TestPOSIXUsers(TestTool): users.user_supplementary_groups.assert_called_with(entry) reset() - m3 = lxml.etree.SubElement(entry, "MemberOf") - m3.text = "extra" + m3 = lxml.etree.SubElement(entry, "MemberOf", group="extra") self.assertFalse(users.VerifyPOSIXUser(entry, [])) users.populate_user_entry.assert_called_with(entry) users._verify.assert_called_with(users.populate_user_entry.return_value) @@ -371,8 +369,7 @@ class TestPOSIXUsers(TestTool): entry = lxml.etree.Element("POSIXUser", name="test", group="test", home="/home/test", shell="/bin/zsh", gecos="Test McTest") - m1 = lxml.etree.SubElement(entry, "MemberOf") - m1.text = "wheel" + m1 = lxml.etree.SubElement(entry, "MemberOf", group="wheel") m2 = lxml.etree.SubElement(entry, "MemberOf") m2.text = "users" diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py index 3cd3786f9..93314f1e3 100755 --- a/tools/bcfg2-profile-templates.py +++ b/tools/bcfg2-profile-templates.py @@ -1,25 +1,35 @@ #!/usr/bin/python -Ott +# -*- coding: utf-8 -*- """ Benchmark template rendering times """ -import os import sys import time +import math import logging import operator import Bcfg2.Logger +import Bcfg2.Options import Bcfg2.Server.Core -LOGGER = None + +def stdev(nums): + mean = float(sum(nums)) / len(nums) + return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums))) def main(): - optinfo = \ - dict(client=Bcfg2.Options.Option("Benchmark templates for one client", - cmd="--client", - odesc="<client>", - long_arg=True, - default=None), - ) + optinfo = dict( + client=Bcfg2.Options.Option("Benchmark templates for one client", + cmd="--client", + odesc="<client>", + long_arg=True, + default=None), + runs=Bcfg2.Options.Option("Number of rendering passes per template", + cmd="--runs", + odesc="<runs>", + long_arg=True, + default=5, + cook=int)) optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) @@ -40,12 +50,11 @@ def main(): core = Bcfg2.Server.Core.BaseCore(setup) logger.info("Bcfg2 server core loaded") + core.load_plugins() + logger.debug("Plugins loaded") core.fam.handle_events_in_interval(0.1) logger.debug("Repository events processed") - # how many times to render each template for each client - runs = 5 - if setup['args']: templates = setup['args'] else: @@ -57,41 +66,57 @@ def main(): clients = [core.build_metadata(setup['client'])] times = dict() + client_count = 0 for metadata in clients: - for struct in core.GetStructures(metadata): - logger.info("Rendering templates from structure %s:%s" % - (struct.tag, struct.get("name"))) - for entry in struct.xpath("//Path"): - path = entry.get("name") - logger.info("Rendering %s..." % path) - times[path] = dict() - avg = 0.0 - for i in range(runs): + client_count += 1 + logger.info("Rendering templates for client %s (%s/%s)" % + (metadata.hostname, client_count, len(clients))) + structs = core.GetStructures(metadata) + struct_count = 0 + for struct in structs: + struct_count += 1 + logger.info("Rendering templates from structure %s:%s (%s/%s)" % + (struct.tag, struct.get("name"), struct_count, + len(structs))) + entries = struct.xpath("//Path") + entry_count = 0 + for entry in entries: + entry_count += 1 + if templates and entry.get("name") not in templates: + continue + logger.info("Rendering Path:%s (%s/%s)..." % + (entry.get("name"), entry_count, len(entries))) + ptimes = times.setdefault(entry.get("name"), []) + for i in range(setup['runs']): start = time.time() try: core.Bind(entry, metadata) - avg += (time.time() - start) / runs + ptimes.append(time.time() - start) except: break - if avg: - logger.debug(" %s: %.02f sec" % (metadata.hostname, avg)) - times[path][metadata.hostname] = avg + if ptimes: + avg = sum(ptimes) / len(ptimes) + if avg: + logger.debug(" %s: %.02f sec" % + (metadata.hostname, avg)) # print out per-file results tmpltimes = [] - for tmpl, clients in times.items(): + for tmpl, ptimes in times.items(): try: - avg = sum(clients.values()) / len(clients) + mean = float(sum(ptimes)) / len(ptimes) except ZeroDivisionError: continue - if avg > 0.01 or templates: - tmpltimes.append((tmpl, avg)) - print("%-50s %s" % ("Template", "Average Render Time")) - for tmpl, avg in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): - print("%-50s %.02f" % (tmpl, avg)) - - # TODO: complain about templates that on average were quick but - # for which some clients were slow + ptimes.sort() + median = ptimes[len(ptimes) / 2] + std = stdev(ptimes) + if mean > 0.01 or median > 0.01 or std > 1 or templates: + tmpltimes.append((tmpl, mean, median, std)) + print("%-50s %-9s %-11s %6s" % + ("Template", "Mean Time", "Median Time", "σ")) + for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): + print("%-50s %9.02f %11.02f %6.02f" % info) + core.shutdown() if __name__ == "__main__": diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index edb5a7101..8c164e52e 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -19,7 +19,8 @@ class LocalCore(BaseCore): setup['logging'] = None Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) setup['syslog'], setup['logging'] = saved - self.fam.handle_events_in_interval(4) + self.load_plugins() + self.fam.handle_events_in_interval(0.1) def _daemonize(self): return True diff --git a/tools/export.py b/tools/export.py index 716c831d9..0f4724e6b 100755 --- a/tools/export.py +++ b/tools/export.py @@ -227,9 +227,6 @@ E.G. 1.2.0pre1 is a valid version. 'VERSION="%s"\n' % version, startswith=True, dryrun=options.dryrun) - # set new version in setup.py - find_and_replace('setup.py', 'version=', ' version="%s",\n' % version, - dryrun=options.dryrun) # set new version in Bcfg2/version.py find_and_replace('src/lib/Bcfg2/version.py', '__version__ =', diff --git a/tools/posixusers_baseline.py b/tools/posixusers_baseline.py index a4abca42d..c45e54f1a 100755 --- a/tools/posixusers_baseline.py +++ b/tools/posixusers_baseline.py @@ -61,8 +61,8 @@ def main(): if entry.tag == 'POSIXUser': entry.set("group", grp.getgrgid(data[3])[0]) for group in users.user_supplementary_groups(entry): - memberof = lxml.etree.SubElement(entry, "MemberOf") - memberof.text = group[0] + memberof = lxml.etree.SubElement(entry, "MemberOf", + group=group[0]) entry.tag = "Bound" + entry.tag baseline.append(entry) diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py index 06f6e6b98..507a16f43 100755 --- a/tools/selinux_baseline.py +++ b/tools/selinux_baseline.py @@ -42,7 +42,10 @@ def main(): baseline.append(lxml.etree.Comment("%s entries" % etype)) extra = handler.FindExtra() for entry in extra: - entry.tag = "Bound%s" % etype + if etype != "SEModule": + entry.tag = "Bound%s" % etype + else: + entry.tag = "%s" % etype baseline.extend(extra) print(lxml.etree.tostring(baseline, pretty_print=True)) diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py index cbd2a6099..07def2ac8 100755 --- a/tools/upgrade/1.3/migrate_dbstats.py +++ b/tools/upgrade/1.3/migrate_dbstats.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) _our_backend = None + def _quote(value): """ Quote a string to use as a table name or column @@ -44,12 +45,12 @@ def _migrate_perms(): fperms = {} logger.info("Creating FilePerms objects") - for data in ( ('owner', 'group', 'perms'), + for data in (('owner', 'group', 'perms'), ('current_owner', 'current_group', 'current_perms')): for grp in legacy_models.Reason.objects.values_list(*data).distinct(): if grp in fperms: continue - fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2]) + fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2]) fp.save() fperms[grp] = fp @@ -60,7 +61,7 @@ def _migrate_perms(): def _migrate_transaction(inter, entries, fperms): """helper""" - logger.debug("Migrating interaction %s for %s" % + logger.debug("Migrating interaction %s for %s" % (inter.id, inter.client.name)) newint = new_models.Interaction(id=inter.id, @@ -107,7 +108,7 @@ def _migrate_transaction(inter, entries, fperms): elif ent.kind == 'Package': act_dict['target_version'] = ei.reason.version act_dict['current_version'] = ei.reason.current_version - logger.debug("Adding package %s %s" % + logger.debug("Adding package %s %s" % (name, act_dict['target_version'])) updates['packages'].append(new_models.PackageEntry.entry_get_or_create(act_dict)) elif ent.kind == 'Path': @@ -116,7 +117,7 @@ def _migrate_transaction(inter, entries, fperms): act_dict['target_perms'] = fperms[( ei.reason.owner, - ei.reason.group, + ei.reason.group, ei.reason.perms )] @@ -141,7 +142,6 @@ def _migrate_transaction(inter, entries, fperms): act_dict['detail_type'] = new_models.PathEntry.DETAIL_PRUNED act_dict['details'] = ei.reason.unpruned - if ei.reason.is_sensitive: act_dict['detail_type'] = new_models.PathEntry.DETAIL_SENSITIVE elif ei.reason.is_binary: @@ -164,7 +164,7 @@ def _migrate_transaction(inter, entries, fperms): for entry_type in updates.keys(): i = 0 while(i < len(updates[entry_type])): - getattr(newint, entry_type).add(*updates[entry_type][i:i+100]) + getattr(newint, entry_type).add(*updates[entry_type][i:i + 100]) i += 100 for perf in inter.performance_items.all(): @@ -220,8 +220,8 @@ def _restructure(): # run any migrations from the previous schema try: - from Bcfg2.Server.Reports.updatefix import update_database - update_database() + from Bcfg2.Server.Reports.updatefix import update_database + update_database() except: logger.error("Failed to run legacy schema updates", exc_info=1) return False @@ -295,4 +295,3 @@ if __name__ == '__main__': Reports(setup).__call__(['update']) _restructure() - diff --git a/tools/yum-listpkgs-xml.py b/tools/yum-listpkgs-xml.py index a052e75af..b4c5f6589 100755 --- a/tools/yum-listpkgs-xml.py +++ b/tools/yum-listpkgs-xml.py @@ -39,5 +39,5 @@ try: sys.argv = [sys.argv[0], '-d', '0', 'list'] yummain.main(sys.argv[1:]) except KeyboardInterrupt: - print("\n\nExiting on user cancel.", file=sys.stderr) + sys.stderr.write("\n\nExiting on user cancel.") sys.exit(1) |