diff options
169 files changed, 8510 insertions, 8434 deletions
diff --git a/doc/development/core.txt b/doc/development/core.txt index 3953d3402..ecbcbebd3 100644 --- a/doc/development/core.txt +++ b/doc/development/core.txt @@ -10,8 +10,10 @@ Bcfg2 1.3 added a pluggable server core system so that the server core itself can be easily swapped out to use different technologies. It -currently ships with two backends: a builtin core written from scratch -using the various server tools in the Python standard library; and an +currently ships with several backends: a builtin core written from +scratch using the various server tools in the Python standard library; +a variant on the builtin core that uses Python 2.6's +:mod:`multiprocessing` library to process requests in parallel; and an experimental `CherryPy <http://www.cherrypy.org/>`_ based core. This page documents the server core interface so that other cores can be written to take advantage of other technologies, e.g., `Tornado @@ -20,20 +22,25 @@ written to take advantage of other technologies, e.g., `Tornado A core implementation needs to: -* Override :func:`Bcfg2.Server.Core.BaseCore._daemonize` to handle - daemonization, writing the PID file, and dropping privileges. -* Override :func:`Bcfg2.Server.Core.BaseCore._run` to handle server +* Override :func:`Bcfg2.Server.Core.Core._run` to handle server startup. -* Override :func:`Bcfg2.Server.Core.BaseCore._block` to run the +* Override :func:`Bcfg2.Server.Core.Core._block` to run the blocking server loop. -* Call :func:`Bcfg2.Server.Core.BaseCore.shutdown` on orderly +* Call :func:`Bcfg2.Server.Core.Core.shutdown` on orderly shutdown. +A core that wants to use the network (i.e., a core that isn't used +entirely for introspection, as in :ref:`bcfg2-info +<server-bcfg2-info>`, or other local tasks) should inherit from +:class:`Bcfg2.Server.Core.NetworkCore`, and must also override +:func:`Bcfg2.Server.Core.NetworkCore._daemonize` to handle daemonization, +writing the PID file, and dropping privileges. + Nearly all XML-RPC handling is delegated entirely to the core implementation. It needs to: -* Call :func:`Bcfg2.Server.Core.BaseCore.authenticate` to authenticate - clients. +* Call :func:`Bcfg2.Server.Core.NetworkCore.authenticate` to + authenticate clients. * Handle :exc:`xmlrpclib.Fault` exceptions raised by the exposed XML-RPC methods as appropriate. * Dispatch XML-RPC method invocations to the appropriate method, diff --git a/doc/development/lint.txt b/doc/development/lint.txt index 6a4651f92..685823ab1 100644 --- a/doc/development/lint.txt +++ b/doc/development/lint.txt @@ -10,14 +10,14 @@ lets you easily write your own plugins to verify various parts of your Bcfg2 specification. -Plugins are loaded in one of two ways: +Plugins are included in a module of the same name as the plugin class +in :mod:`Bcfg2.Server.Lint`, e.g., :mod:`Bcfg2.Server.Lint.Validate`. -* They may be included in a module of the same name as the plugin - class in :mod:`Bcfg2.Server.Lint`, e.g., - :mod:`Bcfg2.Server.Lint.Validate`. -* They may be included directly in a Bcfg2 server plugin, called - "<plugin>Lint", e.g., - :class:`Bcfg2.Server.Plugins.Metadata.MetadataLint`. +.. note:: + + It is no longer possible to include lint plugins directly in a + Bcfg2 server plugin, e.g., + :class:`Bcfg2.Server.Plugins.Metadata.MetadataLint`. Plugin Types ============ @@ -106,10 +106,10 @@ Basics Existing ``bcfg2-lint`` Plugins =============================== -BundlerLint ------------ +Bundler +------- -.. autoclass:: Bcfg2.Server.Plugins.Bundler.BundlerLint +.. automodule:: Bcfg2.Server.Lint.Bundler Comments -------- @@ -126,10 +126,10 @@ GroupNames .. automodule:: Bcfg2.Server.Lint.GroupNames -GroupPatternsLint ------------------ +GroupPatterns +------------- -.. autoclass:: Bcfg2.Server.Plugins.GroupPatterns.GroupPatternsLint +.. automodule:: Bcfg2.Server.Lint.GroupPatterns InfoXML ------- @@ -141,25 +141,25 @@ MergeFiles .. automodule:: Bcfg2.Server.Lint.MergeFiles -MetadataLint ------------- +Metadata +-------- -.. autoclass:: Bcfg2.Server.Plugins.Metadata.MetadataLint +.. automodule:: Bcfg2.Server.Lint.Metadata -PkgmgrLint ----------- +Pkgmgr +------ -.. autoclass:: Bcfg2.Server.Plugins.Pkgmgr.PkgmgrLint +.. automodule:: Bcfg2.Server.Lint.Pkgmgr RequiredAttrs ------------- .. automodule:: Bcfg2.Server.Lint.RequiredAttrs -TemplateHelperLint ------------------- +TemplateHelper +-------------- -.. autoclass:: Bcfg2.Server.Plugins.TemplateHelper.TemplateHelperLint +.. automodule:: Bcfg2.Server.Lint.TemplateHelper Validate -------- diff --git a/doc/development/option_parsing.txt b/doc/development/option_parsing.txt new file mode 100644 index 000000000..52da8fced --- /dev/null +++ b/doc/development/option_parsing.txt @@ -0,0 +1,236 @@ +.. -*- mode: rst -*- + +.. _development-option-parsing: + +==================== +Bcfg2 Option Parsing +==================== + +Bcfg2 uses an option parsing mechanism based on the Python +:mod:`argparse` module. It does several very useful things that +``argparse`` does not: + +* Collects options from various places, which lets us easily specify + per-plugin options, for example; +* Automatically loads components (such as plugins); +* Synthesizes option values from the command line, config files, and + environment variables; +* Can dynamically create commands with many subcommands (e.g., + bcfg2-info and bcfg2-admin); and +* Supports keeping documentation inline with the option declaration, + which will make it easier to generate man pages. + + +Collecting Options +================== + +One of the more important features of the option parser is its ability +to automatically collect options from loaded components (e.g., Bcfg2 +server plugins). Given the highly pluggable architecture of Bcfg2, +this helps ensure two things: + +#. We do not have to specify all options in all places, or even in + most places. Options are specified alongside the class(es) that use + them. +#. All options needed for a given script to run are guaranteed to be + loaded, without the need to specify all components that script uses + manually. + +For instance, assume a few plugins: + +* The ``Foo`` plugin takes one option, ``--foo`` +* The ``Bar`` plugin takes two options, ``--bar`` and ``--force`` + +The plugins are used by the ``bcfg2-quux`` command, which itself takes +two options: ``--plugins`` (which selects the plugins) and +``--test``. The options would be selected at runtime, so for instance +these would be valid: + +.. code-block:: bash + + bcfg2-quux --plugins Foo --foo --test + bcfg2-quux --plugins Foo,Bar --foo --bar --force + bcfg2-quux --plugins Bar --force + +But this would not: + + bcfg2-quux --plugins Foo --bar + +The help message would reflect the options that are available to the +default set of plugins. (For this reason, allowing component lists to +be set in the config file is very useful; that way, usage messages +reflect the components in the config file.) + +Components (in this example, the plugins) can be classes or modules. +There is no required interface for an option component. They may +*optionally* have: + +* An ``options`` attribute that is a list of + :class:`Bcfg2.Options.Options.Option` objects or option groups. +* A function or static method, ``options_parsed_hook``, that is called + when all options have been parsed. (This will be called again if + :func:`Bcfg2.Options.Parser.Parser.reparse` is called.) + +Options are collected through two primary mechanisms: + +#. The :class:`Bcfg2.Options.Actions.ComponentAction` class. When a + ComponentAction subclass is used as the action of an option, then + options contained in the classes (or modules) given in the option + value will be added to the parser. +#. Modules that are not loaded via a + :class:`Bcfg2.Options.Actions.ComponentAction` option may load + options at runtime. + +Since it is preferred to add components instead of just options, +loading options at runtime is generally best accomplished by creating +a container object whose only purpose is to hold options. For +instance: + +.. code-block:: python + + def foo(): + # do stuff + + class _OptionContainer(object): + options = [ + Bcfg2.Options.BooleanOption("--foo", help="Enable foo")] + + @staticmethod + def options_parsed_hook(): + if Bcfg2.Options.setup.foo: + foo() + + Bcfg2.Options.get_parser().add_component(_OptionContainer) + +The Bcfg2.Options module +======================== + +.. currentmodule:: Bcfg2.Options + +.. autodata:: setup + +Options +------- + +The base :class:`Bcfg2.Options.Option` object represents an option. +Unlike options in :mod:`argparse`, an Option object does not need to +be associated with an option parser; it exists on its own. + +.. autoclass:: Option +.. autoclass:: PathOption +.. autoclass:: BooleanOption +.. autoclass:: PositionalArgument + +The Parser +---------- + +.. autoclass:: Parser +.. autofunction:: get_parser +.. autoexception:: OptionParserException + +Option Groups +------------- + +Options can be grouped in various meaningful ways. This uses a +variety of :mod:`argparse` functionality behind the scenes. + +In all cases, options can be added to groups in-line by simply +specifying them in the object group constructor: + +.. code-block:: python + + options = [ + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.Option(...), + Bcfg2.Options.Option(...), + required=True), + ....] + +Nesting object groups is supported in theory, but barely tested. + +.. autoclass:: OptionGroup +.. autoclass:: ExclusiveOptionGroup +.. autoclass:: Subparser +.. autoclass:: WildcardSectionGroup + +Subcommands +----------- + +This library makes it easier to work with programs that have a large +number of subcommands (e.g., :ref:`bcfg2-info <server-bcfg2-info>` and +:ref:`bcfg2-admin <server-admin-index>`). + +The normal implementation pattern is this: + +#. Define all of your subcommands as children of + :class:`Bcfg2.Options.Subcommand`. +#. Define a :class:`Bcfg2.Options.CommandRegistry` object that will be + used to register all of the commands. Registering a command + collect its options and adds it as a + :class:`Bcfg2.Options.Subparser` option group to the main option + parser. +#. Register your commands with + :func:`Bcfg2.Options.register_commands`, parse options, and run. + +:mod:`Bcfg2.Server.Admin` provides a fairly simple implementation, +where the CLI class is itself the command registry: + +.. code-block:: python + + class CLI(Bcfg2.Options.CommandRegistry): + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, + globals().values(), + parent=AdminCmd) + parser = Bcfg2.Options.get_parser( + description="Manage a running Bcfg2 server", + components=[self]) + parser.parse() + +In this case, commands are collected from amongst all global variables +(the most likely scenario), and they must be children of +:class:`Bcfg2.Server.Admin.AdminCmd`, which itself subclasses +:class:`Bcfg2.Options.Subcommand`. + +Commands are defined by subclassing :class:`Bcfg2.Options.Subcommand`. +At a minimum, the :func:`Bcfg2.Options.Subcommand.run` method must be +overridden, and a docstring written. + +.. autoclass:: Subcommand +.. autoclass:: HelpCommand +.. autoclass:: CommandRegistry +.. autofunction:: register_commands + +Actions +------- + +Several custom argparse `actions +<http://docs.python.org/dev/library/argparse.html#action>`_ provide +some of the option collection magic of :mod:`Bcfg2.Options`. + +.. autoclass:: ConfigFileAction +.. autoclass:: ComponentAction +.. autoclass:: PluginsAction + +Option Types +------------ + +:mod:`Bcfg2.Options` provides a number of useful types for use as the `type +<http://docs.python.org/dev/library/argparse.html#type>`_ keyword +argument to +the :class:`Bcfg2.Options.Option` constructor. + +.. autofunction:: Bcfg2.Options.Types.path +.. autofunction:: Bcfg2.Options.Types.comma_list +.. autofunction:: Bcfg2.Options.Types.colon_list +.. autofunction:: Bcfg2.Options.Types.octal +.. autofunction:: Bcfg2.Options.Types.username +.. autofunction:: Bcfg2.Options.Types.groupname +.. autofunction:: Bcfg2.Options.Types.timeout +.. autofunction:: Bcfg2.Options.Types.size + +Common Options +-------------- + +.. autoclass:: Common diff --git a/doc/server/admin/bundle.txt b/doc/server/admin/bundle.txt deleted file mode 100644 index e9cb79781..000000000 --- a/doc/server/admin/bundle.txt +++ /dev/null @@ -1,34 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-admin-bundle: - -bundle -====== - -For a list of all available xml bundles use ``list-xml``. ``list-genshi`` -will list all available genshi bundles.:: - -.. code-block:: sh - - # bcfg2-admin bundles list-xml - # bcfg2-admin bundles list-genshi - -``show`` provides an interactive dialog to get details about the available -bundles.:: - -.. code-block:: sh - - # bcfg2-admin bundles show - Available bundles (Number of bundles: 4) - ---------------------------------------- - [0] motd.xml - [1] snmpd.xml - [2] bcfg2.xml - [3] ntp.xml - Enter the line number of a bundle for details: 3 - Details for the "ntp" bundle: - Package: xntp - Path: /etc/sysconfig/xntp - Path: /etc/sysconfig/clock - Path: /etc/ntp.conf - Service: xntpd diff --git a/doc/server/admin/compare.txt b/doc/server/admin/compare.txt index 6a770055e..ffe19efdf 100644 --- a/doc/server/admin/compare.txt +++ b/doc/server/admin/compare.txt @@ -6,11 +6,10 @@ compare ======= Determine differences between files or directories of client -specification instances.:: +specification instances:: bcfg2-admin compare <file1> <file2> -If you want to compare two directories recursively then use ``-r`` as an -option. :: +Or:: - bcfg2-admin compare -r <dir1> <dir2> + bcfg2-admin compare <dir1> <dir2> diff --git a/doc/server/admin/index.txt b/doc/server/admin/index.txt index 8ea765aac..707f7c724 100644 --- a/doc/server/admin/index.txt +++ b/doc/server/admin/index.txt @@ -16,14 +16,11 @@ functionality. Available modes are listed below. :maxdepth: 1 backup - bundle client compare init minestruct perf pull - query - tidy viz xcmd diff --git a/doc/server/admin/query.txt b/doc/server/admin/query.txt deleted file mode 100644 index 65851a43d..000000000 --- a/doc/server/admin/query.txt +++ /dev/null @@ -1,15 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-admin-query: - -query -===== - -Query clients. - -The default result format is suitable for consumption by `pdsh`_. -This example queries the server for all clients in the *ubuntu* group:: - - bcfg2-admin query g=ubuntu - -.. _pdsh: http://sourceforge.net/projects/pdsh/ diff --git a/doc/server/admin/tidy.txt b/doc/server/admin/tidy.txt deleted file mode 100644 index 816d6cdb3..000000000 --- a/doc/server/admin/tidy.txt +++ /dev/null @@ -1,8 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-admin-tidy: - -tidy -==== - -Clean up useless files in the repo. diff --git a/schemas/acl-metadata.xsd b/schemas/acl-metadata.xsd index 68994c940..643dfec7f 100644 --- a/schemas/acl-metadata.xsd +++ b/schemas/acl-metadata.xsd @@ -62,6 +62,14 @@ </xsd:documentation> </xsd:annotation> <xsd:group ref="MetadataACLElements" minOccurs="1" maxOccurs="unbounded"/> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> <xsd:group name="MetadataACLElements"> diff --git a/schemas/authorizedkeys.xsd b/schemas/authorizedkeys.xsd index fd8f2a7a3..20e568a07 100644 --- a/schemas/authorizedkeys.xsd +++ b/schemas/authorizedkeys.xsd @@ -111,6 +111,14 @@ <xsd:element name="Client" type="AuthorizedKeysGroupType"/> <xsd:element name="AuthorizedKeys" type="AuthorizedKeysType"/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/bundle.xsd b/schemas/bundle.xsd index d5abf2d94..aeacd0517 100644 --- a/schemas/bundle.xsd +++ b/schemas/bundle.xsd @@ -351,6 +351,14 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> <xsd:attribute ref="xml:base"/> </xsd:complexType> diff --git a/schemas/decisions.xsd b/schemas/decisions.xsd index c87d2a984..9df4b1215 100644 --- a/schemas/decisions.xsd +++ b/schemas/decisions.xsd @@ -64,6 +64,14 @@ <xsd:element name="Client" type="DecisionsGroupType"/> <xsd:group ref="py:genshiElements"/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/defaults.xsd b/schemas/defaults.xsd index 17ae84366..f810a6269 100644 --- a/schemas/defaults.xsd +++ b/schemas/defaults.xsd @@ -35,6 +35,14 @@ <xsd:element name="Client" type="DContainerType"/> </xsd:choice> <xsd:attribute name="priority" type="xsd:integer" use="required"/> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> </xsd:element> </xsd:schema> diff --git a/schemas/fileprobes.xsd b/schemas/fileprobes.xsd index 12f60378c..64f01bf8e 100644 --- a/schemas/fileprobes.xsd +++ b/schemas/fileprobes.xsd @@ -37,6 +37,14 @@ <xsd:element name="Group" type="FileProbesGroupType"/> <xsd:element name="Client" type="FileProbesGroupType"/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> </xsd:element> </xsd:schema> diff --git a/schemas/info.xsd b/schemas/info.xsd index 9b898a168..5291562c1 100644 --- a/schemas/info.xsd +++ b/schemas/info.xsd @@ -135,6 +135,14 @@ <xsd:element name='Path' type='InfoGroupType'/> <xsd:element name='Info' type='InfoType'/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> <xsd:element name='FileInfo' type="FileInfoType"/> diff --git a/schemas/nagiosgen.xsd b/schemas/nagiosgen.xsd index b3ccf5095..24c298885 100644 --- a/schemas/nagiosgen.xsd +++ b/schemas/nagiosgen.xsd @@ -35,6 +35,14 @@ <xsd:element name="Group" type="NagiosGenGroupType"/> <xsd:element name="Client" type="NagiosGenGroupType"/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> </xsd:element> </xsd:schema> diff --git a/schemas/packages.xsd b/schemas/packages.xsd index e01093c56..e57280527 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -240,6 +240,14 @@ <xsd:element name="Source" type="SourceType"/> <xsd:element name="Sources" type="SourcesType"/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attribute ref="xml:base"/> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/pkgtype.xsd b/schemas/pkgtype.xsd index 18eda88ab..c76c52824 100644 --- a/schemas/pkgtype.xsd +++ b/schemas/pkgtype.xsd @@ -146,38 +146,33 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute name="installed_action" type="xsd:string" - default="install"> + <xsd:attribute name="install_missing" type="xsd:boolean" + default="true"> <xsd:annotation> <xsd:documentation> - If this is set to any value other than "install", - package installation will be suppressed with the - :ref:`YUM24 and RPM <client-tools-yum>` drivers. + Whether or not to install missing packages. This is + only honored by the the :ref:`RPM + <client-tools-yum>` driver. </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute name="version_fail_action" type="xsd:string" - default="upgrade"> + <xsd:attribute name="fix_version" type="xsd:boolean" default="true"> <xsd:annotation> <xsd:documentation> - If this is set to any value other than "upgrade", a - package that has the incorrect version installed will - not be fixed with the :ref:`YUM24 and RPM - <client-tools-yum>` drivers. Note that - "upgrade" is misleading; if a package is installed - that is newer than the desired version, it will not be - downgraded if this attribute is set to anything other - than "upgrade". + Whether or not to upgrade or downgrade packages that + are installed, but have the wrong version. This is + only honored by the :ref:`RPM + <client-tools-yum>` driver. </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute name="verify_fail_action" type="xsd:string"> + <xsd:attribute name="reinstall_broken" type="xsd:boolean" + default="true"> <xsd:annotation> <xsd:documentation> - If this is set to any value other than "reinstall", a - package that fails package verification will not be - reinstalled with the :ref:`YUM24 and RPM - <client-tools-yum>` drivers. + Whether or not to reinstall packages that fail + verification. This is only honored by the :ref:`RPM + <client-tools-yum>` driver. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/schemas/privkey.xsd b/schemas/privkey.xsd index 0bb1b7184..30bc8a1b3 100644 --- a/schemas/privkey.xsd +++ b/schemas/privkey.xsd @@ -144,10 +144,10 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute name="decrypt" type="EncryptStrictnessEnum"> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> <xsd:annotation> <xsd:documentation> - Override the global strict/lax decryption setting in + Override the global lax_decryption setting in ``bcfg2.conf``. </xsd:documentation> </xsd:annotation> diff --git a/schemas/rules.xsd b/schemas/rules.xsd index be60abef0..fb41ad9d4 100644 --- a/schemas/rules.xsd +++ b/schemas/rules.xsd @@ -198,6 +198,14 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> </xsd:element> diff --git a/schemas/sslca-cert.xsd b/schemas/sslca-cert.xsd index 49d821aaf..a3f6db94d 100644 --- a/schemas/sslca-cert.xsd +++ b/schemas/sslca-cert.xsd @@ -171,6 +171,14 @@ <xsd:element name="subjectAltName" type="SubjectAltNameType"/> <xsd:element name="CertInfo" type="CertInfoType"/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> <xsd:element name="CertInfo" type="CertInfoType"/> diff --git a/schemas/sslca-key.xsd b/schemas/sslca-key.xsd index 2b6a02b98..261b71e1a 100644 --- a/schemas/sslca-key.xsd +++ b/schemas/sslca-key.xsd @@ -91,6 +91,14 @@ <xsd:element name="Client" type="SSLCAKeyGroupType"/> <xsd:element name="KeyInfo" type="KeyInfoType"/> </xsd:choice> + <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Override the global lax_decryption setting in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> <xsd:element name="KeyInfo" type="KeyInfoType"/> diff --git a/schemas/types.xsd b/schemas/types.xsd index dbb0c0390..5dec03cdb 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -103,13 +103,6 @@ </xsd:restriction> </xsd:simpleType> - <xsd:simpleType name="EncryptStrictnessEnum"> - <xsd:restriction base="xsd:string"> - <xsd:enumeration value="strict"/> - <xsd:enumeration value="lax"/> - </xsd:restriction> - </xsd:simpleType> - <xsd:complexType name='ActionType'> <xsd:annotation> <xsd:documentation> diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py deleted file mode 100644 index 994ce7c84..000000000 --- a/src/lib/Bcfg2/Client/Client.py +++ /dev/null @@ -1,337 +0,0 @@ -""" The main Bcfg2 client class """ - -import os -import sys -import stat -import time -import fcntl -import socket -import logging -import tempfile -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Client.XML -import Bcfg2.Client.Proxy -import Bcfg2.Client.Frame -import Bcfg2.Client.Tools -from Bcfg2.Utils import locked, Executor -from Bcfg2.Compat import xmlrpclib -from Bcfg2.version import __version__ - - -class Client(object): - """ The main Bcfg2 client class """ - - def __init__(self): - self.toolset = None - self.tools = None - self.config = None - self._proxy = None - self.setup = Bcfg2.Options.get_option_parser() - - if self.setup['debug']: - level = logging.DEBUG - elif self.setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2', - to_syslog=self.setup['syslog'], - level=level, - to_file=self.setup['logging']) - self.logger = logging.getLogger('bcfg2') - self.logger.debug(self.setup) - - self.cmd = Executor(self.setup['command_timeout']) - - if self.setup['bundle_quick']: - if not self.setup['bundle'] and not self.setup['skipbundle']: - self.logger.error("-Q option requires -b or -B") - raise SystemExit(1) - elif self.setup['remove']: - self.logger.error("-Q option incompatible with -r") - raise SystemExit(1) - if 'drivers' in self.setup and self.setup['drivers'] == 'help': - self.logger.info("The following drivers are available:") - self.logger.info(Bcfg2.Client.Tools.__all__) - raise SystemExit(0) - if self.setup['remove'] and 'services' in self.setup['remove'].lower(): - self.logger.error("Service removal is nonsensical; " - "removed services will only be disabled") - if (self.setup['remove'] and - self.setup['remove'].lower() not in ['all', 'services', 'packages', - 'users']): - self.logger.error("Got unknown argument %s for -r" % - self.setup['remove']) - if self.setup["file"] and self.setup["cache"]: - print("cannot use -f and -c together") - raise SystemExit(1) - if not self.setup['server'].startswith('https://'): - self.setup['server'] = 'https://' + self.setup['server'] - - def _probe_failure(self, probename, msg): - """ handle failure of a probe in the way the user wants us to - (exit or continue) """ - message = "Failed to execute probe %s: %s" % (probename, msg) - if self.setup['probe_exit']: - self.fatal_error(message) - else: - self.logger.error(message) - - def run_probe(self, probe): - """Execute probe.""" - name = probe.get('name') - self.logger.info("Running probe %s" % name) - ret = Bcfg2.Client.XML.Element("probe-data", - name=name, - source=probe.get('source')) - try: - scripthandle, scriptname = tempfile.mkstemp() - script = os.fdopen(scripthandle, 'w') - try: - script.write("#!%s\n" % - (probe.attrib.get('interpreter', '/bin/sh'))) - if sys.hexversion >= 0x03000000: - script.write(probe.text) - else: - script.write(probe.text.encode('utf-8')) - script.close() - os.chmod(scriptname, - stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | - stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | - stat.S_IWUSR) # 0755 - rv = self.cmd.run(scriptname, timeout=self.setup['timeout']) - if rv.stderr: - self.logger.warning("Probe %s has error output: %s" % - (name, rv.stderr)) - if not rv.success: - self._probe_failure(name, "Return value %s" % rv) - self.logger.info("Probe %s has result:" % name) - self.logger.info(rv.stdout) - if sys.hexversion >= 0x03000000: - ret.text = rv.stdout - else: - ret.text = rv.stdout.decode('utf-8') - finally: - os.unlink(scriptname) - except SystemExit: - raise - except: - self._probe_failure(name, sys.exc_info()[1]) - return ret - - def fatal_error(self, message): - """Signal a fatal error.""" - self.logger.error("Fatal error: %s" % (message)) - raise SystemExit(1) - - @property - def proxy(self): - """ get an XML-RPC proxy to the server """ - if self._proxy is None: - self._proxy = Bcfg2.Client.Proxy.ComponentProxy( - self.setup['server'], - self.setup['user'], - self.setup['password'], - key=self.setup['key'], - cert=self.setup['certificate'], - ca=self.setup['ca'], - allowedServerCNs=self.setup['serverCN'], - timeout=self.setup['timeout'], - retries=int(self.setup['retries']), - delay=int(self.setup['retry_delay'])) - return self._proxy - - def run_probes(self, times=None): - """ run probes and upload probe data """ - if times is None: - times = dict() - - try: - probes = Bcfg2.Client.XML.XML(str(self.proxy.GetProbes())) - except (Bcfg2.Client.Proxy.ProxyError, - Bcfg2.Client.Proxy.CertificateError, - socket.gaierror, - socket.error): - err = sys.exc_info()[1] - self.fatal_error("Failed to download probes from bcfg2: %s" % err) - except Bcfg2.Client.XML.ParseError: - err = sys.exc_info()[1] - self.fatal_error("Server returned invalid probe requests: %s" % - err) - - times['probe_download'] = time.time() - - # execute probes - probedata = Bcfg2.Client.XML.Element("ProbeData") - for probe in probes.findall(".//probe"): - probedata.append(self.run_probe(probe)) - - if len(probes.findall(".//probe")) > 0: - try: - # upload probe responses - self.proxy.RecvProbeData( - Bcfg2.Client.XML.tostring( - probedata, - xml_declaration=False).decode('utf-8')) - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to upload probe data: %s" % err) - - times['probe_upload'] = time.time() - - def get_config(self, times=None): - """ load the configuration, either from the cached - configuration file (-f), or from the server """ - if times is None: - times = dict() - - if self.setup['file']: - # read config from file - try: - self.logger.debug("Reading cached configuration from %s" % - self.setup['file']) - return open(self.setup['file'], 'r').read() - except IOError: - self.fatal_error("Failed to read cached configuration from: %s" - % (self.setup['file'])) - else: - # retrieve config from server - if self.setup['profile']: - try: - self.proxy.AssertProfile(self.setup['profile']) - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to set client profile: %s" % err) - - try: - self.proxy.DeclareVersion(__version__) - except xmlrpclib.Fault: - err = sys.exc_info()[1] - if (err.faultCode == xmlrpclib.METHOD_NOT_FOUND or - (err.faultCode == 7 and - err.faultString.startswith("Unknown method"))): - self.logger.debug("Server does not support declaring " - "client version") - else: - self.logger.error("Failed to declare version: %s" % err) - except (Bcfg2.Client.Proxy.ProxyError, - Bcfg2.Client.Proxy.CertificateError, - socket.gaierror, - socket.error): - err = sys.exc_info()[1] - self.logger.error("Failed to declare version: %s" % err) - - self.run_probes(times=times) - - if self.setup['decision'] in ['whitelist', 'blacklist']: - try: - self.setup['decision_list'] = \ - self.proxy.GetDecisionList(self.setup['decision']) - self.logger.info("Got decision list from server:") - self.logger.info(self.setup['decision_list']) - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to get decision list: %s" % err) - - try: - rawconfig = self.proxy.GetConfig().encode('utf-8') - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to download configuration from " - "Bcfg2: %s" % err) - - times['config_download'] = time.time() - return rawconfig - - def run(self): - """Perform client execution phase.""" - times = {} - - # begin configuration - times['start'] = time.time() - - self.logger.info("Starting Bcfg2 client run at %s" % times['start']) - - rawconfig = self.get_config(times=times).decode('utf-8') - - if self.setup['cache']: - try: - open(self.setup['cache'], 'w').write(rawconfig) - os.chmod(self.setup['cache'], 33152) - except IOError: - self.logger.warning("Failed to write config cache file %s" % - (self.setup['cache'])) - times['caching'] = time.time() - - try: - self.config = Bcfg2.Client.XML.XML(rawconfig) - except Bcfg2.Client.XML.ParseError: - syntax_error = sys.exc_info()[1] - self.fatal_error("The configuration could not be parsed: %s" % - syntax_error) - - times['config_parse'] = time.time() - - if self.config.tag == 'error': - self.fatal_error("Server error: %s" % (self.config.text)) - return(1) - - if self.setup['bundle_quick']: - newconfig = Bcfg2.Client.XML.XML('<Configuration/>') - for bundle in self.config.getchildren(): - if (bundle.tag == 'Bundle' and - ((self.setup['bundle'] and - bundle.get('name') in self.setup['bundle']) or - (self.setup['skipbundle'] and - bundle.get('name') not in self.setup['skipbundle']))): - newconfig.append(bundle) - self.config = newconfig - - self.tools = Bcfg2.Client.Frame.Frame(self.config, times) - - if not self.setup['omit_lock_check']: - #check lock here - try: - lockfile = open(self.setup['lockfile'], 'w') - if locked(lockfile.fileno()): - self.fatal_error("Another instance of Bcfg2 is running. " - "If you want to bypass the check, run " - "with the %s option" % - Bcfg2.Options.OMIT_LOCK_CHECK.cmd) - except SystemExit: - raise - except: - lockfile = None - self.logger.error("Failed to open lockfile %s: %s" % - (self.setup['lockfile'], sys.exc_info()[1])) - - # execute the configuration - self.tools.Execute() - - if not self.setup['omit_lock_check']: - # unlock here - if lockfile: - try: - fcntl.lockf(lockfile.fileno(), fcntl.LOCK_UN) - os.remove(self.setup['lockfile']) - except OSError: - self.logger.error("Failed to unlock lockfile %s" % - lockfile.name) - - if not self.setup['file'] and not self.setup['bundle_quick']: - # upload statistics - feedback = self.tools.GenerateStats() - - try: - self.proxy.RecvStats( - Bcfg2.Client.XML.tostring( - feedback, - xml_declaration=False).decode('utf-8')) - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - self.logger.error("Failed to upload configuration statistics: " - "%s" % err) - raise SystemExit(2) - - self.logger.info("Finished Bcfg2 client run at %s" % time.time()) diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py deleted file mode 100644 index 4fece79b8..000000000 --- a/src/lib/Bcfg2/Client/Frame.py +++ /dev/null @@ -1,561 +0,0 @@ -""" Frame is the Client Framework that verifies and installs entries, -and generates statistics. """ - -import copy -import time -import fnmatch -import logging -import Bcfg2.Client.Tools -from Bcfg2.Client import prompt -from Bcfg2.Options import get_option_parser -from Bcfg2.Compat import any, all, cmp # pylint: disable=W0622 - - -def cmpent(ent1, ent2): - """Sort entries.""" - if ent1.tag != ent2.tag: - return cmp(ent1.tag, ent2.tag) - else: - return cmp(ent1.get('name'), ent2.get('name')) - - -def matches_entry(entryspec, entry): - """ Determine if the Decisions-style entry specification matches - the entry. Both are tuples of (tag, name). The entryspec can - handle the wildcard * in either position. """ - if entryspec == entry: - return True - return all(fnmatch.fnmatch(entry[i], entryspec[i]) for i in [0, 1]) - - -def matches_white_list(entry, whitelist): - """ Return True if (<entry tag>, <entry name>) is in the given - whitelist. """ - return any(matches_entry(we, (entry.tag, entry.get('name'))) - for we in whitelist) - - -def passes_black_list(entry, blacklist): - """ Return True if (<entry tag>, <entry name>) is not in the given - blacklist. """ - return not any(matches_entry(be, (entry.tag, entry.get('name'))) - for be in blacklist) - - -# pylint: disable=W0702 -# in frame we frequently want to catch all exceptions, regardless of -# type, so disable the pylint rule that catches that. - - -class Frame(object): - """Frame is the container for all Tool objects and state information.""" - - def __init__(self, config, times): - self.setup = get_option_parser() - self.config = config - self.times = times - self.dryrun = self.setup['dryrun'] - self.times['initialization'] = time.time() - self.tools = [] - - #: A dict of the state of each entry. Keys are the entries. - #: Values are boolean: True means that the entry is good, - #: False means that the entry is bad. - self.states = {} - self.whitelist = [] - self.blacklist = [] - self.removal = [] - self.logger = logging.getLogger(__name__) - drivers = self.setup['drivers'] - for driver in drivers[:]: - if (driver not in Bcfg2.Client.Tools.__all__ and - isinstance(driver, str)): - self.logger.error("Tool driver %s is not available" % driver) - drivers.remove(driver) - - tclass = {} - for tool in drivers: - if not isinstance(tool, str): - tclass[time.time()] = tool - tool_class = "Bcfg2.Client.Tools.%s" % tool - try: - tclass[tool] = getattr(__import__(tool_class, globals(), - locals(), ['*']), - tool) - except ImportError: - continue - except: - self.logger.error("Tool %s unexpectedly failed to load" % tool, - exc_info=1) - - for tool in list(tclass.values()): - try: - self.tools.append(tool(config)) - except Bcfg2.Client.Tools.ToolInstantiationError: - continue - except: - self.logger.error("Failed to instantiate tool %s" % tool, - exc_info=1) - - for tool in self.tools[:]: - for conflict in getattr(tool, 'conflicts', []): - for item in self.tools: - if item.name == conflict: - self.tools.remove(item) - - self.logger.info("Loaded tool drivers:") - self.logger.info([tool.name for tool in self.tools]) - - deprecated = [tool.name for tool in self.tools if tool.deprecated] - if deprecated: - self.logger.warning("Loaded deprecated tool drivers:") - self.logger.warning(deprecated) - experimental = [tool.name for tool in self.tools if tool.experimental] - if experimental: - self.logger.info("Loaded experimental tool drivers:") - self.logger.info(experimental) - - # find entries not handled by any tools - self.unhandled = [entry for struct in config - for entry in struct - if entry not in self.handled] - - if self.unhandled: - self.logger.error("The following entries are not handled by any " - "tool:") - for entry in self.unhandled: - self.logger.error("%s:%s:%s" % (entry.tag, entry.get('type'), - entry.get('name'))) - - self.find_dups(config) - - pkgs = [(entry.get('name'), entry.get('origin')) - for struct in config - for entry in struct - if entry.tag == 'Package'] - if pkgs: - self.logger.debug("The following packages are specified in bcfg2:") - self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] is None]) - self.logger.debug("The following packages are prereqs added by " - "Packages:") - self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages']) - - def find_dups(self, config): - """ Find duplicate entries and warn about them """ - entries = dict() - for struct in config: - for entry in struct: - for tool in self.tools: - if tool.handlesEntry(entry): - pkey = tool.primarykey(entry) - if pkey in entries: - entries[pkey] += 1 - else: - entries[pkey] = 1 - multi = [e for e, c in entries.items() if c > 1] - if multi: - self.logger.debug("The following entries are included multiple " - "times:") - for entry in multi: - self.logger.debug(entry) - - def promptFilter(self, msg, entries): - """Filter a supplied list based on user input.""" - ret = [] - entries.sort(key=lambda e: e.tag + ":" + e.get('name')) - for entry in entries[:]: - if entry in self.unhandled: - # don't prompt for entries that can't be installed - continue - if 'qtext' in entry.attrib: - iprompt = entry.get('qtext') - else: - iprompt = msg % (entry.tag, entry.get('name')) - if prompt(iprompt): - ret.append(entry) - return ret - - def __getattr__(self, name): - if name in ['extra', 'handled', 'modified', '__important__']: - ret = [] - for tool in self.tools: - ret += getattr(tool, name) - return ret - elif name in self.__dict__: - return self.__dict__[name] - raise AttributeError(name) - - def InstallImportant(self): - """Install important entries - - We also process the decision mode stuff here because we want to prevent - non-whitelisted/blacklisted 'important' entries from being installed - prior to determining the decision mode on the client. - """ - # Need to process decision stuff early so that dryrun mode - # works with it - self.whitelist = [entry for entry in self.states - if not self.states[entry]] - if not self.setup['file']: - if self.setup['decision'] == 'whitelist': - dwl = self.setup['decision_list'] - w_to_rem = [e for e in self.whitelist - if not matches_white_list(e, dwl)] - if w_to_rem: - self.logger.info("In whitelist mode: " - "suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) - for e in w_to_rem]) - self.whitelist = [x for x in self.whitelist - if x not in w_to_rem] - elif self.setup['decision'] == 'blacklist': - b_to_rem = \ - [e for e in self.whitelist - if not passes_black_list(e, self.setup['decision_list'])] - if b_to_rem: - self.logger.info("In blacklist mode: " - "suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) - for e in b_to_rem]) - self.whitelist = [x for x in self.whitelist - if x not in b_to_rem] - - # take care of important entries first - if not self.dryrun: - parent_map = dict((c, p) - for p in self.config.getiterator() - for c in p) - for cfile in self.config.findall(".//Path"): - if (cfile.get('name') not in self.__important__ or - cfile.get('type') != 'file' or - cfile not in self.whitelist): - continue - parent = parent_map[cfile] - if ((parent.tag == "Bundle" and - ((self.setup['bundle'] and - parent.get("name") not in self.setup['bundle']) or - (self.setup['skipbundle'] and - parent.get("name") in self.setup['skipbundle']))) or - (parent.tag == "Independent" and - (self.setup['bundle'] or self.setup['skipindep']))): - continue - tools = [t for t in self.tools - if t.handlesEntry(cfile) and t.canVerify(cfile)] - if tools: - if (self.setup['interactive'] and not - self.promptFilter("Install %s: %s? (y/N):", [cfile])): - self.whitelist.remove(cfile) - continue - try: - self.states[cfile] = tools[0].InstallPath(cfile) - if self.states[cfile]: - tools[0].modified.append(cfile) - except: - self.logger.error("Unexpected tool failure", - exc_info=1) - cfile.set('qtext', '') - if tools[0].VerifyPath(cfile, []): - self.whitelist.remove(cfile) - - def Inventory(self): - """ - Verify all entries, - find extra entries, - and build up workqueues - - """ - # initialize all states - for struct in self.config.getchildren(): - for entry in struct.getchildren(): - self.states[entry] = False - for tool in self.tools: - try: - self.states.update(tool.Inventory()) - except: - self.logger.error("%s.Inventory() call failed:" % tool.name, - exc_info=1) - - def Decide(self): # pylint: disable=R0912 - """Set self.whitelist based on user interaction.""" - iprompt = "Install %s: %s? (y/N): " - rprompt = "Remove %s: %s? (y/N): " - if self.setup['remove']: - if self.setup['remove'] == 'all': - self.removal = self.extra - elif self.setup['remove'].lower() == 'services': - self.removal = [entry for entry in self.extra - if entry.tag == 'Service'] - elif self.setup['remove'].lower() == 'packages': - self.removal = [entry for entry in self.extra - if entry.tag == 'Package'] - elif self.setup['remove'].lower() == 'users': - self.removal = [entry for entry in self.extra - if entry.tag in ['POSIXUser', 'POSIXGroup']] - - candidates = [entry for entry in self.states - if not self.states[entry]] - - if self.dryrun: - if self.whitelist: - self.logger.info("In dryrun mode: " - "suppressing entry installation for:") - self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) - for entry in self.whitelist]) - self.whitelist = [] - if self.removal: - self.logger.info("In dryrun mode: " - "suppressing entry removal for:") - self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) - for entry in self.removal]) - self.removal = [] - - # Here is where most of the work goes - # first perform bundle filtering - all_bundle_names = [b.get('name') - for b in self.config.findall('./Bundle')] - bundles = self.config.getchildren() - if self.setup['bundle']: - # warn if non-existent bundle given - for bundle in self.setup['bundle']: - if bundle not in all_bundle_names: - self.logger.info("Warning: Bundle %s not found" % bundle) - bundles = [b for b in bundles - if b.get('name') in self.setup['bundle']] - elif self.setup['indep']: - bundles = [b for b in bundles if b.tag != 'Bundle'] - if self.setup['skipbundle']: - # warn if non-existent bundle given - if not self.setup['bundle_quick']: - for bundle in self.setup['skipbundle']: - if bundle not in all_bundle_names: - self.logger.info("Warning: Bundle %s not found" % - bundle) - bundles = [b for b in bundles - if b.get('name') not in self.setup['skipbundle']] - if self.setup['skipindep']: - bundles = [b for b in bundles if b.tag == 'Bundle'] - - self.whitelist = [e for e in self.whitelist - if any(e in b for b in bundles)] - - # first process prereq actions - for bundle in bundles[:]: - if bundle.tag == 'Bundle': - bmodified = any(item in self.whitelist for item in bundle) - else: - bmodified = False - actions = [a for a in bundle.findall('./Action') - if (a.get('timing') in ['pre', 'both'] and - (bmodified or a.get('when') == 'always'))] - # now we process all "always actions" - if self.setup['interactive']: - self.promptFilter(iprompt, actions) - self.DispatchInstallCalls(actions) - - if bundle.tag != 'Bundle': - continue - - # need to test to fail entries in whitelist - if not all(self.states[a] for a in actions): - # then display bundles forced off with entries - self.logger.info("%s %s failed prerequisite action" % - (bundle.tag, bundle.get('name'))) - bundles.remove(bundle) - b_to_remv = [ent for ent in self.whitelist if ent in bundle] - if b_to_remv: - self.logger.info("Not installing entries from %s %s" % - (bundle.tag, bundle.get('name'))) - self.logger.info(["%s:%s" % (e.tag, e.get('name')) - for e in b_to_remv]) - for ent in b_to_remv: - self.whitelist.remove(ent) - - self.logger.debug("Installing entries in the following bundle(s):") - self.logger.debug(" %s" % ", ".join(b.get("name") for b in bundles - if b.get("name"))) - - if self.setup['interactive']: - self.whitelist = self.promptFilter(iprompt, self.whitelist) - self.removal = self.promptFilter(rprompt, self.removal) - - for entry in candidates: - if entry not in self.whitelist: - self.blacklist.append(entry) - - def DispatchInstallCalls(self, entries): - """Dispatch install calls to underlying tools.""" - for tool in self.tools: - handled = [entry for entry in entries if tool.canInstall(entry)] - if not handled: - continue - try: - self.states.update(tool.Install(handled)) - except: - self.logger.error("%s.Install() call failed:" % tool.name, - exc_info=1) - - def Install(self): - """Install all entries.""" - self.DispatchInstallCalls(self.whitelist) - mods = self.modified - mbundles = [struct for struct in self.config.findall('Bundle') - if any(True for mod in mods if mod in struct)] - - if self.modified: - # Handle Bundle interdeps - if mbundles: - self.logger.info("The Following Bundles have been modified:") - self.logger.info([mbun.get('name') for mbun in mbundles]) - tbm = [(t, b) for t in self.tools for b in mbundles] - for tool, bundle in tbm: - try: - self.states.update(tool.Inventory(structures=[bundle])) - except: - self.logger.error("%s.Inventory() call failed:" % - tool.name, - exc_info=1) - clobbered = [entry for bundle in mbundles for entry in bundle - if (not self.states[entry] and - entry not in self.blacklist)] - if clobbered: - self.logger.debug("Found clobbered entries:") - self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) - for entry in clobbered]) - if not self.setup['interactive']: - self.DispatchInstallCalls(clobbered) - - for bundle in self.config.findall('.//Bundle'): - if (self.setup['bundle'] and - bundle.get('name') not in self.setup['bundle']): - # prune out unspecified bundles when running with -b - continue - if bundle in mbundles: - self.logger.debug("Bundle %s was modified" % - bundle.get('name')) - func = "BundleUpdated" - else: - self.logger.debug("Bundle %s was not modified" % - bundle.get('name')) - func = "BundleNotUpdated" - for tool in self.tools: - try: - self.states.update(getattr(tool, func)(bundle)) - except: - self.logger.error("%s.%s(%s:%s) call failed:" % - (tool.name, func, bundle.tag, - bundle.get("name")), exc_info=1) - - for indep in self.config.findall('.//Independent'): - for tool in self.tools: - try: - self.states.update(tool.BundleNotUpdated(indep)) - except: - self.logger.error("%s.BundleNotUpdated(%s:%s) call failed:" - % (tool.name, indep.tag, - indep.get("name")), exc_info=1) - - def Remove(self): - """Remove extra entries.""" - for tool in self.tools: - extras = [entry for entry in self.removal - if tool.handlesEntry(entry)] - if extras: - try: - tool.Remove(extras) - except: - self.logger.error("%s.Remove() failed" % tool.name, - exc_info=1) - - def CondDisplayState(self, phase): - """Conditionally print tracing information.""" - self.logger.info('Phase: %s' % phase) - self.logger.info('Correct entries: %d' % - list(self.states.values()).count(True)) - self.logger.info('Incorrect entries: %d' % - list(self.states.values()).count(False)) - if phase == 'final' and list(self.states.values()).count(False): - for entry in sorted(self.states.keys(), key=lambda e: e.tag + ":" + - e.get('name')): - if not self.states[entry]: - etype = entry.get('type') - if etype: - self.logger.info("%s:%s:%s" % (entry.tag, etype, - entry.get('name'))) - else: - self.logger.info("%s:%s" % (entry.tag, - entry.get('name'))) - self.logger.info('Total managed entries: %d' % - len(list(self.states.values()))) - self.logger.info('Unmanaged entries: %d' % len(self.extra)) - if phase == 'final' and self.setup['extra']: - for entry in sorted(self.extra, key=lambda e: e.tag + ":" + - e.get('name')): - etype = entry.get('type') - if etype: - self.logger.info("%s:%s:%s" % (entry.tag, etype, - entry.get('name'))) - else: - self.logger.info("%s:%s" % (entry.tag, - entry.get('name'))) - - if ((list(self.states.values()).count(False) == 0) and not self.extra): - self.logger.info('All entries correct.') - - def ReInventory(self): - """Recheck everything.""" - if not self.dryrun and self.setup['kevlar']: - self.logger.info("Rechecking system inventory") - self.Inventory() - - def Execute(self): - """Run all methods.""" - self.Inventory() - self.times['inventory'] = time.time() - self.CondDisplayState('initial') - self.InstallImportant() - self.Decide() - self.Install() - self.times['install'] = time.time() - self.Remove() - self.times['remove'] = time.time() - if self.modified: - self.ReInventory() - self.times['reinventory'] = time.time() - self.times['finished'] = time.time() - self.CondDisplayState('final') - - def GenerateStats(self): - """Generate XML summary of execution statistics.""" - feedback = Bcfg2.Client.XML.Element("upload-statistics") - stats = Bcfg2.Client.XML.SubElement( - feedback, - 'Statistics', - total=str(len(self.states)), - version='2.0', - revision=self.config.get('revision', '-1')) - good_entries = [key for key, val in list(self.states.items()) if val] - good = len(good_entries) - stats.set('good', str(good)) - if any(not val for val in list(self.states.values())): - stats.set('state', 'dirty') - else: - stats.set('state', 'clean') - - # List bad elements of the configuration - for (data, ename) in [(self.modified, 'Modified'), - (self.extra, "Extra"), - (good_entries, "Good"), - ([entry for entry in self.states - if not self.states[entry]], "Bad")]: - container = Bcfg2.Client.XML.SubElement(stats, ename) - for item in data: - item.set('qtext', '') - container.append(copy.deepcopy(item)) - item.text = None - - timeinfo = Bcfg2.Client.XML.Element("OpStamps") - feedback.append(stats) - for (event, timestamp) in list(self.times.items()): - timeinfo.set(event, str(timestamp)) - stats.append(timeinfo) - return feedback diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py index fbf114de6..98d081b10 100644 --- a/src/lib/Bcfg2/Client/Proxy.py +++ b/src/lib/Bcfg2/Client/Proxy.py @@ -1,6 +1,10 @@ import re +import sys +import time import socket import logging +import Bcfg2.Options +from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus # The ssl module is provided by either Python 2.6 or a separate ssl # package that works on older versions of Python (see @@ -16,11 +20,6 @@ except ImportError: SSL_LIB = 'm2crypto' SSL_ERROR = SSL.SSLError -import sys -import time - -# Compatibility imports -from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus version = sys.version_info[:2] has_py26 = version >= (2, 6) @@ -64,6 +63,7 @@ class CertificateError(Exception): _orig_Method = xmlrpclib._Method + class RetryMethod(xmlrpclib._Method): """Method with error handling and retries built in.""" log = logging.getLogger('xmlrpc') @@ -104,7 +104,6 @@ class RetryMethod(xmlrpclib._Method): err = sys.exc_info()[1] msg = err except: - raise etype, err = sys.exc_info()[:2] msg = "Unknown failure: %s (%s)" % (err, etype.__name__) if msg: @@ -218,12 +217,15 @@ class SSLHTTPConnection(httplib.HTTPConnection): other_side_required = ssl.CERT_REQUIRED else: other_side_required = ssl.CERT_NONE - self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") + self.logger.warning("No ca is specified. Cannot authenticate the " + "server with SSL.") if self.cert and not self.key: - self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot " + "authenticate this client with SSL.") self.cert = None if self.key and not self.cert: - self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") + self.logger.warning("SSL key specfied, but no cert. Cannot " + "authenticate this client with SSL.") self.key = None rawsock.settimeout(self.timeout) @@ -234,7 +236,8 @@ class SSLHTTPConnection(httplib.HTTPConnection): self.sock.connect((self.host, self.port)) peer_cert = self.sock.getpeercert() if peer_cert and self.scns: - scn = [x[0][1] for x in peer_cert['subject'] if x[0][0] == 'commonName'][0] + scn = [x[0][1] for x in peer_cert['subject'] + if x[0][0] == 'commonName'][0] if scn not in self.scns: raise CertificateError(scn) self.sock.closeSocket = True @@ -253,20 +256,24 @@ class SSLHTTPConnection(httplib.HTTPConnection): if self.ca: # Use the certificate authority to validate the cert # presented by the server - ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, depth=9) + ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, + depth=9) if ctx.load_verify_locations(self.ca) != 1: raise Exception('No CA certs') else: - self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") + self.logger.warning("No ca is specified. Cannot authenticate the " + "server with SSL.") if self.cert and self.key: # A cert/key is defined, use them to support client # authentication to the server ctx.load_cert(self.cert, self.key) elif self.cert: - self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot " + "authenticate this client with SSL.") elif self.key: - self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") + self.logger.warning("SSL key specfied, but no cert. Cannot " + "authenticate this client with SSL.") self.sock = SSL.Connection(ctx) if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', self.host): @@ -343,26 +350,50 @@ class XMLRPCTransport(xmlrpclib.Transport): # pylint: enable=E1101 -def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, - allowedServerCNs=None, timeout=90, retries=3, delay=1): - - """Constructs proxies to components. - - Arguments: - component_name -- name of the component to connect to - - Additional arguments are passed to the ServerProxy constructor. - - """ - xmlrpclib._Method.max_retries = retries - xmlrpclib._Method.retry_delay = delay - - if user and password: - method, path = urlparse(url)[:2] - newurl = "%s://%s:%s@%s" % (method, quote_plus(user, ''), - quote_plus(password, ''), path) - else: - newurl = url - ssl_trans = XMLRPCTransport(key, cert, ca, - allowedServerCNs, timeout=float(timeout)) - return xmlrpclib.ServerProxy(newurl, allow_none=True, transport=ssl_trans) +class ComponentProxy(xmlrpclib.ServerProxy): + """Constructs proxies to components. """ + + options = [ + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_key, + Bcfg2.Options.Common.ssl_cert, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.password, + Bcfg2.Options.Option( + "-u", "--user", default="root", cf=('communication', 'user'), + help='The user to provide for authentication'), + Bcfg2.Options.Option( + "-R", "--retries", type=int, default=3, + cf=('communication', 'retries'), + help='The number of times to retry network communication'), + Bcfg2.Options.Option( + "-y", "--retry-delay", type=int, default=1, + cf=('communication', 'retry_delay'), + help='The time in seconds to wait between retries'), + Bcfg2.Options.Option( + '--ssl-cns', cf=('communication', 'serverCommonNames'), + type=Bcfg2.Options.Types.colon_list, + help='List of server commonNames'), + Bcfg2.Options.Option( + "-t", "--timeout", type=float, default=90.0, + cf=('communication', 'timeout'), + help='Set the client XML-RPC timeout')] + + def __init__(self): + RetryMethod.max_retries = Bcfg2.Options.setup.retries + RetryMethod.retry_delay = Bcfg2.Options.setup.retry_delay + + if Bcfg2.Options.setup.user and Bcfg2.Options.setup.password: + method, path = urlparse(Bcfg2.Options.setup.server)[:2] + url = "%s://%s:%s@%s" % ( + method, + quote_plus(Bcfg2.Options.setup.user, ''), + quote_plus(Bcfg2.Options.setup.password, ''), + path) + else: + url = Bcfg2.Options.setup.server + ssl_trans = XMLRPCTransport(Bcfg2.Options.setup.key, + Bcfg2.Options.setup.cert, + Bcfg2.Options.setup.ca, + Bcfg2.Options.setup.ssl_cns, + Bcfg2.Options.setup.timeout) + xmlrpclib.ServerProxy.__init__(self, url, + allow_none=True, transport=ssl_trans) diff --git a/src/lib/Bcfg2/Client/Tools/APK.py b/src/lib/Bcfg2/Client/Tools/APK.py index 46f46bb1c..457197c28 100644 --- a/src/lib/Bcfg2/Client/Tools/APK.py +++ b/src/lib/Bcfg2/Client/Tools/APK.py @@ -33,8 +33,6 @@ class APK(Bcfg2.Client.Tools.PkgTool): if entry.attrib['name'] in self.installed: if entry.attrib['version'] in \ ['auto', self.installed[entry.attrib['name']]]: - #if not self.setup['quick'] and \ - # entry.get('verify', 'true') == 'true': #FIXME: Does APK have any sort of verification mechanism? return True else: diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index f449557aa..5f14b43ed 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -4,16 +4,27 @@ import warnings warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) -import apt.cache import os +import apt.cache +import Bcfg2.Options import Bcfg2.Client.Tools + class APT(Bcfg2.Client.Tools.Tool): - """The Debian toolset implements package and service operations and inherits - the rest from Toolset.Toolset. + """The Debian toolset implements package and service operations + and inherits the rest from Tools.Tool. """ + + options = Bcfg2.Client.Tools.Tool.options + [ + Bcfg2.Options.PathOption( + cf=('APT', 'install_path'), default='/usr', dest='apt_install_path', + help='Apt tools install path'), + Bcfg2.Options.PathOption( + cf=('APT', 'var_path'), default='/var', dest='apt_var_path', + help='Apt tools var path'), + Bcfg2.Options.PathOption( + cf=('APT', 'etc_path'), default='/etc', dest='apt_etc_path', + help='System etc path')] - """ - name = 'APT' __execs__ = [] __handles__ = [('Package', 'deb'), ('Path', 'ignore')] __req__ = {'Package': ['name', 'version'], 'Path': ['type']} @@ -21,12 +32,9 @@ class APT(Bcfg2.Client.Tools.Tool): def __init__(self, config): Bcfg2.Client.Tools.Tool.__init__(self, config) - self.install_path = self.setup.get('apt_install_path', '/usr') - self.var_path = self.setup.get('apt_var_path', '/var') - self.etc_path = self.setup.get('apt_etc_path', '/etc') - self.debsums = '%s/bin/debsums' % self.install_path - self.aptget = '%s/bin/apt-get' % self.install_path - self.dpkg = '%s/bin/dpkg' % self.install_path + self.debsums = '%s/bin/debsums' % Bcfg2.Options.setup.apt_install_path + self.aptget = '%s/bin/apt-get' % Bcfg2.Options.setup.apt_install_path + self.dpkg = '%s/bin/dpkg' % Bcfg2.Options.setup.apt_install_path self.__execs__ = [self.debsums, self.aptget, self.dpkg] path_entries = os.environ['PATH'].split(':') @@ -38,7 +46,7 @@ class APT(Bcfg2.Client.Tools.Tool): '-o DPkg::Options::=--force-confmiss ' + \ '--reinstall ' + \ '--force-yes ' - if not self.setup['debug']: + if not Bcfg2.Options.setup.debug: self.pkgcmd += '-q=2 ' self.pkgcmd += '-y install %s' self.ignores = [entry.get('name') for struct in config \ @@ -46,19 +54,23 @@ class APT(Bcfg2.Client.Tools.Tool): if entry.tag == 'Path' and \ entry.get('type') == 'ignore'] self.__important__ = self.__important__ + \ - ["%s/cache/debconf/config.dat" % self.var_path, - "%s/cache/debconf/templates.dat" % self.var_path, - '/etc/passwd', '/etc/group', - '%s/apt/apt.conf' % self.etc_path, - '%s/dpkg/dpkg.cfg' % self.etc_path] + \ - [entry.get('name') for struct in config for entry in struct \ - if entry.tag == 'Path' and \ - entry.get('name').startswith('%s/apt/sources.list' % self.etc_path)] - self.nonexistent = [entry.get('name') for struct in config for entry in struct \ - if entry.tag == 'Path' and entry.get('type') == 'nonexistent'] + [ + "%s/cache/debconf/config.dat" % Bcfg2.Options.setup.apt_var_path, + "%s/cache/debconf/templates.dat" % Bcfg2.Options.setup.apt_var_path, + '/etc/passwd', '/etc/group', + '%s/apt/apt.conf' % Bcfg2.Options.setup.apt_etc_path, + '%s/dpkg/dpkg.cfg' % Bcfg2.Options.setup.apt_etc_path] + \ + [entry.get('name') for struct in config + for entry in struct + if (entry.tag == 'Path' and + entry.get('name').startswith( + '%s/apt/sources.list' % Bcfg2.Options.setup.apt_etc_path))] + self.nonexistent = [ + entry.get('name') for struct in config for entry in struct + if entry.tag == 'Path' and entry.get('type') == 'nonexistent'] os.environ["DEBIAN_FRONTEND"] = 'noninteractive' self.actions = {} - if self.setup['kevlar'] and not self.setup['dryrun']: + if Bcfg2.Options.setup.kevlar and not Bcfg2.Options.setup.dry_run: self.cmd.run("%s --force-confold --configure --pending" % self.dpkg) self.cmd.run("%s clean" % self.aptget) @@ -184,8 +196,9 @@ class APT(Bcfg2.Client.Tools.Tool): return False else: # version matches - if not self.setup['quick'] and entry.get('verify', 'true') == 'true' \ - and checksums: + if (not Bcfg2.Options.setup.quick and + entry.get('verify', 'true') == 'true' + and checksums): pkgsums = self.VerifyDebsums(entry, modlist) return pkgsums return True diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index 05e35befc..921d5723e 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -2,10 +2,9 @@ import os import sys -import select import Bcfg2.Client.Tools -from Bcfg2.Client.Frame import matches_white_list, passes_black_list -from Bcfg2.Compat import input # pylint: disable=W0622 +from Bcfg2.Utils import safe_input +from Bcfg2.Client import matches_white_list, passes_black_list class Action(Bcfg2.Client.Tools.Tool): @@ -17,13 +16,13 @@ class Action(Bcfg2.Client.Tools.Tool): def _action_allowed(self, action): """ Return true if the given action is allowed to be run by the whitelist or blacklist """ - if self.setup['decision'] == 'whitelist' and \ - not matches_white_list(action, self.setup['decision_list']): + if (Bcfg2.Options.setup.decision == 'whitelist' and + not matches_white_list(action, Bcfg2.Options.setup.decision_list)): self.logger.info("In whitelist mode: suppressing Action: %s" % action.get('name')) return False - if self.setup['decision'] == 'blacklist' and \ - not passes_black_list(action, self.setup['decision_list']): + if (Bcfg2.Options.setup.decision == 'blacklist' and + not passes_black_list(action, Bcfg2.Options.setup.decision_list)): self.logger.info("In blacklist mode: suppressing Action: %s" % action.get('name')) return False @@ -37,19 +36,15 @@ class Action(Bcfg2.Client.Tools.Tool): shell = True shell_string = '(in shell) ' - if not self.setup['dryrun']: - if self.setup['interactive']: + if not Bcfg2.Options.setup.dryrun: + if Bcfg2.Options.setup.interactive: prompt = ('Run Action %s%s, %s: (y/N): ' % (shell_string, entry.get('name'), entry.get('command'))) - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], - 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - ans = input(prompt) + ans = safe_input(prompt) if ans not in ['y', 'Y']: return False - if self.setup['servicemode'] == 'build': + if Bcfg2.Options.setup.service_mode == 'build': if entry.get('build', 'true') == 'false': self.logger.debug("Action: Deferring execution of %s due " "to build mode" % entry.get('command')) diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index 4833f3f68..c2c7e21c1 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -3,7 +3,6 @@ """This is chkconfig support.""" import os - import Bcfg2.Client.Tools import Bcfg2.Client.XML @@ -96,15 +95,15 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): bootcmd = '/sbin/chkconfig %s %s' % (entry.get('name'), bootstatus) bootcmdrv = self.cmd.run(bootcmd).success - if self.setup['servicemode'] == 'disabled': + if Bcfg2.Options.setup.servicemode == 'disabled': # 'disabled' means we don't attempt to modify running svcs return bootcmdrv - buildmode = self.setup['servicemode'] == 'build' - if (entry.get('status') == 'on' and not buildmode) and \ - entry.get('current_status') == 'off': + buildmode = Bcfg2.Options.setup.servicemode == 'build' + if ((entry.get('status') == 'on' and not buildmode) and + entry.get('current_status') == 'off'): svccmdrv = self.start_service(entry) - elif (entry.get('status') == 'off' or buildmode) and \ - entry.get('current_status') == 'on': + elif ((entry.get('status') == 'off' or buildmode) and + entry.get('current_status') == 'on'): svccmdrv = self.stop_service(entry) else: svccmdrv = True # ignore status attribute diff --git a/src/lib/Bcfg2/Client/Tools/MacPorts.py b/src/lib/Bcfg2/Client/Tools/MacPorts.py index dcf58cfec..c28f8c743 100644 --- a/src/lib/Bcfg2/Client/Tools/MacPorts.py +++ b/src/lib/Bcfg2/Client/Tools/MacPorts.py @@ -39,8 +39,6 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool): if entry.attrib['name'] in self.installed: if (self.installed[entry.attrib['name']] == entry.attrib['version'] or entry.attrib['version'] == 'any'): - #if not self.setup['quick'] and \ - # entry.get('verify', 'true') == 'true': #FIXME: We should be able to check this once # http://trac.macports.org/ticket/15709 is implemented return True diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 9f47fb53a..482320e0d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -6,6 +6,7 @@ import stat import time import difflib import tempfile +import Bcfg2.Options from Bcfg2.Client.Tools.POSIX.base import POSIXTool from Bcfg2.Compat import unicode, b64encode, b64decode # pylint: disable=W0622 @@ -43,7 +44,7 @@ class POSIXFile(POSIXTool): tempdata = entry.text if isinstance(tempdata, unicode) and unicode != str: try: - tempdata = tempdata.encode(self.setup['encoding']) + tempdata = tempdata.encode(Bcfg2.Options.setup.encoding) except UnicodeEncodeError: err = sys.exc_info()[1] self.logger.error("POSIX: Error encoding file %s: %s" % @@ -82,7 +83,7 @@ class POSIXFile(POSIXTool): self.logger.debug("POSIX: %s has incorrect contents" % entry.get("name")) self._get_diffs( - entry, interactive=self.setup['interactive'], + entry, interactive=Bcfg2.Options.setup.interactive, sensitive=entry.get('sensitive', 'false').lower() == 'true', is_binary=is_binary, content=content) return POSIXTool.verify(self, entry, modlist) and not different @@ -170,7 +171,8 @@ class POSIXFile(POSIXTool): (entry.get("name"), sys.exc_info()[1])) return False if not is_binary: - is_binary |= not self._is_string(content, self.setup['encoding']) + is_binary |= not self._is_string(content, + Bcfg2.Options.setup.encoding) if is_binary: # don't compute diffs if the file is binary prompt.append('Binary file, no printable diff') @@ -183,7 +185,7 @@ class POSIXFile(POSIXTool): if diff: udiff = '\n'.join(l.rstrip('\n') for l in diff) if hasattr(udiff, "decode"): - udiff = udiff.decode(self.setup['encoding']) + udiff = udiff.decode(Bcfg2.Options.setup.encoding) try: prompt.append(udiff) except UnicodeEncodeError: diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py index 4f1f8e5aa..db0fa96ab 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py @@ -4,20 +4,31 @@ import os import re import sys import shutil -from datetime import datetime +import Bcfg2.Options import Bcfg2.Client.Tools +from datetime import datetime from Bcfg2.Compat import walk_packages from Bcfg2.Client.Tools.POSIX.base import POSIXTool class POSIX(Bcfg2.Client.Tools.Tool): """POSIX File support code.""" - name = 'POSIX' + + options = Bcfg2.Client.Tools.Tool.options + [ + Bcfg2.Options.PathOption( + cf=('paranoid', 'path'), default='/var/cache/bcfg2', + dest='paranoid_path', + help='Specify path for paranoid file backups'), + Bcfg2.Options.Option( + cf=('paranoid', 'max_copies'), default=1, type=int, + dest='paranoid_copies', + help='Specify the number of paranoid copies you want'), + Bcfg2.Options.BooleanOption( + '-P', '--paranoid', cf=('client', 'paranoid'), + help='Make automatic backups of config files')] def __init__(self, config): Bcfg2.Client.Tools.Tool.__init__(self, config) - self.ppath = self.setup['ppath'] - self.max_copies = self.setup['max_copies'] self._handlers = self._load_handlers() self.logger.debug("POSIX: Handlers loaded: %s" % (", ".join(self._handlers.keys()))) @@ -89,7 +100,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): self.logger.debug("POSIX: Verifying entry %s:%s:%s" % (entry.tag, entry.get("type"), entry.get("name"))) ret = self._handlers[entry.get("type")].verify(entry, modlist) - if self.setup['interactive'] and not ret: + if Bcfg2.Options.setup.interactive and not ret: entry.set('qtext', '%s\nInstall %s %s: (y/N) ' % (entry.get('qtext', ''), @@ -103,35 +114,39 @@ class POSIX(Bcfg2.Client.Tools.Tool): bkupnam + r'_\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}$') # current list of backups for this file try: - bkuplist = [f for f in os.listdir(self.ppath) if - bkup_re.match(f)] + bkuplist = [f + for f in os.listdir(Bcfg2.Options.setup.paranoid_path) + if bkup_re.match(f)] except OSError: err = sys.exc_info()[1] self.logger.error("POSIX: Failed to create backup list in %s: %s" % - (self.ppath, err)) + (Bcfg2.Options.setup.paranoid_path, err)) return bkuplist.sort() - while len(bkuplist) >= int(self.max_copies): + while len(bkuplist) >= int(Bcfg2.Options.setup.paranoid_copies): # remove the oldest backup available oldest = bkuplist.pop(0) self.logger.info("POSIX: Removing old backup %s" % oldest) try: - os.remove(os.path.join(self.ppath, oldest)) + os.remove(os.path.join(Bcfg2.Options.setup.paranoid_path, + oldest)) except OSError: err = sys.exc_info()[1] - self.logger.error("POSIX: Failed to remove old backup %s: %s" % - (os.path.join(self.ppath, oldest), err)) + self.logger.error( + "POSIX: Failed to remove old backup %s: %s" % + (os.path.join(Bcfg2.Options.setup.paranoid_path, oldest), + err)) def _paranoid_backup(self, entry): """ Take a backup of the specified entry for paranoid mode """ if (entry.get("paranoid", 'false').lower() == 'true' and - self.setup.get("paranoid", False) and + Bcfg2.Options.setup.paranoid and entry.get('current_exists', 'true') == 'true' and not os.path.isdir(entry.get("name"))): self._prune_old_backups(entry) bkupnam = "%s_%s" % (entry.get('name').replace('/', '_'), datetime.isoformat(datetime.now())) - bfile = os.path.join(self.ppath, bkupnam) + bfile = os.path.join(Bcfg2.Options.setup.paranoid_path, bkupnam) try: shutil.copy(entry.get('name'), bfile) self.logger.info("POSIX: Backup of %s saved to %s" % diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index fb5d06e54..c9164cb88 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -105,23 +105,23 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): path = entry.get("name") rv = True - if entry.get("owner") and entry.get("group"): - try: - self.logger.debug("POSIX: Setting ownership of %s to %s:%s" % - (path, - self._norm_entry_uid(entry), - self._norm_entry_gid(entry))) - os.chown(path, self._norm_entry_uid(entry), - self._norm_entry_gid(entry)) - except KeyError: - self.logger.error('POSIX: Failed to change ownership of %s' % - path) - rv = False - os.chown(path, 0, 0) - except OSError: - self.logger.error('POSIX: Failed to change ownership of %s' % - path) - rv = False + if os.geteuid() == 0: + if entry.get("owner") and entry.get("group"): + try: + self.logger.debug("POSIX: Setting ownership of %s to %s:%s" + % (path, + self._norm_entry_uid(entry), + self._norm_entry_gid(entry))) + os.chown(path, self._norm_entry_uid(entry), + self._norm_entry_gid(entry)) + except (OSError, KeyError): + self.logger.error('POSIX: Failed to change ownership of %s' + % path) + rv = False + if sys.exc_info()[0] == KeyError: + os.chown(path, 0, 0) + else: + self.logger.debug("POSIX: Run as non-root, not setting ownership") if entry.get("mode"): wanted_mode = int(entry.get('mode'), 8) diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 9d7441b5c..19657f12a 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -8,9 +8,31 @@ import Bcfg2.Client.Tools from Bcfg2.Utils import PackedDigitRange +def uid_range_type(val): + return PackedDigitRange(*Bcfg2.Options.Types.comma_list(val)) + + class POSIXUsers(Bcfg2.Client.Tools.Tool): """ A tool to handle creating users and groups with useradd/mod/del and groupadd/mod/del """ + options = Bcfg2.Client.Tools.Tool.options + [ + Bcfg2.Options.Option( + cf=('POSIXUsers', 'uid_whitelist'), default=[], + type=uid_range_type, + help="UID ranges the POSIXUsers tool will manage"), + Bcfg2.Options.Option( + cf=('POSIXUsers', 'gid_whitelist'), default=[], + type=uid_range_type, + help="GID ranges the POSIXUsers tool will manage"), + Bcfg2.Options.Option( + cf=('POSIXUsers', 'uid_blacklist'), default=[], + type=uid_range_type, + help="UID ranges the POSIXUsers tool will not manage"), + Bcfg2.Options.Option( + cf=('POSIXUsers', 'gid_blacklist'), default=[], + type=uid_range_type, + help="GID ranges the POSIXUsers tool will not manage")] + __execs__ = ['/usr/sbin/useradd', '/usr/sbin/usermod', '/usr/sbin/userdel', '/usr/sbin/groupadd', '/usr/sbin/groupmod', '/usr/sbin/groupdel'] @@ -34,20 +56,10 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): self.set_defaults = dict(POSIXUser=self.populate_user_entry, POSIXGroup=lambda g: g) self._existing = None - self._whitelist = dict(POSIXUser=None, POSIXGroup=None) - self._blacklist = dict(POSIXUser=None, POSIXGroup=None) - if self.setup['posix_uid_whitelist']: - self._whitelist['POSIXUser'] = \ - PackedDigitRange(*self.setup['posix_uid_whitelist']) - else: - self._blacklist['POSIXUser'] = \ - PackedDigitRange(*self.setup['posix_uid_blacklist']) - if self.setup['posix_gid_whitelist']: - self._whitelist['POSIXGroup'] = \ - PackedDigitRange(*self.setup['posix_gid_whitelist']) - else: - self._blacklist['POSIXGroup'] = \ - PackedDigitRange(*self.setup['posix_gid_blacklist']) + self._whitelist = dict(POSIXUser=Bcfg2.Options.setup.uid_whitelist, + POSIXGroup=Bcfg2.Options.setup.gid_whitelist) + self._blacklist = dict(POSIXUser=Bcfg2.Options.setup.uid_blacklist, + POSIXGroup=Bcfg2.Options.setup.gid_blacklist) @property def existing(self): @@ -164,7 +176,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): % (entry.tag, entry.get("name"), actual, expected)])) rv = False - if self.setup['interactive'] and not rv: + if Bcfg2.Options.setup.interactive and not rv: entry.set('qtext', '%s\nInstall %s %s: (y/N) ' % (entry.get('qtext', ''), entry.tag, entry.get('name'))) @@ -173,7 +185,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): def VerifyPOSIXGroup(self, entry, _): """ Verify a POSIXGroup entry """ rv = self._verify(entry) - if self.setup['interactive'] and not rv: + if Bcfg2.Options.setup.interactive and not rv: entry.set('qtext', '%s\nInstall %s %s: (y/N) ' % (entry.get('qtext', ''), entry.tag, entry.get('name'))) diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index d7d60a66d..2ab9b7403 100644 --- a/src/lib/Bcfg2/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py @@ -38,8 +38,6 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): return True elif self.installed[entry.attrib['name']] == \ entry.attrib['version']: - #if not self.setup['quick'] and \ - # entry.get('verify', 'true') == 'true': #FIXME: need to figure out if pacman # allows you to verify packages return True diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py index e52da081b..a877b564f 100644 --- a/src/lib/Bcfg2/Client/Tools/Portage.py +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -5,9 +5,13 @@ import Bcfg2.Client.Tools class Portage(Bcfg2.Client.Tools.PkgTool): - """The Gentoo toolset implements package and service operations and - inherits the rest from Toolset.Toolset.""" - name = 'Portage' + """The Gentoo toolset implements package and service operations + and inherits the rest from Tools.Tool.""" + + options = Bcfg2.Client.Tools.PkgTool.options + [ + Bcfg2.Options.BooleanOption( + cf=('Portage', 'binpkgonly'), help='Portage binary packages only')] + __execs__ = ['/usr/bin/emerge', '/usr/bin/equery'] __handles__ = [('Package', 'ebuild')] __req__ = {'Package': ['name', 'version']} @@ -25,8 +29,7 @@ class Portage(Bcfg2.Client.Tools.PkgTool): self._pkg_pattern = re.compile(r'(.*)-(\d.*)') self._ebuild_pattern = re.compile('(ebuild|binary)') self.installed = {} - self._binpkgonly = self.setup.get('portage_binpkgonly', False) - if self._binpkgonly: + if Bcfg2.Options.setup.binpkgonly: self.pkgtool = self._binpkgtool self.RefreshPackages() @@ -61,7 +64,7 @@ class Portage(Bcfg2.Client.Tools.PkgTool): version = self.installed[entry.get('name')] entry.set('current_version', version) - if not self.setup['quick']: + if not Bcfg2.Options.setup.quick: if ('verify' not in entry.attrib or entry.get('verify').lower() == 'true'): diff --git a/src/lib/Bcfg2/Client/Tools/RPM.py b/src/lib/Bcfg2/Client/Tools/RPM.py index be5ad01e2..1ebc61c93 100644 --- a/src/lib/Bcfg2/Client/Tools/RPM.py +++ b/src/lib/Bcfg2/Client/Tools/RPM.py @@ -1075,6 +1075,42 @@ if __name__ == "__main__": class RPM(Bcfg2.Client.Tools.PkgTool): """Support for RPM packages.""" + options = Bcfg2.Client.Tools.PkgTool.options + [ + Bcfg2.Options.Option( + cf=('RPM', 'installonlypackages'), dest="rpm_installonly", + type=Bcfg2.Options.Types.comma_list, + default=['kernel', 'kernel-bigmem', 'kernel-enterprise', + 'kernel-smp', 'kernel-modules', 'kernel-debug', + 'kernel-unsupported', 'kernel-devel', 'kernel-source', + 'kernel-default', 'kernel-largesmp-devel', + 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], + help='RPM install-only packages'), + Bcfg2.Options.BooleanOption( + cf=('RPM', 'pkg_checks'), default=True, dest="rpm_pkg_checks", + help="Perform RPM package checks"), + Bcfg2.Options.BooleanOption( + cf=('RPM', 'pkg_verify'), default=True, dest="rpm_pkg_verify", + help="Perform RPM package verify"), + Bcfg2.Options.BooleanOption( + cf=('RPM', 'install_missing'), default=True, + dest="rpm_install_missing", + help="Install missing packages"), + Bcfg2.Options.Option( + cf=('RPM', 'erase_flags'), default=["allmatches"], + dest="rpm_erase_flags", + help="RPM erase flags"), + Bcfg2.Options.BooleanOption( + cf=('RPM', 'fix_version'), default=True, + dest="rpm_fix_version", + help="Fix (upgrade or downgrade) packages with the wrong version"), + Bcfg2.Options.BooleanOption( + cf=('RPM', 'reinstall_broken'), default=True, + dest="rpm_reinstall_broken", + help="Reinstall packages that fail to verify"), + Bcfg2.Options.Option( + cf=('RPM', 'verify_flags'), default=[], dest="rpm_verify_flags", + help="RPM verify flags")] + __execs__ = ['/bin/rpm', '/var/lib/rpm'] __handles__ = [('Package', 'rpm')] @@ -1109,43 +1145,36 @@ class RPM(Bcfg2.Client.Tools.PkgTool): self.modlists = {} self.gpg_keyids = self.getinstalledgpg() - opt_prefix = self.name.lower() - self.installOnlyPkgs = self.setup["%s_installonly" % opt_prefix] + self.installOnlyPkgs = Bcfg2.Options.setup.rpm_installonly if 'gpg-pubkey' not in self.installOnlyPkgs: self.installOnlyPkgs.append('gpg-pubkey') - self.erase_flags = self.setup['%s_erase_flags' % opt_prefix] - self.pkg_checks = self.setup['%s_pkg_checks' % opt_prefix] - self.pkg_verify = self.setup['%s_pkg_verify' % opt_prefix] - self.installed_action = self.setup['%s_installed_action' % opt_prefix] - self.version_fail_action = self.setup['%s_version_fail_action' % - opt_prefix] - self.verify_fail_action = self.setup['%s_verify_fail_action' % - opt_prefix] - self.verify_flags = self.setup['%s_verify_flags' % opt_prefix] + self.verify_flags = Bcfg2.Options.setup.rpm_verify_flags if '' in self.verify_flags: self.verify_flags.remove('') self.logger.debug('%s: installOnlyPackages = %s' % (self.name, self.installOnlyPkgs)) self.logger.debug('%s: erase_flags = %s' % - (self.name, self.erase_flags)) + (self.name, Bcfg2.Options.setup.rpm_erase_flags)) self.logger.debug('%s: pkg_checks = %s' % - (self.name, self.pkg_checks)) + (self.name, Bcfg2.Options.setup.rpm_pkg_checks)) self.logger.debug('%s: pkg_verify = %s' % - (self.name, self.pkg_verify)) - self.logger.debug('%s: installed_action = %s' % - (self.name, self.installed_action)) - self.logger.debug('%s: version_fail_action = %s' % - (self.name, self.version_fail_action)) - self.logger.debug('%s: verify_fail_action = %s' % - (self.name, self.verify_fail_action)) + (self.name, Bcfg2.Options.setup.rpm_pkg_verify)) + self.logger.debug('%s: install_missing = %s' % + (self.name, Bcfg2.Options.setup.install_missing)) + self.logger.debug('%s: fix_version = %s' % + (self.name, Bcfg2.Options.setup.rpm_fix_version)) + self.logger.debug('%s: reinstall_broken = %s' % + (self.name, + Bcfg2.Options.setup.rpm_reinstall_broken)) self.logger.debug('%s: verify_flags = %s' % (self.name, self.verify_flags)) # Force a re- prelink of all packages if prelink exists. # Many, if not most package verifies can be caused by out of # date prelinking. - if os.path.isfile('/usr/sbin/prelink') and not self.setup['dryrun']: + if (os.path.isfile('/usr/sbin/prelink') and + not Bcfg2.Options.setup.dry_run): rv = self.cmd.run('/usr/sbin/prelink -a -mR') if rv.success: self.logger.debug('Pre-emptive prelink succeeded') @@ -1176,7 +1205,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): refresh_ts.setVSFlags(rpm._RPMVSF_NODIGESTS|rpm._RPMVSF_NOSIGNATURES) for nevra in rpmpackagelist(refresh_ts): self.installed.setdefault(nevra['name'], []).append(nevra) - if self.setup['debug']: + if Bcfg2.Options.setup.debug: print("The following package instances are installed:") for name, instances in list(self.installed.items()): self.logger.debug(" " + name) @@ -1217,7 +1246,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): instance = Bcfg2.Client.XML.SubElement(entry, 'Package') for attrib in list(entry.attrib.keys()): instance.attrib[attrib] = entry.attrib[attrib] - if (self.pkg_checks and + if (Bcfg2.Options.setup.rpm_pkg_checks and entry.get('pkg_checks', 'true').lower() == 'true'): if 'any' in [entry.get('version'), pinned_version]: version, release = 'any', 'any' @@ -1240,7 +1269,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): if entry.get('name') in self.installed: # There is at least one instance installed. - if (self.pkg_checks and + if (Bcfg2.Options.setup.rpm_pkg_checks and entry.get('pkg_checks', 'true').lower() == 'true'): rpmTs = rpm.TransactionSet() rpmHeader = None @@ -1269,7 +1298,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): self.logger.debug(" %s" % self.str_evra(inst)) self.instance_status[inst]['installed'] = True - if (self.pkg_verify and + if (Bcfg2.Options.setup.rpm_pkg_verify and inst.get('pkg_verify', 'true').lower() == 'true'): flags = inst.get('verify_flags', '').split(',') + self.verify_flags if pkg.get('gpgkeyid', '')[-8:] not in self.gpg_keyids and \ @@ -1280,7 +1309,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): pkg.get('gpgkeyid', ''))) self.logger.debug(' Disabling signature check.') - if self.setup.get('quick', False): + if Bcfg2.Options.setup.quick: if prelink_exists: flags += ['nomd5', 'nosize'] else: @@ -1328,7 +1357,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): self.logger.debug(" %s" % self.str_evra(inst)) self.instance_status[inst]['installed'] = True - if (self.pkg_verify and + if (Bcfg2.Options.setup.rpm_pkg_verify and inst.get('pkg_verify', 'true').lower() == 'true'): flags = inst.get('verify_flags', '').split(',') + self.verify_flags if pkg.get('gpgkeyid', '')[-8:] not in self.gpg_keyids and \ @@ -1339,7 +1368,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): pkg.get('gpgkeyid', ''))) self.logger.info(' Disabling signature check.') - if self.setup.get('quick', False): + if Bcfg2.Options.setup.quick: if prelink_exists: flags += ['nomd5', 'nosize'] else: @@ -1374,7 +1403,8 @@ class RPM(Bcfg2.Client.Tools.PkgTool): instance_fail = False # Dump the rpm verify results. #****Write something to format this nicely.***** - if self.setup['debug'] and self.instance_status[inst].get('verify', None): + if (Bcfg2.Options.setup.debug and + self.instance_status[inst].get('verify', None)): self.logger.debug(self.instance_status[inst]['verify']) self.instance_status[inst]['verify_fail'] = False @@ -1502,7 +1532,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): self.logger.info(" This package will be deleted in a future version of the RPM driver.") #pkgspec_list.append(pkg_spec) - erase_results = rpm_erase(pkgspec_list, self.erase_flags) + erase_results = rpm_erase(pkgspec_list, Bcfg2.Options.setup.rpm_erase_flags) if erase_results == []: self.modified += packages for pkg in pkgspec_list: @@ -1530,7 +1560,9 @@ class RPM(Bcfg2.Client.Tools.PkgTool): % (pkgspec.get('name'), self.str_evra(pkgspec))) self.logger.info(" This package will be deleted in a future version of the RPM driver.") continue # Don't delete the gpg-pubkey packages for now. - erase_results = rpm_erase([pkgspec], self.erase_flags) + erase_results = rpm_erase( + [pkgspec], + Bcfg2.Options.setup.rpm_erase_flags) if erase_results == []: pkg_modified = True self.logger.info("Deleted %s %s" % \ @@ -1555,28 +1587,27 @@ class RPM(Bcfg2.Client.Tools.PkgTool): """ fix = False - if inst_status.get('installed', False) == False: - if instance.get('installed_action', 'install') == "install" and \ - self.installed_action == "install": + if not inst_status.get('installed', False): + if (instance.get('install_missing', 'true').lower() == "true" and + Bcfg2.Options.setup.rpm_install_missing): fix = True else: self.logger.debug('Installed Action for %s %s is to not install' % \ (inst_status.get('pkg').get('name'), self.str_evra(instance))) - elif inst_status.get('version_fail', False) == True: - if instance.get('version_fail_action', 'upgrade') == "upgrade" and \ - self.version_fail_action == "upgrade": + elif inst_status.get('version_fail', False): + if (instance.get('fix_version', 'true').lower() == "true" and + Bcfg2.Options.setup.rpm_fix_version): fix = True else: self.logger.debug('Version Fail Action for %s %s is to not upgrade' % \ (inst_status.get('pkg').get('name'), self.str_evra(instance))) - elif inst_status.get('verify_fail', False) == True and self.name == "RPM": - # yum can't reinstall packages so only do this for rpm. - if instance.get('verify_fail_action', 'reinstall') == "reinstall" and \ - self.verify_fail_action == "reinstall": + elif inst_status.get('verify_fail', False): + if (instance.get('reinstall_broken', 'true').lower() == "true" and + Bcfg2.Options.setup.rpm_reinstall_broken): for inst in inst_status.get('verify'): # This needs to be a for loop rather than a straight get() # because the underlying routines handle multiple packages @@ -1633,9 +1664,8 @@ class RPM(Bcfg2.Client.Tools.PkgTool): # Remove extra instances. # Can not reverify because we don't have a package entry. if len(self.extra_instances) > 0: - if (self.setup.get('remove') == 'all' or \ - self.setup.get('remove') == 'packages') and\ - not self.setup.get('dryrun'): + if (Bcfg2.Options.setup.remove in ['all', 'packages'] and + not Bcfg2.Options.setup.dry_run): self.Remove(self.extra_instances) else: self.logger.info("The following extra package instances will be removed by the '-r' option:") @@ -1744,7 +1774,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): for inst in upgrade_pkgs]) self.RefreshPackages() - if not self.setup['kevlar']: + if not Bcfg2.Options.setup.kevlar: for pkg_entry in packages: self.logger.debug("Reverifying Failed Package %s" % (pkg_entry.get('name'))) states[pkg_entry] = self.VerifyPackage(pkg_entry, \ @@ -1847,7 +1877,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): return False # We don't want to do any checks so we don't care what the entry has in it. - if (not self.pkg_checks or + if (not Bcfg2.Options.setup.rpm_pkg_checks or entry.get('pkg_checks', 'true').lower() == 'false'): return True @@ -1914,7 +1944,7 @@ class RPM(Bcfg2.Client.Tools.PkgTool): if name not in packages: extra_entry = Bcfg2.Client.XML.Element('Package', name=name, type=self.pkgtype) for installed_inst in instances: - if self.setup['extra']: + if Bcfg2.Options.setup.extra: self.logger.info("Extra Package %s %s." % \ (name, self.str_evra(installed_inst))) tmp_entry = Bcfg2.Client.XML.SubElement(extra_entry, 'Instance', \ @@ -1927,7 +1957,6 @@ class RPM(Bcfg2.Client.Tools.PkgTool): extras.append(extra_entry) return extras - def FindExtraInstances(self, pkg_entry, installed_entry): """ Check for installed instances that are not in the config. diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 92572ef1d..ef89ef46d 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -141,7 +141,7 @@ class SELinux(Bcfg2.Client.Tools.Tool): def GenericSEVerify(self, entry, _): """Dispatch verify to the proper method according to entry tag""" rv = self.handlers[entry.tag].Verify(entry) - if entry.get('qtext') and self.setup['interactive']: + if entry.get('qtext') and Bcfg2.Options.setup.interactive: entry.set('qtext', '%s\nInstall %s: (y/N) ' % (entry.get('qtext'), @@ -174,7 +174,6 @@ class SELinuxEntryHandler(object): def __init__(self, tool, config): self.tool = tool self.logger = logging.getLogger(self.__class__.__name__) - self.setup = tool.setup self.config = config self._records = None self._all = None diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 7be7b6fa3..f149be7af 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -80,7 +80,7 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Package %s not installed" % entry.get("name")) else: - if (self.setup['quick'] or + if (Bcfg2.Options.setup.quick or entry.attrib.get('verify', 'true') == 'false'): return True rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 147615f47..ae238174b 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -119,6 +119,34 @@ class YumDisplay(yum.callbacks.ProcessTransBaseCallback): class YUM(Bcfg2.Client.Tools.PkgTool): """Support for Yum packages.""" + + options = Bcfg2.Client.Tools.PkgTool.options + [ + Bcfg2.Options.BooleanOption( + cf=('YUM', 'pkg_checks'), default=True, dest="yum_pkg_checks", + help="Perform YUM package checks"), + Bcfg2.Options.BooleanOption( + cf=('YUM', 'pkg_verify'), default=True, dest="yum_pkg_verify", + help="Perform YUM package verify"), + Bcfg2.Options.BooleanOption( + cf=('YUM', 'install_missing'), default=True, + dest="yum_install_missing", + help="Install missing packages"), + Bcfg2.Options.Option( + cf=('YUM', 'erase_flags'), default=["allmatches"], + dest="yum_erase_flags", + help="YUM erase flags"), + Bcfg2.Options.BooleanOption( + cf=('YUM', 'fix_version'), default=True, + dest="yum_fix_version", + help="Fix (upgrade or downgrade) packages with the wrong version"), + Bcfg2.Options.BooleanOption( + cf=('YUM', 'reinstall_broken'), default=True, + dest="yum_reinstall_broken", + help="Reinstall packages that fail to verify"), + Bcfg2.Options.Option( + cf=('YUM', 'verify_flags'), default=[], dest="yum_verify_flags", + help="YUM verify flags")] + pkgtype = 'yum' __execs__ = [] __handles__ = [('Package', 'yum'), @@ -173,26 +201,23 @@ class YUM(Bcfg2.Client.Tools.PkgTool): else: dest[pname] = dict(data) - # Process the Yum section from the config file. These are all - # boolean flags, either we do stuff or we don't - self.pkg_checks = self.setup["yum_pkg_checks"] - self.pkg_verify = self.setup["yum_pkg_verify"] - self.do_install = self.setup["yum_installed_action"] == "install" - self.do_upgrade = self.setup["yum_version_fail_action"] == "upgrade" - self.do_reinst = self.setup["yum_verify_fail_action"] == "reinstall" - self.verify_flags = self.setup["yum_verify_flags"] - self.installonlypkgs = self.yumbase.conf.installonlypkgs if 'gpg-pubkey' not in self.installonlypkgs: self.installonlypkgs.append('gpg-pubkey') - self.logger.debug("Yum: Install missing: %s" % self.do_install) - self.logger.debug("Yum: pkg_checks: %s" % self.pkg_checks) - self.logger.debug("Yum: pkg_verify: %s" % self.pkg_verify) - self.logger.debug("Yum: Upgrade on version fail: %s" % self.do_upgrade) - self.logger.debug("Yum: Reinstall on verify fail: %s" % self.do_reinst) + self.logger.debug("Yum: Install missing: %s" % + Bcfg2.Options.setup.yum_install_missing) + self.logger.debug("Yum: pkg_checks: %s" % + Bcfg2.Options.setup.yum_pkg_checks) + self.logger.debug("Yum: pkg_verify: %s" % + Bcfg2.Options.setup.yum_pkg_verify) + self.logger.debug("Yum: Upgrade on version fail: %s" % + Bcfg2.Options.setup.yum_fix_version) + self.logger.debug("Yum: Reinstall on verify fail: %s" % + Bcfg2.Options.setup.yum_reinstall_broken) self.logger.debug("Yum: installonlypkgs: %s" % self.installonlypkgs) - self.logger.debug("Yum: verify_flags: %s" % self.verify_flags) + self.logger.debug("Yum: verify_flags: %s" % + Bcfg2.Options.setup.yum_verify_flags) def _loadYumBase(self): ''' this may be called before PkgTool.__init__() is called on @@ -203,18 +228,14 @@ class YUM(Bcfg2.Client.Tools.PkgTool): packages. ''' rv = yum.YumBase() # pylint: disable=C0103 - if hasattr(self, "setup"): - setup = self.setup - else: - setup = Bcfg2.Options.get_option_parser() if hasattr(self, "logger"): logger = self.logger else: logger = logging.getLogger(self.name) - if setup['debug']: + if Bcfg2.Options.setup.debug: debuglevel = 3 - elif setup['verbose']: + elif Bcfg2.Options.setup.verbose: debuglevel = 2 else: debuglevel = 0 @@ -314,7 +335,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): using. Disabling file checksums is a new feature yum 3.2.17-ish """ try: - return pkg.verify(fast=self.setup.get('quick', False)) + return pkg.verify(fast=Bcfg2.Options.setup.quick) except TypeError: # Older Yum API return pkg.verify() @@ -439,9 +460,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool): package_fail = False qtext_versions = [] virt_pkg = False - pkg_checks = (self.pkg_checks and + pkg_checks = (Bcfg2.Options.setup.yum_pkg_checks and entry.get('pkg_checks', 'true').lower() == 'true') - pkg_verify = (self.pkg_verify and + pkg_verify = (Bcfg2.Options.setup.yum_pkg_verify and entry.get('pkg_verify', 'true').lower() == 'true') yum_group = False @@ -534,7 +555,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): inst.get('verify_flags').lower().replace(' ', ',').split(',') else: - verify_flags = self.verify_flags + verify_flags = Bcfg2.Options.setup.yum_verify_flags if 'arch' in nevra: # If arch is specified use it to select the package @@ -622,7 +643,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): qtext_versions.append("U(%s)" % str(all_pkg_objs[0])) continue - if self.setup.get('quick', False): + if Bcfg2.Options.setup.quick: # Passed -q on the command line continue if not (pkg_verify and @@ -696,7 +717,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): install_only = False if virt_pkg or \ - (install_only and not self.setup['kevlar']) or \ + (install_only and not Bcfg2.Options.setup.kevlar) or \ yum_group: # virtual capability supplied, we are probably dealing # with multiple packages of different names. This check @@ -904,8 +925,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Remove extra instances. # Can not reverify because we don't have a package entry. if self.extra_instances is not None and len(self.extra_instances) > 0: - if (self.setup.get('remove') == 'all' or - self.setup.get('remove') == 'packages'): + if Bcfg2.Options.setup.remove in ['all', 'packages']: self.Remove(self.extra_instances) else: self.logger.info("The following extra package instances will " @@ -930,11 +950,14 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra2string(build_yname(pkg.get('name'), inst))) continue status = self.instance_status[inst] - if not status.get('installed', False) and self.do_install: + if (not status.get('installed', False) and + Bcfg2.Options.setup.yum_install_missing): queue_pkg(pkg, inst, install_pkgs) - elif status.get('version_fail', False) and self.do_upgrade: + elif (status.get('version_fail', False) and + Bcfg2.Options.setup.yum_fix_version): queue_pkg(pkg, inst, upgrade_pkgs) - elif status.get('verify_fail', False) and self.do_reinst: + elif (status.get('verify_fail', False) and + Bcfg2.Options.setup.yum_reinstall_broken): queue_pkg(pkg, inst, reinstall_pkgs) else: # Either there was no Install/Version/Verify @@ -1010,7 +1033,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self._runYumTransaction() - if not self.setup['kevlar']: + if not Bcfg2.Options.setup.kevlar: for pkg_entry in [p for p in packages if self.canVerify(p)]: self.logger.debug("Reverifying Failed Package %s" % pkg_entry.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index 885e22761..ce75005fe 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -4,11 +4,11 @@ import os import sys import stat import logging +import Bcfg2.Options import Bcfg2.Client import Bcfg2.Client.XML from Bcfg2.Utils import Executor, ClassName from Bcfg2.Compat import walk_packages # pylint: disable=W0622 -import Bcfg2.Options __all__ = [m[1] for m in walk_packages(path=__path__)] @@ -28,6 +28,12 @@ class Tool(object): .. autoattribute:: Bcfg2.Client.Tools.Tool.__important__ """ + options = [ + Bcfg2.Options.Option( + cf=('client', 'command_timeout'), + help="Timeout when running external commands other than probes", + type=Bcfg2.Options.Types.timeout)] + #: The name of the tool. By default this uses #: :class:`Bcfg2.Client.Tools.ClassName` to ensure that it is the #: same as the name of the class. @@ -77,10 +83,6 @@ class Tool(object): :type config: lxml.etree._Element :raises: :exc:`Bcfg2.Client.Tools.ToolInstantiationError` """ - #: A :class:`Bcfg2.Options.OptionParser` object describing the - #: option set Bcfg2 was invoked with - self.setup = Bcfg2.Options.get_option_parser() - #: A :class:`logging.Logger` object that will be used by this #: tool for logging self.logger = logging.getLogger(self.name) @@ -90,7 +92,7 @@ class Tool(object): #: An :class:`Bcfg2.Utils.Executor` object for #: running external commands. - self.cmd = Executor(timeout=self.setup['command_timeout']) + self.cmd = Executor(timeout=Bcfg2.Options.setup.command_timeout) #: A list of entries that have been modified by this tool self.modified = [] @@ -136,7 +138,7 @@ class Tool(object): :param bundle: The bundle that has been updated :type bundle: lxml.etree._Element :returns: dict - A dict of the state of entries suitable for - updating :attr:`Bcfg2.Client.Frame.Frame.states` + updating :attr:`Bcfg2.Client.Client.states` """ return dict() @@ -146,7 +148,7 @@ class Tool(object): :param bundle: The bundle that has been updated :type bundle: lxml.etree._Element :returns: dict - A dict of the state of entries suitable for - updating :attr:`Bcfg2.Client.Frame.Frame.states` + updating :attr:`Bcfg2.Client.Client.states` """ return dict() @@ -172,7 +174,7 @@ class Tool(object): be used. :type structures: list of lxml.etree._Element :returns: dict - A dict of the state of entries suitable for - updating :attr:`Bcfg2.Client.Frame.Frame.states` + updating :attr:`Bcfg2.Client.Client.states` """ if not structures: structures = self.config.getchildren() @@ -210,7 +212,7 @@ class Tool(object): :param entries: The entries to install :type entries: list of lxml.etree._Element :returns: dict - A dict of the state of entries suitable for - updating :attr:`Bcfg2.Client.Frame.Frame.states` + updating :attr:`Bcfg2.Client.Client.states` """ states = dict() for entry in entries: @@ -435,7 +437,7 @@ class PkgTool(Tool): :param entries: The entries to install :type entries: list of lxml.etree._Element :returns: dict - A dict of the state of entries suitable for - updating :attr:`Bcfg2.Client.Frame.Frame.states` + updating :attr:`Bcfg2.Client.Client.states` """ self.logger.info("Trying single pass package install for pkgtype %s" % self.pkgtype) @@ -493,6 +495,12 @@ class PkgTool(Tool): class SvcTool(Tool): """ Base class for tools that handle Service entries """ + options = Tool.options + [ + Bcfg2.Options.Option( + '-s', '--service-mode', default='default', + choices=['default', 'disabled', 'build'], + help='Set client service mode')] + def __init__(self, config): Tool.__init__(self, config) #: List of services that have been restarted @@ -571,14 +579,14 @@ class SvcTool(Tool): return bool(self.cmd.run(self.get_svc_command(service, 'status'))) def Remove(self, services): - if self.setup['servicemode'] != 'disabled': + if Bcfg2.Options.setup.service_mode != 'disabled': for entry in services: entry.set("status", "off") self.InstallService(entry) Remove.__doc__ = Tool.Remove.__doc__ def BundleUpdated(self, bundle): - if self.setup['servicemode'] == 'disabled': + if Bcfg2.Options.setup.service_mode == 'disabled': return for entry in bundle: @@ -587,15 +595,16 @@ class SvcTool(Tool): restart = entry.get("restart", "true").lower() if (restart == "false" or - (restart == "interactive" and not self.setup['interactive'])): + (restart == "interactive" and + not Bcfg2.Options.setup.interactive)): continue success = False if entry.get('status') == 'on': - if self.setup['servicemode'] == 'build': + if Bcfg2.Options.setup.service_mode == 'build': success = self.stop_service(entry) elif entry.get('name') not in self.restarted: - if self.setup['interactive']: + if Bcfg2.Options.setup.interactive: if not Bcfg2.Client.prompt('Restart service %s? (y/N) ' % entry.get('name')): continue diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 6d1cb9d40..2761fcddb 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -2,8 +2,55 @@ import os import sys -import select -from Bcfg2.Compat import input # pylint: disable=W0622 +import stat +import time +import fcntl +import socket +import fnmatch +import logging +import argparse +import tempfile +import Bcfg2.Logger +import Bcfg2.Options +import XML # pylint: disable=W0403 +import Proxy # pylint: disable=W0403 +import Tools # pylint: disable=W0403 +from Bcfg2.Utils import locked, Executor, safe_input +from Bcfg2.version import __version__ +# pylint: disable=W0622 +from Bcfg2.Compat import xmlrpclib, walk_packages, any, all, cmp +# pylint: enable=W0622 + + +def cmpent(ent1, ent2): + """Sort entries.""" + if ent1.tag != ent2.tag: + return cmp(ent1.tag, ent2.tag) + else: + return cmp(ent1.get('name'), ent2.get('name')) + + +def matches_entry(entryspec, entry): + """ Determine if the Decisions-style entry specification matches + the entry. Both are tuples of (tag, name). The entryspec can + handle the wildcard * in either position. """ + if entryspec == entry: + return True + return all(fnmatch.fnmatch(entry[i], entryspec[i]) for i in [0, 1]) + + +def matches_white_list(entry, whitelist): + """ Return True if (<entry tag>, <entry name>) is in the given + whitelist. """ + return any(matches_entry(we, (entry.tag, entry.get('name'))) + for we in whitelist) + + +def passes_black_list(entry, blacklist): + """ Return True if (<entry tag>, <entry name>) is not in the given + blacklist. """ + return not any(matches_entry(be, (entry.tag, entry.get('name'))) + for be in blacklist) def prompt(msg): @@ -16,10 +63,8 @@ def prompt(msg): contain "[y/N]" if desired, etc. :type msg: string :returns: bool - True if yes, False if no """ - while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) try: - ans = input(msg) + ans = safe_input(msg) return ans in ['y', 'Y'] except UnicodeEncodeError: ans = input(msg.encode('utf-8')) @@ -27,3 +72,821 @@ def prompt(msg): except EOFError: # handle ^C on rhel-based platforms raise SystemExit(1) + except: + print("Error while reading input: %s" % sys.exc_info()[1]) + return False + + +class ClientDriverAction(Bcfg2.Options.ComponentAction): + """ Action to load client drivers """ + bases = ['Bcfg2.Client.Tools'] + fail_silently = True + + +class Client(object): + """ The main Bcfg2 client class """ + + options = Proxy.ComponentProxy.options + [ + Bcfg2.Options.Common.syslog, + Bcfg2.Options.Common.location, + Bcfg2.Options.Common.interactive, + Bcfg2.Options.BooleanOption( + "-q", "--quick", help="Disable some checksum verification"), + Bcfg2.Options.Option( + cf=('client', 'probe_timeout'), + type=Bcfg2.Options.Types.timeout, + help="Timeout when running client probes"), + Bcfg2.Options.Option( + "-b", "--only-bundles", default=[], + type=Bcfg2.Options.Types.colon_list, + help='Only configure the given bundle(s)'), + Bcfg2.Options.Option( + "-B", "--except-bundles", default=[], + type=Bcfg2.Options.Types.colon_list, + help='Configure everything except the given bundle(s)'), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption( + "-Q", "--bundle-quick", + help='Only verify the given bundle(s)'), + Bcfg2.Options.Option( + '-r', '--remove', + choices=['all', 'services', 'packages', 'users'], + help='Force removal of additional configuration items')), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.PathOption( + '-f', '--file', type=argparse.FileType('r'), + help='Configure from a file rather than querying the server'), + Bcfg2.Options.PathOption( + '-c', '--cache', type=argparse.FileType('w'), + help='Store the configuration in a file')), + Bcfg2.Options.BooleanOption( + '--exit-on-probe-failure', default=True, + cf=('client', 'exit_on_probe_failure'), + help="The client should exit if a probe fails"), + Bcfg2.Options.Option( + '-p', '--profile', cf=('client', 'profile'), + help='Assert the given profile for the host'), + Bcfg2.Options.Option( + '-l', '--decision', cf=('client', 'decision'), + choices=['whitelist', 'blacklist', 'none'], + help='Run client in server decision list mode'), + Bcfg2.Options.BooleanOption( + "-O", "--no-lock", help='Omit lock check'), + Bcfg2.Options.PathOption( + cf=('components', 'lockfile'), default='/var/lock/bcfg2.run', + help='Client lock file'), + Bcfg2.Options.BooleanOption( + "-n", "--dry-run", help='Do not actually change the system'), + Bcfg2.Options.Option( + "-D", "--drivers", cf=('client', 'drivers'), + type=Bcfg2.Options.Types.comma_list, + default=[m[1] for m in walk_packages(path=Tools.__path__)], + action=ClientDriverAction, help='Client drivers'), + Bcfg2.Options.BooleanOption( + "-e", "--show-extra", help='Enable extra entry output'), + Bcfg2.Options.BooleanOption( + "-k", "--kevlar", help='Run in bulletproof mode')] + + def __init__(self): + self.config = None + self._proxy = None + self.logger = logging.getLogger('bcfg2') + self.cmd = Executor(Bcfg2.Options.setup.probe_timeout) + self.tools = [] + self.times = dict() + self.times['initialization'] = time.time() + + if Bcfg2.Options.setup.bundle_quick: + if (not Bcfg2.Options.setup.only_bundles and + not Bcfg2.Options.setup.except_bundles): + self.logger.error("-Q option requires -b or -B") + raise SystemExit(1) + if Bcfg2.Options.setup.remove == 'services': + self.logger.error("Service removal is nonsensical; " + "removed services will only be disabled") + if not Bcfg2.Options.setup.server.startswith('https://'): + Bcfg2.Options.setup.server = \ + 'https://' + Bcfg2.Options.setup.server + + #: A dict of the state of each entry. Keys are the entries. + #: Values are boolean: True means that the entry is good, + #: False means that the entry is bad. + self.states = {} + self.whitelist = [] + self.blacklist = [] + self.removal = [] + self.unhandled = [] + self.logger = logging.getLogger(__name__) + + def _probe_failure(self, probename, msg): + """ handle failure of a probe in the way the user wants us to + (exit or continue) """ + message = "Failed to execute probe %s: %s" % (probename, msg) + if Bcfg2.Options.setup.exit_on_probe_failure: + self.fatal_error(message) + else: + self.logger.error(message) + + def run_probe(self, probe): + """Execute probe.""" + name = probe.get('name') + self.logger.info("Running probe %s" % name) + ret = XML.Element("probe-data", name=name, source=probe.get('source')) + try: + scripthandle, scriptname = tempfile.mkstemp() + script = os.fdopen(scripthandle, 'w') + try: + script.write("#!%s\n" % + (probe.attrib.get('interpreter', '/bin/sh'))) + if sys.hexversion >= 0x03000000: + script.write(probe.text) + else: + script.write(probe.text.encode('utf-8')) + script.close() + os.chmod(scriptname, + stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | + stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | + stat.S_IWUSR) # 0755 + rv = self.cmd.run(scriptname) + if rv.stderr: + self.logger.warning("Probe %s has error output: %s" % + (name, rv.stderr)) + if not rv.success: + self._probe_failure(name, "Return value %s" % rv.retval) + self.logger.info("Probe %s has result:" % name) + self.logger.info(rv.stdout) + if sys.hexversion >= 0x03000000: + ret.text = rv.stdout + else: + ret.text = rv.stdout.decode('utf-8') + finally: + os.unlink(scriptname) + except SystemExit: + raise + except: + self._probe_failure(name, sys.exc_info()[1]) + return ret + + def fatal_error(self, message): + """Signal a fatal error.""" + self.logger.error("Fatal error: %s" % (message)) + raise SystemExit(1) + + @property + def proxy(self): + """ get an XML-RPC proxy to the server """ + if self._proxy is None: + self._proxy = Proxy.ComponentProxy() + return self._proxy + + def run_probes(self): + """ run probes and upload probe data """ + try: + probes = XML.XML(str(self.proxy.GetProbes())) + except (Proxy.ProxyError, + Proxy.CertificateError, + socket.gaierror, + socket.error): + err = sys.exc_info()[1] + self.fatal_error("Failed to download probes from bcfg2: %s" % err) + except XML.ParseError: + err = sys.exc_info()[1] + self.fatal_error("Server returned invalid probe requests: %s" % + err) + + self.times['probe_download'] = time.time() + + # execute probes + probedata = XML.Element("ProbeData") + for probe in probes.findall(".//probe"): + probedata.append(self.run_probe(probe)) + + if len(probes.findall(".//probe")) > 0: + try: + # upload probe responses + self.proxy.RecvProbeData( + XML.tostring(probedata, + xml_declaration=False).decode('utf-8')) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to upload probe data: %s" % err) + + self.times['probe_upload'] = time.time() + + def get_config(self): + """ load the configuration, either from the cached + configuration file (-f), or from the server """ + if Bcfg2.Options.setup.file: + # read config from file + try: + self.logger.debug("Reading cached configuration from %s" % + Bcfg2.Options.setup.file.name) + return Bcfg2.Options.setup.file.read() + except IOError: + self.fatal_error("Failed to read cached configuration from: %s" + % Bcfg2.Options.setup.file.name) + else: + # retrieve config from server + if Bcfg2.Options.setup.profile: + try: + self.proxy.AssertProfile(Bcfg2.Options.setup.profile) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to set client profile: %s" % err) + + try: + self.proxy.DeclareVersion(__version__) + except xmlrpclib.Fault: + err = sys.exc_info()[1] + if (err.faultCode == xmlrpclib.METHOD_NOT_FOUND or + (err.faultCode == 7 and + err.faultString.startswith("Unknown method"))): + self.logger.debug("Server does not support declaring " + "client version") + else: + self.logger.error("Failed to declare version: %s" % err) + except (Proxy.ProxyError, + Proxy.CertificateError, + socket.gaierror, + socket.error): + err = sys.exc_info()[1] + self.logger.error("Failed to declare version: %s" % err) + + self.run_probes() + + if Bcfg2.Options.setup.decision in ['whitelist', 'blacklist']: + try: + # TODO: read decision list from --decision-list + Bcfg2.Options.setup.decision_list = \ + self.proxy.GetDecisionList( + Bcfg2.Options.setup.decision) + self.logger.info("Got decision list from server:") + self.logger.info(Bcfg2.Options.setup.decision_list) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to get decision list: %s" % err) + + try: + rawconfig = self.proxy.GetConfig().encode('utf-8') + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to download configuration from " + "Bcfg2: %s" % err) + + self.times['config_download'] = time.time() + + if Bcfg2.Options.setup.cache: + try: + Bcfg2.Options.setup.cache.write(rawconfig) + os.chmod(Bcfg2.Options.setup.cache, 384) # 0600 + except IOError: + self.logger.warning("Failed to write config cache file %s" % + (Bcfg2.Options.setup.cache)) + self.times['caching'] = time.time() + + return rawconfig + + def parse_config(self, rawconfig): + """ Parse the XML configuration received from the Bcfg2 server """ + try: + self.config = XML.XML(rawconfig) + except XML.ParseError: + syntax_error = sys.exc_info()[1] + self.fatal_error("The configuration could not be parsed: %s" % + syntax_error) + + self.load_tools() + + # find entries not handled by any tools + self.unhandled = [entry for struct in self.config + for entry in struct + if entry not in self.handled] + + if self.unhandled: + self.logger.error("The following entries are not handled by any " + "tool:") + for entry in self.unhandled: + self.logger.error("%s:%s:%s" % (entry.tag, entry.get('type'), + entry.get('name'))) + + # find duplicates + self.find_dups(self.config) + + pkgs = [(entry.get('name'), entry.get('origin')) + for struct in self.config + for entry in struct + if entry.tag == 'Package'] + if pkgs: + self.logger.debug("The following packages are specified in bcfg2:") + self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] is None]) + self.logger.debug("The following packages are prereqs added by " + "Packages:") + self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages']) + + self.times['config_parse'] = time.time() + + def run(self): + """Perform client execution phase.""" + # begin configuration + self.times['start'] = time.time() + + self.logger.info("Starting Bcfg2 client run at %s" % + self.times['start']) + + self.parse_config(self.get_config().decode('utf-8')) + + if self.config.tag == 'error': + self.fatal_error("Server error: %s" % (self.config.text)) + + if Bcfg2.Options.setup.bundle_quick: + newconfig = XML.XML('<Configuration/>') + for bundle in self.config.getchildren(): + name = bundle.get("name") + if (name and (name in Bcfg2.Options.setup.only_bundles or + name not in Bcfg2.Options.setup.except_bundles)): + newconfig.append(bundle) + self.config = newconfig + + if not Bcfg2.Options.setup.no_lock: + #check lock here + try: + lockfile = open(Bcfg2.Options.setup.lockfile, 'w') + if locked(lockfile.fileno()): + self.fatal_error("Another instance of Bcfg2 is running. " + "If you want to bypass the check, run " + "with the -O/--no-lock option") + except SystemExit: + raise + except: + lockfile = None + self.logger.error("Failed to open lockfile %s: %s" % + (Bcfg2.Options.setup.lockfile, + sys.exc_info()[1])) + + # execute the configuration + self.Execute() + + if not Bcfg2.Options.setup.no_lock: + # unlock here + if lockfile: + try: + fcntl.lockf(lockfile.fileno(), fcntl.LOCK_UN) + os.remove(Bcfg2.Options.setup.lockfile) + except OSError: + self.logger.error("Failed to unlock lockfile %s" % + lockfile.name) + + if (not Bcfg2.Options.setup.file and + not Bcfg2.Options.setup.bundle_quick): + # upload statistics + feedback = self.GenerateStats() + + try: + self.proxy.RecvStats( + XML.tostring(feedback, + xml_declaration=False).decode('utf-8')) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.logger.error("Failed to upload configuration statistics: " + "%s" % err) + raise SystemExit(2) + + self.logger.info("Finished Bcfg2 client run at %s" % time.time()) + + def load_tools(self): + """ Load all applicable client tools """ + for tool in Bcfg2.Options.setup.drivers: + try: + self.tools.append(tool(self.config)) + except Tools.ToolInstantiationError: + continue + except: + self.logger.error("Failed to instantiate tool %s" % tool, + exc_info=1) + + for tool in self.tools[:]: + for conflict in getattr(tool, 'conflicts', []): + for item in self.tools: + if item.name == conflict: + self.tools.remove(item) + + self.logger.info("Loaded tool drivers:") + self.logger.info([tool.name for tool in self.tools]) + + deprecated = [tool.name for tool in self.tools if tool.deprecated] + if deprecated: + self.logger.warning("Loaded deprecated tool drivers:") + self.logger.warning(deprecated) + experimental = [tool.name for tool in self.tools if tool.experimental] + if experimental: + self.logger.warning("Loaded experimental tool drivers:") + self.logger.warning(experimental) + + def find_dups(self, config): + """ Find duplicate entries and warn about them """ + entries = dict() + for struct in config: + for entry in struct: + for tool in self.tools: + if tool.handlesEntry(entry): + pkey = tool.primarykey(entry) + if pkey in entries: + entries[pkey] += 1 + else: + entries[pkey] = 1 + multi = [e for e, c in entries.items() if c > 1] + if multi: + self.logger.debug("The following entries are included multiple " + "times:") + for entry in multi: + self.logger.debug(entry) + + def promptFilter(self, msg, entries): + """Filter a supplied list based on user input.""" + ret = [] + entries.sort(key=lambda e: e.tag + ":" + e.get('name')) + for entry in entries[:]: + if entry in self.unhandled: + # don't prompt for entries that can't be installed + continue + if 'qtext' in entry.attrib: + iprompt = entry.get('qtext') + else: + iprompt = msg % (entry.tag, entry.get('name')) + if prompt(iprompt): + ret.append(entry) + return ret + + def __getattr__(self, name): + if name in ['extra', 'handled', 'modified', '__important__']: + ret = [] + for tool in self.tools: + ret += getattr(tool, name) + return ret + elif name in self.__dict__: + return self.__dict__[name] + raise AttributeError(name) + + def InstallImportant(self): + """Install important entries + + We also process the decision mode stuff here because we want to prevent + non-whitelisted/blacklisted 'important' entries from being installed + prior to determining the decision mode on the client. + """ + # Need to process decision stuff early so that dryrun mode + # works with it + self.whitelist = [entry for entry in self.states + if not self.states[entry]] + if not Bcfg2.Options.setup.file: + if Bcfg2.Options.setup.decision == 'whitelist': + dwl = Bcfg2.Options.setup.decision_list + w_to_rem = [e for e in self.whitelist + if not matches_white_list(e, dwl)] + if w_to_rem: + self.logger.info("In whitelist mode: " + "suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in w_to_rem]) + self.whitelist = [x for x in self.whitelist + if x not in w_to_rem] + elif Bcfg2.Options.setup.decision == 'blacklist': + b_to_rem = \ + [e for e in self.whitelist + if not + passes_black_list(e, Bcfg2.Options.setup.decision_list)] + if b_to_rem: + self.logger.info("In blacklist mode: " + "suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in b_to_rem]) + self.whitelist = [x for x in self.whitelist + if x not in b_to_rem] + + # take care of important entries first + if not Bcfg2.Options.setup.dry_run: + for parent in self.config.findall(".//Path/.."): + name = parent.get("name") + if (name and (name in Bcfg2.Options.setup.only_bundles or + name not in Bcfg2.Options.setup.except_bundles)): + continue + for cfile in parent.findall("./Path"): + if (cfile.get('name') not in self.__important__ or + cfile.get('type') != 'file' or + cfile not in self.whitelist): + continue + tools = [t for t in self.tools + if t.handlesEntry(cfile) and t.canVerify(cfile)] + if not tools: + continue + if (Bcfg2.Options.setup.interactive and not + self.promptFilter("Install %s: %s? (y/N):", [cfile])): + self.whitelist.remove(cfile) + continue + try: + self.states[cfile] = tools[0].InstallPath(cfile) + if self.states[cfile]: + tools[0].modified.append(cfile) + except: # pylint: disable=W0702 + self.logger.error("Unexpected tool failure", + exc_info=1) + cfile.set('qtext', '') + if tools[0].VerifyPath(cfile, []): + self.whitelist.remove(cfile) + + def Inventory(self): + """ + Verify all entries, + find extra entries, + and build up workqueues + + """ + # initialize all states + for struct in self.config.getchildren(): + for entry in struct.getchildren(): + self.states[entry] = False + for tool in self.tools: + try: + self.states.update(tool.Inventory()) + except: # pylint: disable=W0702 + self.logger.error("%s.Inventory() call failed:" % tool.name, + exc_info=1) + + def Decide(self): # pylint: disable=R0912 + """Set self.whitelist based on user interaction.""" + iprompt = "Install %s: %s? (y/N): " + rprompt = "Remove %s: %s? (y/N): " + if Bcfg2.Options.setup.remove: + if Bcfg2.Options.setup.remove == 'all': + self.removal = self.extra + elif Bcfg2.Options.setup.remove.lower() == 'services': + self.removal = [entry for entry in self.extra + if entry.tag == 'Service'] + elif Bcfg2.Options.setup.remove.lower() == 'packages': + self.removal = [entry for entry in self.extra + if entry.tag == 'Package'] + elif Bcfg2.Options.setup.remove.lower() == 'users': + self.removal = [entry for entry in self.extra + if entry.tag in ['POSIXUser', 'POSIXGroup']] + + candidates = [entry for entry in self.states + if not self.states[entry]] + + if Bcfg2.Options.setup.dry_run: + if self.whitelist: + self.logger.info("In dryrun mode: " + "suppressing entry installation for:") + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) + for entry in self.whitelist]) + self.whitelist = [] + if self.removal: + self.logger.info("In dryrun mode: " + "suppressing entry removal for:") + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) + for entry in self.removal]) + self.removal = [] + + # Here is where most of the work goes + # first perform bundle filtering + all_bundle_names = [b.get('name') + for b in self.config.findall('./Bundle')] + bundles = self.config.getchildren() + if Bcfg2.Options.setup.only_bundles: + # warn if non-existent bundle given + for bundle in Bcfg2.Options.setup.only_bundles: + if bundle not in all_bundle_names: + self.logger.info("Warning: Bundle %s not found" % bundle) + bundles = [b for b in bundles + if b.get('name') in Bcfg2.Options.setup.only_bundles] + if Bcfg2.Options.setup.except_bundles: + # warn if non-existent bundle given + if not Bcfg2.Options.setup.bundle_quick: + for bundle in Bcfg2.Options.setup.except_bundles: + if bundle not in all_bundle_names: + self.logger.info("Warning: Bundle %s not found" % + bundle) + bundles = [ + b for b in bundles + if b.get('name') not in Bcfg2.Options.setup.except_bundles] + self.whitelist = [e for e in self.whitelist + if any(e in b for b in bundles)] + + # first process prereq actions + for bundle in bundles[:]: + if bundle.tag == 'Bundle': + bmodified = any(item in self.whitelist for item in bundle) + else: + bmodified = False + actions = [a for a in bundle.findall('./Action') + if (a.get('timing') in ['pre', 'both'] and + (bmodified or a.get('when') == 'always'))] + # now we process all "always actions" + if Bcfg2.Options.setup.interactive: + self.promptFilter(iprompt, actions) + self.DispatchInstallCalls(actions) + + if bundle.tag != 'Bundle': + continue + + # need to test to fail entries in whitelist + if not all(self.states[a] for a in actions): + # then display bundles forced off with entries + self.logger.info("%s %s failed prerequisite action" % + (bundle.tag, bundle.get('name'))) + bundles.remove(bundle) + b_to_remv = [ent for ent in self.whitelist if ent in bundle] + if b_to_remv: + self.logger.info("Not installing entries from %s %s" % + (bundle.tag, bundle.get('name'))) + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in b_to_remv]) + for ent in b_to_remv: + self.whitelist.remove(ent) + + self.logger.debug("Installing entries in the following bundle(s):") + self.logger.debug(" %s" % ", ".join(b.get("name") for b in bundles + if b.get("name"))) + + if Bcfg2.Options.setup.interactive: + self.whitelist = self.promptFilter(iprompt, self.whitelist) + self.removal = self.promptFilter(rprompt, self.removal) + + for entry in candidates: + if entry not in self.whitelist: + self.blacklist.append(entry) + + def DispatchInstallCalls(self, entries): + """Dispatch install calls to underlying tools.""" + for tool in self.tools: + handled = [entry for entry in entries if tool.canInstall(entry)] + if not handled: + continue + try: + self.states.update(tool.Install(handled)) + except: # pylint: disable=W0702 + self.logger.error("%s.Install() call failed:" % tool.name, + exc_info=1) + + def Install(self): + """Install all entries.""" + self.DispatchInstallCalls(self.whitelist) + mods = self.modified + mbundles = [struct for struct in self.config.findall('Bundle') + if any(True for mod in mods if mod in struct)] + + if self.modified: + # Handle Bundle interdeps + if mbundles: + self.logger.info("The Following Bundles have been modified:") + self.logger.info([mbun.get('name') for mbun in mbundles]) + tbm = [(t, b) for t in self.tools for b in mbundles] + for tool, bundle in tbm: + try: + self.states.update(tool.Inventory(structures=[bundle])) + except: # pylint: disable=W0702 + self.logger.error("%s.Inventory() call failed:" % + tool.name, + exc_info=1) + clobbered = [entry for bundle in mbundles for entry in bundle + if (not self.states[entry] and + entry not in self.blacklist)] + if clobbered: + self.logger.debug("Found clobbered entries:") + self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) + for entry in clobbered]) + if not Bcfg2.Options.setup.interactive: + self.DispatchInstallCalls(clobbered) + + for bundle in self.config.findall('.//Bundle'): + if (Bcfg2.Options.setup.only_bundles and + bundle.get('name') not in Bcfg2.Options.setup.only_bundles): + # prune out unspecified bundles when running with -b + continue + if bundle in mbundles: + self.logger.debug("Bundle %s was modified" % + bundle.get('name')) + func = "BundleUpdated" + else: + self.logger.debug("Bundle %s was not modified" % + bundle.get('name')) + func = "BundleNotUpdated" + for tool in self.tools: + try: + self.states.update(getattr(tool, func)(bundle)) + except: # pylint: disable=W0702 + self.logger.error("%s.%s(%s:%s) call failed:" % + (tool.name, func, bundle.tag, + bundle.get("name")), exc_info=1) + + for indep in self.config.findall('.//Independent'): + for tool in self.tools: + try: + self.states.update(tool.BundleNotUpdated(indep)) + except: # pylint: disable=W0702 + self.logger.error("%s.BundleNotUpdated(%s:%s) call failed:" + % (tool.name, indep.tag, + indep.get("name")), exc_info=1) + + def Remove(self): + """Remove extra entries.""" + for tool in self.tools: + extras = [entry for entry in self.removal + if tool.handlesEntry(entry)] + if extras: + try: + tool.Remove(extras) + except: # pylint: disable=W0702 + self.logger.error("%s.Remove() failed" % tool.name, + exc_info=1) + + def CondDisplayState(self, phase): + """Conditionally print tracing information.""" + self.logger.info('Phase: %s' % phase) + self.logger.info('Correct entries: %d' % + list(self.states.values()).count(True)) + self.logger.info('Incorrect entries: %d' % + list(self.states.values()).count(False)) + if phase == 'final' and list(self.states.values()).count(False): + for entry in sorted(self.states.keys(), key=lambda e: e.tag + ":" + + e.get('name')): + if not self.states[entry]: + etype = entry.get('type') + if etype: + self.logger.info("%s:%s:%s" % (entry.tag, etype, + entry.get('name'))) + else: + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) + self.logger.info('Total managed entries: %d' % + len(list(self.states.values()))) + self.logger.info('Unmanaged entries: %d' % len(self.extra)) + if phase == 'final' and Bcfg2.Options.setup.show_extra: + for entry in sorted(self.extra, + key=lambda e: e.tag + ":" + e.get('name')): + etype = entry.get('type') + if etype: + self.logger.info("%s:%s:%s" % (entry.tag, etype, + entry.get('name'))) + else: + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) + + if ((list(self.states.values()).count(False) == 0) and not self.extra): + self.logger.info('All entries correct.') + + def ReInventory(self): + """Recheck everything.""" + if not Bcfg2.Options.setup.dry_run and Bcfg2.Options.setup.kevlar: + self.logger.info("Rechecking system inventory") + self.Inventory() + + def Execute(self): + """Run all methods.""" + self.Inventory() + self.times['inventory'] = time.time() + self.CondDisplayState('initial') + self.InstallImportant() + self.Decide() + self.Install() + self.times['install'] = time.time() + self.Remove() + self.times['remove'] = time.time() + if self.modified: + self.ReInventory() + self.times['reinventory'] = time.time() + self.times['finished'] = time.time() + self.CondDisplayState('final') + + def GenerateStats(self): + """Generate XML summary of execution statistics.""" + feedback = XML.Element("upload-statistics") + stats = XML.SubElement(feedback, + 'Statistics', total=str(len(self.states)), + version='2.0', + revision=self.config.get('revision', '-1')) + good_entries = [key for key, val in list(self.states.items()) if val] + good = len(good_entries) + stats.set('good', str(good)) + if any(not val for val in list(self.states.values())): + stats.set('state', 'dirty') + else: + stats.set('state', 'clean') + + # List bad elements of the configuration + for (data, ename) in [(self.modified, 'Modified'), + (self.extra, "Extra"), + (good_entries, "Good"), + ([entry for entry in self.states + if not self.states[entry]], "Bad")]: + container = XML.SubElement(stats, ename) + for item in data: + item.set('qtext', '') + container.append(item) + item.text = None + + timeinfo = XML.Element("OpStamps") + feedback.append(stats) + for (event, timestamp) in list(self.times.items()): + timeinfo.set(event, str(timestamp)) + stats.append(timeinfo) + return feedback diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index e537b6148..f9fd42d33 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -9,6 +9,7 @@ import socket import struct import sys import termios +import Bcfg2.Options logging.raiseExceptions = 0 @@ -150,8 +151,11 @@ def add_console_handler(level=logging.DEBUG): logging.root.addHandler(console) -def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG): +def add_syslog_handler(procname=None, syslog_facility='daemon', + level=logging.DEBUG): """Add a logging handler that logs as procname to syslog_facility.""" + if procname is None: + procname = Bcfg2.Options.get_parser().prog try: try: syslog = FragmentingSysLogHandler(procname, @@ -175,9 +179,9 @@ def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG): print("Failed to activate syslogging") -def add_file_handler(to_file, level=logging.DEBUG): - """Add a logging handler that logs to to_file.""" - filelog = logging.FileHandler(to_file) +def add_file_handler(level=logging.DEBUG): + """Add a logging handler that logs to a file.""" + filelog = logging.FileHandler(Bcfg2.Options.setup.logfile) try: filelog.set_name("file") # pylint: disable=E1101 except AttributeError: @@ -188,34 +192,128 @@ def add_file_handler(to_file, level=logging.DEBUG): logging.root.addHandler(filelog) -def setup_logging(procname, to_console=True, to_syslog=True, - syslog_facility='daemon', level=0, to_file=None): +def default_log_level(): + """ Get the default log level, according to the configuration """ + if Bcfg2.Options.setup.debug: + return logging.DEBUG + elif Bcfg2.Options.setup.verbose: + return logging.INFO + else: + return logging.WARNING + + +def setup_logging(): """Setup logging for Bcfg2 software.""" if hasattr(logging, 'already_setup'): return + level = default_log_level() params = [] + to_console = True + if hasattr(Bcfg2.Options.setup, "daemon"): + if Bcfg2.Options.setup.daemon: + to_console = False + # if a command can be daemonized, but hasn't been, then we + # assume that they're running it in the foreground and thus + # want some more output. + clvl = min(level, logging.INFO) + else: + clvl = level if to_console: - if to_console is True: - to_console = logging.WARNING - if level == 0: - clvl = to_console - else: - clvl = min(to_console, level) params.append("%s to console" % logging.getLevelName(clvl)) - add_console_handler(clvl) - if to_syslog: - if level == 0: - slvl = logging.INFO - else: - slvl = min(level, logging.INFO) + add_console_handler(level=clvl) + + if hasattr(Bcfg2.Options.setup, "syslog") and Bcfg2.Options.setup.syslog: + slvl = min(level, logging.INFO) params.append("%s to syslog" % logging.getLevelName(slvl)) - add_syslog_handler(procname, syslog_facility, level=slvl) - if to_file is not None: - params.append("%s to %s" % (logging.getLevelName(level), to_file)) - add_file_handler(to_file, level=level) + add_syslog_handler(level=slvl) + + if Bcfg2.Options.setup.logfile: + params.append("%s to %s" % (logging.getLevelName(level), + Bcfg2.Options.setup.logfile)) + add_file_handler(level=level) logging.root.setLevel(logging.DEBUG) logging.root.debug("Configured logging: %s" % "; ".join(params)) + print("Configured logging: %s" % "; ".join(params)) logging.already_setup = True + + +class Debuggable(object): + """ Mixin to add a debugging interface to an object """ + + options = [] + + #: List of names of methods to be exposed as XML-RPC functions, if + #: applicable to the child class + __rmi__ = ['toggle_debug', 'set_debug'] + + #: How exposed XML-RPC functions should be dispatched to child + #: processes. + __child_rmi__ = __rmi__[:] + + def __init__(self, name=None): + """ + :param name: The name of the logger object to get. If none is + supplied, the full name of the class (including + module) will be used. + :type name: string + """ + if name is None: + name = "%s.%s" % (self.__class__.__module__, + self.__class__.__name__) + self.debug_flag = Bcfg2.Options.setup.debug + self.logger = logging.getLogger(name) + + def set_debug(self, debug): + """ Explicitly enable or disable debugging. + + :returns: bool - The new value of the debug flag + """ + self.debug_flag = debug + return debug + + def toggle_debug(self): + """ Turn debugging output on or off. + + :returns: bool - The new value of the debug flag + """ + return self.set_debug(not self.debug_flag) + + def debug_log(self, message, flag=None): + """ Log a message at the debug level. + + :param message: The message to log + :type message: string + :param flag: Override the current debug flag with this value + :type flag: bool + :returns: None + """ + if (flag is None and self.debug_flag) or flag: + self.logger.error(message) + + +class _OptionContainer(object): + """ Container for options loaded at import-time to configure + logging """ + options = [ + Bcfg2.Options.BooleanOption( + '-d', '--debug', help='Enable debugging output', + cf=('logging', 'debug')), + Bcfg2.Options.BooleanOption( + '-v', '--verbose', help='Enable verbose output', + cf=('logging', 'verbose')), + Bcfg2.Options.PathOption( + '-o', '--logfile', help='Set path of file log', + cf=('logging', 'path'))] + + @staticmethod + def options_parsed_hook(): + """ initialize settings from /etc/bcfg2-web.conf or + /etc/bcfg2.conf, or set up basic defaults. this lets + manage.py work in all cases """ + setup_logging() + + +Bcfg2.Options.get_parser().add_component(_OptionContainer) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py deleted file mode 100644 index a96ea9a3b..000000000 --- a/src/lib/Bcfg2/Options.py +++ /dev/null @@ -1,1363 +0,0 @@ -"""Option parsing library for utilities.""" - -import copy -import getopt -import inspect -import os -import re -import shlex -import sys -import grp -import pwd -from Bcfg2.Client.Tools import __path__ as toolpath -from Bcfg2.Compat import ConfigParser, walk_packages -from Bcfg2.version import __version__ - - -class OptionFailure(Exception): - """ raised when malformed Option objects are instantiated """ - pass - -DEFAULT_CONFIG_LOCATION = '/etc/bcfg2.conf' -DEFAULT_INSTALL_PREFIX = '/usr' - - -class DefaultConfigParser(ConfigParser.ConfigParser): - """ A config parser that can be used to query options with default - values in the event that the option is not found """ - - def __init__(self, *args, **kwargs): - """Make configuration options case sensitive""" - ConfigParser.ConfigParser.__init__(self, *args, **kwargs) - self.optionxform = str - - def get(self, section, option, **kwargs): - """ convenience method for getting config items """ - default = None - if 'default' in kwargs: - default = kwargs['default'] - del kwargs['default'] - try: - return ConfigParser.ConfigParser.get(self, section, option, - **kwargs) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - if default is not None: - return default - else: - raise - - def getboolean(self, section, option, **kwargs): - """ convenience method for getting boolean config items """ - default = None - if 'default' in kwargs: - default = kwargs['default'] - del kwargs['default'] - try: - return ConfigParser.ConfigParser.getboolean(self, section, - option, **kwargs) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, - ValueError): - if default is not None: - return default - else: - raise - - -class Option(object): - """ a single option, which might be read from the command line, - environment, or config file """ - - # pylint: disable=C0103,R0913 - def __init__(self, desc, default, cmd=None, odesc=False, - env=False, cf=False, cook=False, long_arg=False, - deprecated_cf=None): - self.desc = desc - self.default = default - self.cmd = cmd - self.long = long_arg - if not self.long: - if cmd and (cmd[0] != '-' or len(cmd) != 2): - raise OptionFailure("Poorly formed command %s" % cmd) - elif cmd and not cmd.startswith('--'): - raise OptionFailure("Poorly formed command %s" % cmd) - self.odesc = odesc - self.env = env - self.cf = cf - self.deprecated_cf = deprecated_cf - self.boolean = False - if not odesc and not cook and isinstance(self.default, bool): - self.boolean = True - self.cook = cook - self.value = None - # pylint: enable=C0103,R0913 - - def get_cooked_value(self, value): - """ get the value of this option after performing any option - munging specified in the 'cook' keyword argument to the - constructor """ - if self.boolean: - return True - if self.cook: - return self.cook(value) - else: - return value - - def __str__(self): - rv = ["%s: " % self.__class__.__name__, self.desc] - if self.cmd or self.cf: - rv.append(" (") - if self.cmd: - if self.odesc: - if self.long: - rv.append("%s=%s" % (self.cmd, self.odesc)) - else: - rv.append("%s %s" % (self.cmd, self.odesc)) - else: - rv.append("%s" % self.cmd) - - if self.cf: - if self.cmd: - rv.append("; ") - rv.append("[%s].%s" % self.cf) - if self.cmd or self.cf: - rv.append(")") - if hasattr(self, "value"): - rv.append(": %s" % self.value) - return "".join(rv) - - def buildHelpMessage(self): - """ build the help message for this option """ - vals = [] - if not self.cmd: - return '' - if self.odesc: - if self.long: - vals.append("%s=%s" % (self.cmd, self.odesc)) - else: - vals.append("%s %s" % (self.cmd, self.odesc)) - else: - vals.append(self.cmd) - vals.append(self.desc) - return " %-28s %s\n" % tuple(vals) - - def buildGetopt(self): - """ build a string suitable for describing this short option - to getopt """ - gstr = '' - if self.long: - return gstr - if self.cmd: - gstr = self.cmd[1] - if self.odesc: - gstr += ':' - return gstr - - def buildLongGetopt(self): - """ build a string suitable for describing this long option to - getopt """ - if self.odesc: - return self.cmd[2:] + '=' - else: - return self.cmd[2:] - - def parse(self, opts, rawopts, configparser=None): - """ parse a single option. try parsing the data out of opts - (the results of getopt), rawopts (the raw option string), the - environment, and finally the config parser. either opts or - rawopts should be provided, but not both """ - if self.cmd and opts: - # Processing getopted data - optinfo = [opt[1] for opt in opts if opt[0] == self.cmd] - if optinfo: - if optinfo[0]: - self.value = self.get_cooked_value(optinfo[0]) - else: - self.value = True - return - if self.cmd and self.cmd in rawopts: - if self.odesc: - data = rawopts[rawopts.index(self.cmd) + 1] - else: - data = True - self.value = self.get_cooked_value(data) - return - # No command line option found - if self.env and self.env in os.environ: - self.value = self.get_cooked_value(os.environ[self.env]) - return - if self.cf and configparser: - try: - self.value = self.get_cooked_value(configparser.get(*self.cf)) - return - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - pass - if self.deprecated_cf: - try: - self.value = self.get_cooked_value( - configparser.get(*self.deprecated_cf)) - print("Warning: [%s] %s is deprecated, use [%s] %s instead" - % (self.deprecated_cf[0], self.deprecated_cf[1], - self.cf[0], self.cf[1])) - return - except (ConfigParser.NoSectionError, - ConfigParser.NoOptionError): - pass - - # Default value not cooked - self.value = self.default - - -class OptionSet(dict): - """ a set of Option objects that interfaces with getopt and - DefaultConfigParser to populate a dict of <option name>:<value> - """ - - def __init__(self, *args, **kwargs): - dict.__init__(self, *args) - self.hm = self.buildHelpMessage() # pylint: disable=C0103 - if 'configfile' in kwargs: - self.cfile = kwargs['configfile'] - else: - self.cfile = DEFAULT_CONFIG_LOCATION - if 'quiet' in kwargs: - self.quiet = kwargs['quiet'] - else: - self.quiet = False - self.cfp = DefaultConfigParser() - if len(self.cfp.read(self.cfile)) == 0 and not self.quiet: - # suppress warnings if called from bcfg2-admin init - caller = inspect.stack()[-1][1].split('/')[-1] - if caller == 'bcfg2-admin' and len(sys.argv) > 1: - if sys.argv[1] == 'init': - return - else: - print("Warning! Unable to read specified configuration file: " - "%s" % self.cfile) - - def buildGetopt(self): - """ build a short option description string suitable for use - by getopt.getopt """ - return ''.join([opt.buildGetopt() for opt in list(self.values())]) - - def buildLongGetopt(self): - """ build a list of long options suitable for use by - getopt.getopt """ - return [opt.buildLongGetopt() for opt in list(self.values()) - if opt.long] - - def buildHelpMessage(self): - """ Build the help mesage for this option set, or use self.hm - if it is set """ - if hasattr(self, 'hm'): - return self.hm - hlist = [] # list of _non-empty_ help messages - for opt in list(self.values()): - helpmsg = opt.buildHelpMessage() - if helpmsg: - hlist.append(helpmsg) - return ''.join(hlist) - - def helpExit(self, msg='', code=1): - """ print help and exit """ - if msg: - print(msg) - print("Usage:") - print(self.buildHelpMessage()) - raise SystemExit(code) - - def versionExit(self, code=0): - """ print the version of bcfg2 and exit """ - print("%s %s on Python %s" % - (os.path.basename(sys.argv[0]), - __version__, - ".".join(str(v) for v in sys.version_info[0:3]))) - raise SystemExit(code) - - def parse(self, argv, do_getopt=True): - '''Parse options from command line.''' - if VERSION not in self.values(): - self['__version__'] = VERSION - if do_getopt: - try: - opts, args = getopt.getopt(argv, self.buildGetopt(), - self.buildLongGetopt()) - except getopt.GetoptError: - err = sys.exc_info()[1] - self.helpExit(err) - if '-h' in argv: - self.helpExit('', 0) - if '--version' in argv: - self.versionExit() - self['args'] = args - for key in list(self.keys()): - if key == 'args': - continue - option = self[key] - if do_getopt: - option.parse(opts, [], configparser=self.cfp) - else: - option.parse([], argv, configparser=self.cfp) - if hasattr(option, 'value'): - val = option.value - self[key] = val - if "__version__" in self: - del self['__version__'] - - -def list_split(c_string): - """ split an option string on commas, optionally surrounded by - whitespace, returning a list """ - if c_string: - return re.split(r'\s*,\s*', c_string) - return [] - - -def colon_split(c_string): - """ split an option string on colons, returning a list """ - if c_string: - return c_string.split(r':') - return [] - - -def dict_split(c_string): - """ split an option string on commas, optionally surrounded by - whitespace and split the resulting items again on equals signs, - returning a dict """ - result = dict() - if c_string: - items = re.split(r'\s*,\s*', c_string) - for item in items: - if r'=' in item: - key, value = item.split(r'=', 1) - try: - result[key] = get_bool(value) - except ValueError: - try: - result[key] = get_int(value) - except ValueError: - result[key] = value - else: - result[item] = True - return result - - -def get_bool(val): - """ given a string value of a boolean configuration option, return - an actual bool (True or False) """ - # these values copied from ConfigParser.RawConfigParser.getboolean - # with the addition of True and False - truelist = ["1", "yes", "True", "true", "on"] - falselist = ["0", "no", "False", "false", "off"] - if val in truelist: - return True - elif val in falselist: - return False - else: - raise ValueError("Not a boolean value", val) - - -def get_int(val): - """ given a string value of an integer configuration option, - return an actual int """ - return int(val) - - -def get_timeout(val): - """ convert the timeout value into a float or None """ - if val is None: - return val - timeout = float(val) # pass ValueError up the stack - if timeout <= 0: - return None - return timeout - - -def get_size(value): - """ Given a number of bytes in a human-readable format (e.g., - '512m', '2g'), get the absolute number of bytes as an integer """ - if value == -1: - return value - mat = re.match(r'(\d+)([KkMmGg])?', value) - if not mat: - raise ValueError("Not a valid size", value) - rvalue = int(mat.group(1)) - mult = mat.group(2).lower() - if mult == 'k': - return rvalue * 1024 - elif mult == 'm': - return rvalue * 1024 * 1024 - elif mult == 'g': - return rvalue * 1024 * 1024 * 1024 - else: - return rvalue - - -def get_gid(val): - """ This takes a group name or gid and returns the corresponding - gid. """ - try: - return int(val) - except ValueError: - return int(grp.getgrnam(val)[2]) - - -def get_uid(val): - """ This takes a group name or gid and returns the corresponding - gid. """ - try: - return int(val) - except ValueError: - return int(pwd.getpwnam(val)[2]) - - -# Options accepts keyword argument list with the following values: -# default: default value for the option -# cmd: command line switch -# odesc: option description -# cf: tuple containing section/option -# cook: method for parsing option -# long_arg: (True|False) specifies whether cmd is a long argument - -# General options -CFILE = \ - Option('Specify configuration file', - default=DEFAULT_CONFIG_LOCATION, - cmd='-C', - odesc='<conffile>', - env="BCFG2_CONFIG") -LOCKFILE = \ - Option('Specify lockfile', - default='/var/lock/bcfg2.run', - odesc='<Path to lockfile>', - cf=('components', 'lockfile')) -HELP = \ - Option('Print this usage message', - default=False, - cmd='-h') -VERSION = \ - Option('Print the version and exit', - default=False, - cmd='--version', long_arg=True) -DAEMON = \ - Option("Daemonize process, storing pid", - default=None, - cmd='-D', - odesc='<pidfile>') -INSTALL_PREFIX = \ - Option('Installation location', - default=DEFAULT_INSTALL_PREFIX, - odesc='</path>', - cf=('server', 'prefix')) -SENDMAIL_PATH = \ - Option('Path to sendmail', - default='/usr/lib/sendmail', - cf=('reports', 'sendmailpath')) -INTERACTIVE = \ - Option('Run interactively, prompting the user for each change', - default=False, - cmd='-I', ) -ENCODING = \ - Option('Encoding of cfg files', - default='UTF-8', - cmd='-E', - odesc='<encoding>', - cf=('components', 'encoding')) -PARANOID_PATH = \ - Option('Specify path for paranoid file backups', - default='/var/cache/bcfg2', - odesc='<paranoid backup path>', - cf=('paranoid', 'path')) -PARANOID_MAX_COPIES = \ - Option('Specify the number of paranoid copies you want', - default=1, - odesc='<max paranoid copies>', - cf=('paranoid', 'max_copies')) -OMIT_LOCK_CHECK = \ - Option('Omit lock check', - default=False, - cmd='-O') -CORE_PROFILE = \ - Option('profile', - default=False, - cmd='-p') -SCHEMA_PATH = \ - Option('Path to XML Schema files', - default='%s/share/bcfg2/schemas' % DEFAULT_INSTALL_PREFIX, - cmd='--schema', - odesc='<schema path>', - cf=('lint', 'schema'), - long_arg=True) -INTERPRETER = \ - Option("Python interpreter to use", - default='best', - cmd="--interpreter", - odesc='<python|bpython|ipython|best>', - cf=('bcfg2-info', 'interpreter'), - long_arg=True) - -# Metadata options (mdata section) -MDATA_OWNER = \ - Option('Default Path owner', - default='root', - odesc='owner permissions', - cf=('mdata', 'owner')) -MDATA_GROUP = \ - Option('Default Path group', - default='root', - odesc='group permissions', - cf=('mdata', 'group')) -MDATA_IMPORTANT = \ - Option('Default Path priority (importance)', - default='False', - odesc='Important entries are installed first', - cf=('mdata', 'important')) -MDATA_MODE = \ - Option('Default mode for Path', - default='644', - odesc='octal file mode', - cf=('mdata', 'mode')) -MDATA_SECONTEXT = \ - Option('Default SELinux context', - default='__default__', - odesc='SELinux context', - cf=('mdata', 'secontext')) -MDATA_PARANOID = \ - Option('Default Path paranoid setting', - default='true', - odesc='Path paranoid setting', - cf=('mdata', 'paranoid')) -MDATA_SENSITIVE = \ - Option('Default Path sensitive setting', - default='false', - odesc='Path sensitive setting', - cf=('mdata', 'sensitive')) - -# Server options -SERVER_REPOSITORY = \ - Option('Server repository path', - default='/var/lib/bcfg2', - cmd='-Q', - odesc='<repository path>', - cf=('server', 'repository')) -SERVER_PLUGINS = \ - Option('Server plugin list', - # default server plugins - default=['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', 'Rules', - 'SSHbase'], - cf=('server', 'plugins'), - cook=list_split) -SERVER_FILEMONITOR = \ - Option('Server file monitor', - default='default', - odesc='File monitoring driver', - cf=('server', 'filemonitor')) -SERVER_FAM_IGNORE = \ - Option('File globs to ignore', - default=['*~', '*#', '.#*', '*.swp', '*.swpx', '.*.swx', - 'SCCS', '.svn', '4913', '.gitignore'], - cf=('server', 'ignore_files'), - cook=list_split) -SERVER_FAM_BLOCK = \ - Option('FAM blocks on startup until all events are processed', - default=False, - cook=get_bool, - cf=('server', 'fam_blocking')) -SERVER_LISTEN_ALL = \ - Option('Listen on all interfaces', - default=False, - cmd='--listen-all', - cf=('server', 'listen_all'), - cook=get_bool, - long_arg=True) -SERVER_LOCATION = \ - Option('Server Location', - default='https://localhost:6789', - cmd='-S', - odesc='https://server:port', - cf=('components', 'bcfg2')) -SERVER_KEY = \ - Option('Path to SSL key', - default="/etc/pki/tls/private/bcfg2.key", - cmd='--ssl-key', - odesc='<ssl key>', - cf=('communication', 'key'), - long_arg=True) -SERVER_CERT = \ - Option('Path to SSL certificate', - default="/etc/pki/tls/certs/bcfg2.crt", - odesc='<ssl cert>', - cf=('communication', 'certificate')) -SERVER_CA = \ - Option('Path to SSL CA Cert', - default=None, - odesc='<ca cert>', - cf=('communication', 'ca')) -SERVER_PASSWORD = \ - Option('Communication Password', - default=None, - cmd='-x', - odesc='<password>', - cf=('communication', 'password')) -SERVER_PROTOCOL = \ - Option('Server Protocol', - default='xmlrpc/ssl', - cf=('communication', 'protocol')) -SERVER_BACKEND = \ - Option('Server Backend', - default='best', - cf=('server', 'backend')) -SERVER_DAEMON_USER = \ - Option('User to run the server daemon as', - default=0, - cf=('server', 'user'), - cook=get_uid) -SERVER_DAEMON_GROUP = \ - Option('Group to run the server daemon as', - default=0, - cf=('server', 'group'), - cook=get_gid) -SERVER_VCS_ROOT = \ - Option('Server VCS repository root', - default=None, - odesc='<VCS repository root>', - cf=('server', 'vcs_root')) -SERVER_UMASK = \ - Option('Server umask', - default='0077', - odesc='<Server umask>', - cf=('server', 'umask')) -SERVER_AUTHENTICATION = \ - Option('Default client authentication method', - default='cert+password', - odesc='{cert|bootstrap|cert+password}', - cf=('communication', 'authentication')) -SERVER_CHILDREN = \ - Option('Spawn this number of children for the multiprocessing core. ' - 'By default spawns children equivalent to the number of processors ' - 'in the machine.', - default=None, - cmd='--children', - odesc='<children>', - cf=('server', 'children'), - cook=get_int, - long_arg=True) - -# database options -DB_ENGINE = \ - Option('Database engine', - default='sqlite3', - cf=('database', 'engine')) -DB_NAME = \ - Option('Database name', - default=os.path.join(SERVER_REPOSITORY.default, "etc/bcfg2.sqlite"), - cf=('database', 'name')) -DB_USER = \ - Option('Database username', - default=None, - cf=('database', 'user')) -DB_PASSWORD = \ - Option('Database password', - default=None, - cf=('database', 'password')) -DB_HOST = \ - Option('Database host', - default='localhost', - cf=('database', 'host')) -DB_PORT = \ - Option('Database port', - default='', - cf=('database', 'port'), - deprecated_cf=('statistics', 'database_port')) -DB_OPTIONS = \ - Option('Database options', - default=dict(), - cf=('database', 'options'), - cook=dict_split) -DB_SCHEMA = \ - Option('Database schema', - default='', - cf=('database', 'schema')) - -# Django options -WEB_CFILE = \ - Option('Web interface configuration file', - default="/etc/bcfg2-web.conf", - cmd='-W', - odesc='<conffile>', - cf=('reporting', 'config'), - deprecated_cf=('statistics', 'web_prefix'),) -DJANGO_TIME_ZONE = \ - Option('Django timezone', - default=None, - cf=('reporting', 'time_zone'), - deprecated_cf=('statistics', 'web_prefix'),) -DJANGO_DEBUG = \ - Option('Django debug', - default=None, - cf=('reporting', 'web_debug'), - deprecated_cf=('statistics', 'web_prefix'), - cook=get_bool,) -DJANGO_WEB_PREFIX = \ - Option('Web prefix', - default=None, - cf=('reporting', 'web_prefix')) - -# Reporting options -REPORTING_FILE_LIMIT = \ - Option('Reporting file size limit', - default=get_size('1m'), - cf=('reporting', 'file_limit'), - cook=get_size,) - -# Reporting options -REPORTING_TRANSPORT = \ - Option('Reporting transport', - default='DirectStore', - cf=('reporting', 'transport'),) - -# Client options -CLIENT_KEY = \ - Option('Path to SSL key', - default=None, - cmd='--ssl-key', - odesc='<ssl key>', - cf=('communication', 'key'), - long_arg=True) -CLIENT_CERT = \ - Option('Path to SSL certificate', - default=None, - cmd='--ssl-cert', - odesc='<ssl cert>', - cf=('communication', 'certificate'), - long_arg=True) -CLIENT_CA = \ - Option('Path to SSL CA Cert', - default=None, - cmd='--ca-cert', - odesc='<ca cert>', - cf=('communication', 'ca'), - long_arg=True) -CLIENT_SCNS = \ - Option('List of server commonNames', - default=None, - cmd='--ssl-cns', - odesc='<CN1:CN2>', - cf=('communication', 'serverCommonNames'), - cook=list_split, - long_arg=True) -CLIENT_PROFILE = \ - Option('Assert the given profile for the host', - default=None, - cmd='-p', - odesc='<profile>', - cf=('client', 'profile')) -CLIENT_RETRIES = \ - Option('The number of times to retry network communication', - default='3', - cmd='-R', - odesc='<retry count>', - cf=('communication', 'retries')) -CLIENT_RETRY_DELAY = \ - Option('The time in seconds to wait between retries', - default='1', - cmd='-y', - odesc='<retry delay>', - cf=('communication', 'retry_delay')) -CLIENT_DRYRUN = \ - Option('Do not actually change the system', - default=False, - cmd='-n') -CLIENT_EXTRA_DISPLAY = \ - Option('enable extra entry output', - default=False, - cmd='-e') -CLIENT_PARANOID = \ - Option('Make automatic backups of config files', - default=False, - cmd='-P', - cf=('client', 'paranoid'), - cook=get_bool) -CLIENT_DRIVERS = \ - Option('Specify tool driver set', - default=[m[1] for m in walk_packages(path=toolpath)], - cmd='-D', - odesc='<driver1,driver2>', - cf=('client', 'drivers'), - cook=list_split) -CLIENT_CACHE = \ - Option('Store the configuration in a file', - default=None, - cmd='-c', - odesc='<cache path>') -CLIENT_REMOVE = \ - Option('Force removal of additional configuration items', - default=None, - cmd='-r', - odesc='<entry type|all>') -CLIENT_BUNDLE = \ - Option('Only configure the given bundle(s)', - default=[], - cmd='-b', - odesc='<bundle:bundle>', - cook=colon_split) -CLIENT_SKIPBUNDLE = \ - Option('Configure everything except the given bundle(s)', - default=[], - cmd='-B', - odesc='<bundle:bundle>', - cook=colon_split) -CLIENT_BUNDLEQUICK = \ - Option('Only verify/configure the given bundle(s)', - default=False, - cmd='-Q') -CLIENT_INDEP = \ - Option('Only configure independent entries, ignore bundles', - default=False, - cmd='-z') -CLIENT_SKIPINDEP = \ - Option('Do not configure independent entries', - default=False, - cmd='-Z') -CLIENT_KEVLAR = \ - Option('Run in kevlar (bulletproof) mode', - default=False, - cmd='-k', ) -CLIENT_FILE = \ - Option('Configure from a file rather than querying the server', - default=None, - cmd='-f', - odesc='<specification path>') -CLIENT_QUICK = \ - Option('Disable some checksum verification', - default=False, - cmd='-q') -CLIENT_USER = \ - Option('The user to provide for authentication', - default='root', - cmd='-u', - odesc='<user>', - cf=('communication', 'user')) -CLIENT_SERVICE_MODE = \ - Option('Set client service mode', - default='default', - cmd='-s', - odesc='<default|disabled|build>') -CLIENT_TIMEOUT = \ - Option('Set the client XML-RPC timeout', - default=90, - cmd='-t', - odesc='<timeout>', - cf=('communication', 'timeout')) -CLIENT_DLIST = \ - Option('Run client in server decision list mode', - default='none', - cmd='-l', - odesc='<whitelist|blacklist|none>', - cf=('client', 'decision')) -CLIENT_DECISION_LIST = \ - Option('Decision List', - default=False, - cmd='--decision-list', - odesc='<file>', - long_arg=True) -CLIENT_EXIT_ON_PROBE_FAILURE = \ - Option("The client should exit if a probe fails", - default=True, - cmd='--exit-on-probe-failure', - long_arg=True, - cf=('client', 'exit_on_probe_failure'), - cook=get_bool) -CLIENT_PROBE_TIMEOUT = \ - Option("Timeout when running client probes", - default=None, - cf=('client', 'probe_timeout'), - cook=get_timeout) -CLIENT_COMMAND_TIMEOUT = \ - Option("Timeout when client runs other external commands (not probes)", - default=None, - cf=('client', 'command_timeout'), - cook=get_timeout) - -# bcfg2-test and bcfg2-lint options -TEST_NOSEOPTS = \ - Option('Options to pass to nosetests. Only honored with --children 0', - default=[], - cmd='--nose-options', - odesc='<opts>', - cf=('bcfg2_test', 'nose_options'), - cook=shlex.split, - long_arg=True) -TEST_IGNORE = \ - Option('Ignore these entries if they fail to build.', - default=[], - cmd='--ignore', - odesc='<Type>:<name>,<Type>:<name>', - cf=('bcfg2_test', 'ignore_entries'), - cook=list_split, - long_arg=True) -TEST_CHILDREN = \ - Option('Spawn this number of children for bcfg2-test (python 2.6+)', - default=0, - cmd='--children', - odesc='<children>', - cf=('bcfg2_test', 'children'), - cook=get_int, - long_arg=True) -TEST_XUNIT = \ - Option('Output an XUnit result file with --children', - default=None, - cmd='--xunit', - odesc='<xunit file>', - cf=('bcfg2_test', 'xunit'), - long_arg=True) -LINT_CONFIG = \ - Option('Specify bcfg2-lint configuration file', - default='/etc/bcfg2-lint.conf', - cmd='--lint-config', - odesc='<conffile>', - long_arg=True) -LINT_PLUGINS = \ - Option('bcfg2-lint plugin list', - default=None, # default is Bcfg2.Server.Lint.__all__ - cf=('lint', 'plugins'), - cook=list_split) -LINT_SHOW_ERRORS = \ - Option('Show error handling', - default=False, - cmd='--list-errors', - long_arg=True) -LINT_FILES_ON_STDIN = \ - Option('Operate on a list of files supplied on stdin', - default=False, - cmd='--stdin', - long_arg=True) - -# individual client tool options -CLIENT_APT_TOOLS_INSTALL_PATH = \ - Option('Apt tools install path', - default='/usr', - cf=('APT', 'install_path')) -CLIENT_APT_TOOLS_VAR_PATH = \ - Option('Apt tools var path', - default='/var', - cf=('APT', 'var_path')) -CLIENT_SYSTEM_ETC_PATH = \ - Option('System etc path', - default='/etc', - cf=('APT', 'etc_path')) -CLIENT_PORTAGE_BINPKGONLY = \ - Option('Portage binary packages only', - default=False, - cf=('Portage', 'binpkgonly'), - cook=get_bool) -CLIENT_RPM_INSTALLONLY = \ - Option('RPM install-only packages', - default=['kernel', 'kernel-bigmem', 'kernel-enterprise', - 'kernel-smp', 'kernel-modules', 'kernel-debug', - 'kernel-unsupported', 'kernel-devel', 'kernel-source', - 'kernel-default', 'kernel-largesmp-devel', - 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], - cf=('RPM', 'installonlypackages'), - cook=list_split) -CLIENT_RPM_PKG_CHECKS = \ - Option("Perform RPM package checks", - default=True, - cf=('RPM', 'pkg_checks'), - cook=get_bool) -CLIENT_RPM_PKG_VERIFY = \ - Option("Perform RPM package verify", - default=True, - cf=('RPM', 'pkg_verify'), - cook=get_bool) -CLIENT_RPM_INSTALLED_ACTION = \ - Option("RPM installed action", - default="install", - cf=('RPM', 'installed_action')) -CLIENT_RPM_ERASE_FLAGS = \ - Option("RPM erase flags", - default=["allmatches"], - cf=('RPM', 'erase_flags'), - cook=list_split) -CLIENT_RPM_VERSION_FAIL_ACTION = \ - Option("RPM version fail action", - default="upgrade", - cf=('RPM', 'version_fail_action')) -CLIENT_RPM_VERIFY_FAIL_ACTION = \ - Option("RPM verify fail action", - default="reinstall", - cf=('RPM', 'verify_fail_action')) -CLIENT_RPM_VERIFY_FLAGS = \ - Option("RPM verify flags", - default=[], - cf=('RPM', 'verify_flags'), - cook=list_split) -CLIENT_YUM_PKG_CHECKS = \ - Option("Perform YUM package checks", - default=True, - cf=('YUM', 'pkg_checks'), - cook=get_bool) -CLIENT_YUM_PKG_VERIFY = \ - Option("Perform YUM package verify", - default=True, - cf=('YUM', 'pkg_verify'), - cook=get_bool) -CLIENT_YUM_INSTALLED_ACTION = \ - Option("YUM installed action", - default="install", - cf=('YUM', 'installed_action')) -CLIENT_YUM_VERSION_FAIL_ACTION = \ - Option("YUM version fail action", - default="upgrade", - cf=('YUM', 'version_fail_action')) -CLIENT_YUM_VERIFY_FAIL_ACTION = \ - Option("YUM verify fail action", - default="reinstall", - cf=('YUM', 'verify_fail_action')) -CLIENT_YUM_VERIFY_FLAGS = \ - Option("YUM verify flags", - default=[], - cf=('YUM', 'verify_flags'), - cook=list_split) -CLIENT_POSIX_UID_WHITELIST = \ - Option("UID ranges the POSIXUsers tool will manage", - default=[], - cf=('POSIXUsers', 'uid_whitelist'), - cook=list_split) -CLIENT_POSIX_GID_WHITELIST = \ - Option("GID ranges the POSIXUsers tool will manage", - default=[], - cf=('POSIXUsers', 'gid_whitelist'), - cook=list_split) -CLIENT_POSIX_UID_BLACKLIST = \ - Option("UID ranges the POSIXUsers tool will not manage", - default=[], - cf=('POSIXUsers', 'uid_blacklist'), - cook=list_split) -CLIENT_POSIX_GID_BLACKLIST = \ - Option("GID ranges the POSIXUsers tool will not manage", - default=[], - cf=('POSIXUsers', 'gid_blacklist'), - cook=list_split) - -# Logging options -LOGGING_FILE_PATH = \ - Option('Set path of file log', - default=None, - cmd='-o', - odesc='<path>', - cf=('logging', 'path')) -LOGGING_SYSLOG = \ - Option('Log to syslog', - default=True, - cook=get_bool, - cf=('logging', 'syslog')) -DEBUG = \ - Option("Enable debugging output", - default=False, - cmd='-d', - cook=get_bool, - cf=('logging', 'debug')) -VERBOSE = \ - Option("Enable verbose output", - default=False, - cmd='-v', - cook=get_bool, - cf=('logging', 'verbose')) -LOG_PERFORMANCE = \ - Option("Periodically log performance statistics", - default=False, - cf=('logging', 'performance')) -PERFLOG_INTERVAL = \ - Option("Performance statistics logging interval in seconds", - default=300.0, - cook=get_timeout, - cf=('logging', 'performance_interval')) - -# Plugin-specific options -CFG_VALIDATION = \ - Option('Run validation on Cfg files', - default=True, - cmd='--cfg-validation', - cf=('cfg', 'validation'), - long_arg=True, - cook=get_bool) - -# bcfg2-crypt options -ENCRYPT = \ - Option('Encrypt the specified file', - default=False, - cmd='--encrypt', - long_arg=True) -DECRYPT = \ - Option('Decrypt the specified file', - default=False, - cmd='--decrypt', - long_arg=True) -CRYPT_STDOUT = \ - Option('Decrypt or encrypt the specified file to stdout', - default=False, - cmd='--stdout', - long_arg=True) -CRYPT_PASSPHRASE = \ - Option('Encryption passphrase name', - default=None, - cmd='-p', - odesc='<passphrase>') -CRYPT_XPATH = \ - Option('XPath expression to select elements to encrypt', - default=None, - cmd='--xpath', - odesc='<xpath>', - long_arg=True) -CRYPT_PROPERTIES = \ - Option('Encrypt the specified file as a Properties file', - default=False, - cmd="--properties", - long_arg=True) -CRYPT_CFG = \ - Option('Encrypt the specified file as a Cfg file', - default=False, - cmd="--cfg", - long_arg=True) -CRYPT_REMOVE = \ - Option('Remove the plaintext file after encrypting', - default=False, - cmd="--remove", - long_arg=True) - -# Option groups -CLI_COMMON_OPTIONS = dict(configfile=CFILE, - debug=DEBUG, - help=HELP, - version=VERSION, - verbose=VERBOSE, - encoding=ENCODING, - logging=LOGGING_FILE_PATH, - syslog=LOGGING_SYSLOG) - -DAEMON_COMMON_OPTIONS = dict(daemon=DAEMON, - umask=SERVER_UMASK, - listen_all=SERVER_LISTEN_ALL, - daemon_uid=SERVER_DAEMON_USER, - daemon_gid=SERVER_DAEMON_GROUP) - -SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, - plugins=SERVER_PLUGINS, - password=SERVER_PASSWORD, - filemonitor=SERVER_FILEMONITOR, - ignore=SERVER_FAM_IGNORE, - fam_blocking=SERVER_FAM_BLOCK, - location=SERVER_LOCATION, - key=SERVER_KEY, - cert=SERVER_CERT, - ca=SERVER_CA, - protocol=SERVER_PROTOCOL, - web_configfile=WEB_CFILE, - backend=SERVER_BACKEND, - vcs_root=SERVER_VCS_ROOT, - authentication=SERVER_AUTHENTICATION, - perflog=LOG_PERFORMANCE, - perflog_interval=PERFLOG_INTERVAL, - children=SERVER_CHILDREN, - client_timeout=CLIENT_TIMEOUT) - -CRYPT_OPTIONS = dict(encrypt=ENCRYPT, - decrypt=DECRYPT, - crypt_stdout=CRYPT_STDOUT, - passphrase=CRYPT_PASSPHRASE, - xpath=CRYPT_XPATH, - properties=CRYPT_PROPERTIES, - cfg=CRYPT_CFG, - remove=CRYPT_REMOVE) - -PATH_METADATA_OPTIONS = dict(owner=MDATA_OWNER, - group=MDATA_GROUP, - mode=MDATA_MODE, - secontext=MDATA_SECONTEXT, - important=MDATA_IMPORTANT, - paranoid=MDATA_PARANOID, - sensitive=MDATA_SENSITIVE) - -DRIVER_OPTIONS = \ - dict(apt_install_path=CLIENT_APT_TOOLS_INSTALL_PATH, - apt_var_path=CLIENT_APT_TOOLS_VAR_PATH, - apt_etc_path=CLIENT_SYSTEM_ETC_PATH, - portage_binpkgonly=CLIENT_PORTAGE_BINPKGONLY, - rpm_installonly=CLIENT_RPM_INSTALLONLY, - rpm_pkg_checks=CLIENT_RPM_PKG_CHECKS, - rpm_pkg_verify=CLIENT_RPM_PKG_VERIFY, - rpm_installed_action=CLIENT_RPM_INSTALLED_ACTION, - rpm_erase_flags=CLIENT_RPM_ERASE_FLAGS, - rpm_version_fail_action=CLIENT_RPM_VERSION_FAIL_ACTION, - rpm_verify_fail_action=CLIENT_RPM_VERIFY_FAIL_ACTION, - rpm_verify_flags=CLIENT_RPM_VERIFY_FLAGS, - yum_pkg_checks=CLIENT_YUM_PKG_CHECKS, - yum_pkg_verify=CLIENT_YUM_PKG_VERIFY, - yum_installed_action=CLIENT_YUM_INSTALLED_ACTION, - yum_version_fail_action=CLIENT_YUM_VERSION_FAIL_ACTION, - yum_verify_fail_action=CLIENT_YUM_VERIFY_FAIL_ACTION, - yum_verify_flags=CLIENT_YUM_VERIFY_FLAGS, - posix_uid_whitelist=CLIENT_POSIX_UID_WHITELIST, - posix_gid_whitelist=CLIENT_POSIX_GID_WHITELIST, - posix_uid_blacklist=CLIENT_POSIX_UID_BLACKLIST, - posix_gid_blacklist=CLIENT_POSIX_GID_BLACKLIST) - -CLIENT_COMMON_OPTIONS = \ - dict(extra=CLIENT_EXTRA_DISPLAY, - quick=CLIENT_QUICK, - lockfile=LOCKFILE, - drivers=CLIENT_DRIVERS, - dryrun=CLIENT_DRYRUN, - paranoid=CLIENT_PARANOID, - ppath=PARANOID_PATH, - max_copies=PARANOID_MAX_COPIES, - bundle=CLIENT_BUNDLE, - skipbundle=CLIENT_SKIPBUNDLE, - bundle_quick=CLIENT_BUNDLEQUICK, - indep=CLIENT_INDEP, - skipindep=CLIENT_SKIPINDEP, - file=CLIENT_FILE, - interactive=INTERACTIVE, - cache=CLIENT_CACHE, - profile=CLIENT_PROFILE, - remove=CLIENT_REMOVE, - server=SERVER_LOCATION, - user=CLIENT_USER, - password=SERVER_PASSWORD, - retries=CLIENT_RETRIES, - retry_delay=CLIENT_RETRY_DELAY, - kevlar=CLIENT_KEVLAR, - omit_lock_check=OMIT_LOCK_CHECK, - decision=CLIENT_DLIST, - servicemode=CLIENT_SERVICE_MODE, - key=CLIENT_KEY, - certificate=CLIENT_CERT, - ca=CLIENT_CA, - serverCN=CLIENT_SCNS, - timeout=CLIENT_TIMEOUT, - decision_list=CLIENT_DECISION_LIST, - probe_exit=CLIENT_EXIT_ON_PROBE_FAILURE, - probe_timeout=CLIENT_PROBE_TIMEOUT, - command_timeout=CLIENT_COMMAND_TIMEOUT) -CLIENT_COMMON_OPTIONS.update(DRIVER_OPTIONS) -CLIENT_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS) - -DATABASE_COMMON_OPTIONS = dict(web_configfile=WEB_CFILE, - configfile=CFILE, - db_engine=DB_ENGINE, - db_name=DB_NAME, - db_user=DB_USER, - db_password=DB_PASSWORD, - db_host=DB_HOST, - db_port=DB_PORT, - db_options=DB_OPTIONS, - db_schema=DB_SCHEMA, - time_zone=DJANGO_TIME_ZONE, - django_debug=DJANGO_DEBUG, - web_prefix=DJANGO_WEB_PREFIX) - -REPORTING_COMMON_OPTIONS = dict(reporting_file_limit=REPORTING_FILE_LIMIT, - reporting_transport=REPORTING_TRANSPORT) - -TEST_COMMON_OPTIONS = dict(noseopts=TEST_NOSEOPTS, - test_ignore=TEST_IGNORE, - children=TEST_CHILDREN, - xunit=TEST_XUNIT, - validate=CFG_VALIDATION) - -INFO_COMMON_OPTIONS = dict(ppath=PARANOID_PATH, - max_copies=PARANOID_MAX_COPIES) -INFO_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS) -INFO_COMMON_OPTIONS.update(SERVER_COMMON_OPTIONS) - - -class OptionParser(OptionSet): - """ OptionParser bootstraps option parsing, getting the value of - the config file. This should only be instantiated by - :func:`get_option_parser`, below, not by individual plugins or - scripts. """ - - def __init__(self, args, argv=None, quiet=False): - if argv is None: - argv = sys.argv[1:] - # the bootstrap is always quiet, since it's running with a - # default config file and so might produce warnings otherwise - self.bootstrap = OptionSet([('configfile', CFILE)], quiet=True) - self.bootstrap.parse(argv, do_getopt=False) - OptionSet.__init__(self, args, configfile=self.bootstrap['configfile'], - quiet=quiet) - self.optinfo = copy.copy(args) - # these will be set by parse() and then used by reparse() - self.argv = [] - self.do_getopt = True - - def reparse(self, argv=None, do_getopt=None): - """ parse the options again, taking any changes (e.g., to the - config file) into account """ - self.parse(argv=argv, do_getopt=do_getopt) - - def parse(self, argv=None, do_getopt=None): - for key, opt in self.optinfo.items(): - self[key] = opt - if "args" not in self.optinfo and "args" in self: - del self['args'] - self.argv = argv or sys.argv[1:] - if self.do_getopt is None: - if do_getopt: - self.do_getopt = do_getopt - else: - self.do_getopt = True - if do_getopt is None: - do_getopt = self.do_getopt - OptionSet.parse(self, self.argv, do_getopt=do_getopt) - - def add_option(self, name, opt): - """ Add an option to the parser """ - self[name] = opt - self.optinfo[name] = opt - - def add_options(self, options): - """ Add a set of options to the parser """ - self.update(options) - self.optinfo.update(options) - - def update(self, optdict): - dict.update(self, optdict) - self.optinfo.update(optdict) - - -#: A module-level OptionParser object that all plugins, etc., can use. -#: This should not be used directly, but retrieved via -#: :func:`get_option_parser`. -_PARSER = None - - -def load_option_parser(args, argv=None, quiet=False): - """ Load an :class:`Bcfg2.Options.OptionParser` object, caching it - in :attr:`_PARSER` for later retrieval via - :func:`get_option_parser`. - - :param args: The argument set to parse. - :type args: dict of :class:`Bcfg2.Options.Option` objects - :param argv: The command-line argument list. If this is not - provided, :attr:`sys.argv` will be used. - :type argv: list of strings - :param quiet: Be quiet when bootstrapping the argument parser. - :type quiet: bool - :returns: :class:`Bcfg2.Options.OptionParser` - """ - global _PARSER # pylint: disable=W0603 - _PARSER = OptionParser(args, argv=argv, quiet=quiet) - return _PARSER - - -def get_option_parser(): - """ Get an already-created :class:`Bcfg2.Options.OptionParser` object. If - :attr:`_PARSER` has not been populated, then a new OptionParser - will be created with basic arguments. - - :returns: :class:`Bcfg2.Options.OptionParser` - """ - if _PARSER is None: - return load_option_parser(CLI_COMMON_OPTIONS) - return _PARSER diff --git a/src/lib/Bcfg2/Options/Actions.py b/src/lib/Bcfg2/Options/Actions.py new file mode 100644 index 000000000..cb83f3ae7 --- /dev/null +++ b/src/lib/Bcfg2/Options/Actions.py @@ -0,0 +1,164 @@ +""" Custom argparse actions """ + +import sys +import argparse +from Parser import get_parser + +__all__ = ["ConfigFileAction", "ComponentAction", "PluginsAction"] + + +class ComponentAction(argparse.Action): + """ ComponentAction automatically imports classes and modules + based on the value of the option, and automatically collects + options from the loaded classes and modules. It cannot be used by + itself, but must be subclassed, with either :attr:`mapping` or + :attr:`bases` overridden. See + :class:`Bcfg2.Options.PluginsAction` for an example. + + ComponentActions expect to be given a list of class names. If + :attr:`bases` is overridden, then it will attempt to import those + classes from identically named modules within the given bases. + For instance: + + .. code-block:: python + + class FooComponentAction(Bcfg2.Options.ComponentAction): + bases = ["Bcfg2.Server.Foo"] + + + class FooLoader(object): + options = [ + Bcfg2.Options.Option( + "--foo", + type=Bcfg2.Options.Types.comma_list, + default=["One"], + action=FooComponentAction)] + + If "--foo One,Two,Three" were given on the command line, then + ``FooComponentAction`` would attempt to import + ``Bcfg2.Server.Foo.One.One``, ``Bcfg2.Server.Foo.Two.Two``, and + ``Bcfg2.Server.Foo.Three.Three``. (It would also call + :func:`Bcfg2.Options.Parser.add_component` with each of those + classes as arguments.) + + Note that, although ComponentActions expect lists of components + (by default; this can be overridden by setting :attr:`islist`), + you must still explicitly specify a ``type`` argument to the + :class:`Bcfg2.Options.Option` constructor to split the value into + a list. + + Note also that, unlike other actions, the default value of a + ComponentAction option does not need to be the actual literal + final value. (I.e., you don't have to import + ``Bcfg2.Server.Foo.One.One`` and set it as the default in the + example above; the string "One" suffices.) + """ + + #: A list of parent modules where modules or classes should be + #: imported from. + bases = [] + + #: A mapping of ``<name> => <object>`` that components will be + #: loaded from. This can be used to permit much more complex + #: behavior than just a list of :attr:`bases`. + mapping = dict() + + #: If ``module`` is True, then only the module will be loaded, not + #: a class from the module. For instance, in the example above, + #: ``FooComponentAction`` would attempt instead to import + #: ``Bcfg2.Server.Foo.One``, ``Bcfg2.Server.Foo.Two``, and + #: ``Bcfg2.Server.Foo.Three``. + module = False + + #: By default, ComponentActions expect a list of components to + #: load. If ``islist`` is False, then it will only expect a + #: single component. + islist = True + + #: If ``fail_silently`` is True, then failures to import modules + #: or classes will not be logged. This is useful when the default + #: is to import everything, some of which are expected to fail. + fail_silently = False + + def __init__(self, *args, **kwargs): + if self.mapping: + if 'choices' not in kwargs: + kwargs['choices'] = self.mapping.keys() + self._final = False + argparse.Action.__init__(self, *args, **kwargs) + + def _import(self, module, name): + try: + return getattr(__import__(module, fromlist=[name]), name) + except (AttributeError, ImportError): + if not self.fail_silently: + print("Failed to load %s from %s: %s" % + (name, module, sys.exc_info()[1])) + return None + + def _load_component(self, name): + """ Import a single class or module, adding it as a component to + the parser. + + :param name: The name of the class or module to import, without + the base prepended. + :type name: string + :returns: the imported class or module + """ + cls = None + if self.mapping and name in self.mapping: + cls = self.mapping[name] + elif "." in name: + cls = self._import(*name.rsplit(".", 1)) + else: + for base in self.bases: + if self.module: + mod = base + else: + mod = "%s.%s" % (base, name) + cls = self._import(mod, name) + if cls is not None: + break + if cls: + get_parser().add_component(cls) + else: + print("Could not load component %s" % name) + return cls + + def finalize(self, parser, namespace): + """ Finalize a default value by loading the components given + in it. This lets a default be specified with a list of + strings instead of a list of classes. """ + if not self._final: + self.__call__(parser, namespace, self.default) + + def __call__(self, parser, namespace, values, option_string=None): + if values is None: + result = None + else: + if self.islist: + result = [] + for val in values: + cls = self._load_component(val) + if cls is not None: + result.append(cls) + else: + result = self._load_component(values) + self._final = True + setattr(namespace, self.dest, values) + + +class ConfigFileAction(argparse.Action): + """ ConfigFileAction automatically loads and parses a + supplementary config file (e.g., ``bcfg2-web.conf`` or + ``bcfg2-lint.conf``). """ + + def __call__(self, parser, namespace, values, option_string=None): + get_parser().add_config_file(self.dest, values) + setattr(namespace, self.dest, values) + + +class PluginsAction(ComponentAction): + """ :class:`Bcfg2.Options.ComponentAction` subclass for loading + Bcfg2 server plugins. """ + bases = ['Bcfg2.Server.Plugins'] diff --git a/src/lib/Bcfg2/Options/Common.py b/src/lib/Bcfg2/Options/Common.py new file mode 100644 index 000000000..302be61f4 --- /dev/null +++ b/src/lib/Bcfg2/Options/Common.py @@ -0,0 +1,135 @@ +""" Common options used in multiple different contexts. """ + +import Types +from Actions import PluginsAction, ComponentAction +from Parser import repository as _repository_option +from Options import Option, PathOption, BooleanOption + +__all__ = ["Common"] + + +class classproperty(object): + """ Decorator that can be used to create read-only class + properties. """ + + def __init__(self, getter): + self.getter = getter + + def __get__(self, instance, owner): + return self.getter(owner) + + +class ReportingTransportAction(ComponentAction): + """ :class:`Bcfg2.Options.ComponentAction` that loads a single + reporting transport from :mod:`Bcfg2.Reporting.Transport`. """ + islist = False + bases = ['Bcfg2.Reporting.Transport'] + + +class ReportingStorageAction(ComponentAction): + """ :class:`Bcfg2.Options.ComponentAction` that loads a single + reporting storage driver from :mod:`Bcfg2.Reporting.Storage`. """ + islist = False + bases = ['Bcfg2.Reporting.Storage'] + + +class Common(object): + """ Common options used in multiple different contexts. """ + _plugins = None + _filemonitor = None + _reporting_storage = None + _reporting_transport = None + + @classproperty + def plugins(cls): + """ Load a list of Bcfg2 server plugins """ + if cls._plugins is None: + cls._plugins = Option( + cf=('server', 'plugins'), + type=Types.comma_list, help="Server plugin list", + action=PluginsAction, + default=['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', 'Rules', + 'SSHbase']) + return cls._plugins + + @classproperty + def filemonitor(cls): + """ Load a single Bcfg2 file monitor (from + :attr:`Bcfg2.Server.FileMonitor.available`) """ + if cls._filemonitor is None: + import Bcfg2.Server.FileMonitor + + class FileMonitorAction(ComponentAction): + islist = False + mapping = Bcfg2.Server.FileMonitor.available + + cls._filemonitor = Option( + cf=('server', 'filemonitor'), action=FileMonitorAction, + default='default', help='Server file monitoring driver') + return cls._filemonitor + + @classproperty + def reporting_storage(cls): + """ Load a Reporting storage backend """ + if cls._reporting_storage is None: + cls._reporting_storage = Option( + cf=('reporting', 'storage'), dest="reporting_storage", + help='Reporting storage engine', + action=ReportingStorageAction, default='DjangoORM') + return cls._reporting_storage + + @classproperty + def reporting_transport(cls): + """ Load a Reporting transport backend """ + if cls._reporting_transport is None: + cls._reporting_transport = Option( + cf=('reporting', 'transport'), dest="reporting_transport", + help='Reporting transport', + action=ReportingTransportAction, default='DirectStore') + return cls._reporting_transport + + #: Set the path to the Bcfg2 repository + repository = _repository_option + + #: Daemonize process, storing PID + daemon = PathOption( + '-D', '--daemon', help="Daemonize process, storing PID") + + #: Run interactively, prompting the user for each change + interactive = BooleanOption( + "-I", "--interactive", + help='Run interactively, prompting the user for each change') + + #: Log to syslog + syslog = BooleanOption( + cf=('logging', 'syslog'), help="Log to syslog") + + #: Server location + location = Option( + '-S', '--server', cf=('components', 'bcfg2'), + default='https://localhost:6789', metavar='<https://server:port>', + help="Server location") + + #: Communication password + password = Option( + '-x', '--password', cf=('communication', 'password'), + metavar='<password>', help="Communication Password") + + #: Path to SSL key + ssl_key = PathOption( + '--ssl-key', cf=('communication', 'key'), dest="key", + help='Path to SSL key', default="/etc/pki/tls/private/bcfg2.key") + + #: Path to SSL certificate + ssl_cert = PathOption( + cf=('communication', 'certificate'), dest="cert", + help='Path to SSL certificate', default="/etc/pki/tls/certs/bcfg2.crt") + + #: Path to SSL CA certificate + ssl_ca = PathOption( + cf=('communication', 'ca'), help='Path to SSL CA Cert') + + #: Default Path paranoid setting + default_paranoid = Option( + cf=('mdata', 'paranoid'), dest="default_paranoid", default='true', + choices=['true', 'false'], help='Default Path paranoid setting') diff --git a/src/lib/Bcfg2/Options/OptionGroups.py b/src/lib/Bcfg2/Options/OptionGroups.py new file mode 100644 index 000000000..d77c39878 --- /dev/null +++ b/src/lib/Bcfg2/Options/OptionGroups.py @@ -0,0 +1,209 @@ +""" Option grouping classes """ + +import re +import copy +import fnmatch +from Options import Option +from itertools import chain + +__all__ = ["OptionGroup", "ExclusiveOptionGroup", "Subparser", + "WildcardSectionGroup"] + +#: A dict that records a mapping of argparse action name (e.g., +#: "store_true") to the argparse Action class for it. See +#: :func:`_get_action_class` +_action_map = dict() + + +class OptionContainer(list): + """ Parent class of all option groups """ + + def list_options(self): + """ Get a list of all options contained in this group, + including options contained in option groups in this group, + and so on. """ + return list(chain(*[o.list_options() for o in self])) + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, list.__repr__(self)) + + def add_to_parser(self, parser): + """ Add this option group to a :class:`Bcfg2.Options.Parser` + object. """ + for opt in self: + opt.add_to_parser(parser) + + +class OptionGroup(OptionContainer): + """ Generic option group that is used only to organize options. + This uses :meth:`argparse.ArgumentParser.add_argument_group` + behind the scenes. """ + + def __init__(self, *items, **kwargs): + """ + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param title: The title of the option group + :type title: string + :param description: A longer description of the option group + :param description: string + """ + OptionContainer.__init__(self, items) + self.title = kwargs.pop('title') + self.description = kwargs.pop('description', None) + + def add_to_parser(self, parser): + group = parser.add_argument_group(self.title, self.description) + OptionContainer.add_to_parser(self, group) + + +class ExclusiveOptionGroup(OptionContainer): + """ Option group that ensures that only one argument in the group + is present. This uses + :meth:`argparse.ArgumentParser.add_mutually_exclusive_group` + behind the scenes.""" + + def __init__(self, *items, **kwargs): + """ + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param required: Exactly one argument in the group *must* be + specified. + :type required: boolean + """ + OptionContainer.__init__(self, items) + self.required = kwargs.pop('required', False) + + def add_to_parser(self, parser): + group = parser.add_mutually_exclusive_group(required=self.required) + OptionContainer.add_to_parser(self, group) + + +class Subparser(OptionContainer): + """ Option group that adds options in it to a subparser. This + uses a lot of functionality tied to `argparse Sub-commands + <http://docs.python.org/dev/library/argparse.html#sub-commands>`_. + + The subcommand string itself is stored in the + :attr:`Bcfg2.Options.setup` namespace as ``subcommand``. + + This is commonly used with :class:`Bcfg2.Options.Subcommand` + groups. + """ + + _subparsers = dict() + + def __init__(self, *items, **kwargs): + """ + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param name: The name of the subparser. Required. + :type name: string + :param help: A help message for the subparser + :param help: string + """ + self.name = kwargs.pop('name') + self.help = kwargs.pop('help', None) + OptionContainer.__init__(self, items) + + def __repr__(self): + return "%s %s(%s)" % (self.__class__.__name__, + self.name, + list.__repr__(self)) + + def add_to_parser(self, parser): + if parser not in self._subparsers: + self._subparsers[parser] = parser.add_subparsers(dest='subcommand') + subparser = self._subparsers[parser].add_parser(self.name, + help=self.help) + OptionContainer.add_to_parser(self, subparser) + + +class WildcardSectionGroup(OptionContainer, Option): + """ WildcardSectionGroups contain options that may exist in + several different sections of the config that match a glob. It + works by creating options on the fly to match the sections + described in the glob. For example, consider: + + .. code-block:: python + + options = [ + Bcfg2.Options.WildcardSectionGroup( + Bcfg2.Options.Option(cf=("myplugin:*", "number"), type=int), + Bcfg2.Options.Option(cf=("myplugin:*", "description"))] + + If the config file contained ``[myplugin:foo]`` and + ``[myplugin:bar]`` sections, then this would automagically create + options for each of those. The end result would be: + + .. code-block:: python + + >>> Bcfg2.Options.setup + Namespace(myplugin_bar_description='Bar description', myplugin_bar_number=2, myplugin_foo_description='Foo description', myplugin_foo_number=1, myplugin_sections=['myplugin:foo', 'myplugin:bar']) + + All options must have the same section glob. + + The options are stored in an automatically-generated destination + given by:: + + <prefix><section>_<destination> + + ``<destination>`` is the original `dest + <http://docs.python.org/dev/library/argparse.html#dest>`_ of the + option. ``<section>`` is the section that it's found in. + ``<prefix>`` is automatically generated from the section glob by + replacing all consecutive characters disallowed in Python variable + names into underscores. (This can be overridden with the + constructor.) + + This group stores an additional option, the sections themselves, + in an option given by ``<prefix>sections``. + """ + + #: Regex to automatically get a destination for this option + _dest_re = re.compile(r'(\A(_|[^A-Za-z])+)|((_|[^A-Za-z0-9])+)') + + def __init__(self, *items, **kwargs): + """ + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param prefix: The prefix to use for options generated by this + option group. By default this is generated + automatically from the config glob; see above + for details. + :type prefix: string + :param dest: The destination for the list of known sections + that match the glob. + :param dest: string + """ + OptionContainer.__init__(self, []) + self._section_glob = items[0].cf[0] + # get a default destination + self._prefix = kwargs.get("prefix", + self._dest_re.sub('_', self._section_glob)) + Option.__init__(self, dest=kwargs.get('dest', + self._prefix + "sections")) + self._options = items + + def list_options(self): + return [self] + OptionContainer.list_options(self) + + def from_config(self, cfp): + sections = [] + for section in cfp.sections(): + if fnmatch.fnmatch(section, self._section_glob): + sections.append(section) + newopts = [] + for opt_tmpl in self._options: + option = copy.deepcopy(opt_tmpl) + option.cf = (section, option.cf[1]) + option.dest = prefix + section + "_" + option.dest + newopts.append(option) + self.extend(newopts) + for parser in self.parsers: + parser.add_options(newopts) + return sections + + def add_to_parser(self, parser): + Option.add_to_parser(self, parser) + OptionContainer.add_to_parser(self, parser) diff --git a/src/lib/Bcfg2/Options/Options.py b/src/lib/Bcfg2/Options/Options.py new file mode 100644 index 000000000..18e5cc75d --- /dev/null +++ b/src/lib/Bcfg2/Options/Options.py @@ -0,0 +1,305 @@ +""" The base :class:`Bcfg2.Options.Option` object represents an +option. Unlike options in :mod:`argparse`, an Option object does not +need to be associated with an option parser; it exists on its own.""" + +import os +import copy +import Types +import fnmatch +import argparse +from Bcfg2.Compat import ConfigParser + + +__all__ = ["Option", "BooleanOption", "PathOption", "PositionalArgument"] + +#: A dict that records a mapping of argparse action name (e.g., +#: "store_true") to the argparse Action class for it. See +#: :func:`_get_action_class` +_action_map = dict() + + +def _get_action_class(action_name): + """ Given an argparse action name (e.g., "store_true"), get the + related :class:`argparse.Action` class. The mapping that stores + this information in :mod:`argparse` itself is unfortunately + private, so it's an implementation detail that we shouldn't depend + on. So we just instantiate a dummy parser, add a dummy argument, + and determine the class that way. """ + if (isinstance(action_name, type) and + issubclass(action_name, argparse.Action)): + return action_name + if action_name not in _action_map: + action = argparse.ArgumentParser().add_argument(action_name, + action=action_name) + _action_map[action_name] = action.__class__ + return _action_map[action_name] + + +class Option(object): + """ Representation of an option that can be specified on the + command line, as an environment variable, or in a config + file. Precedence is in that order; that is, an option specified on + the command line takes precendence over an option given by the + environment, which takes precedence over an option specified in + the config file. """ + + #: Keyword arguments that should not be passed on to the + #: :class:`argparse.ArgumentParser` constructor + _local_args = ['cf', 'env', 'man'] + + def __init__(self, *args, **kwargs): + """ See :meth:`argparse.ArgumentParser.add_argument` for a + full list of accepted parameters. + + In addition to supporting all arguments and keyword arguments + from :meth:`argparse.ArgumentParser.add_argument`, several + additional keyword arguments are allowed. + + :param cf: A tuple giving the section and option name that + this argument can be referenced as in the config + file. The option name may contain the wildcard + '*', in which case the value will be a dict of all + options matching the glob. (To use a wildcard in + the section, use a + :class:`Bcfg2.Options.WildcardSectionGroup`.) + :type cf: tuple + :param env: An environment variable that the value of this + option can be taken from. + :type env: string + :param man: A detailed description of the option that will be + used to populate automatically-generated manpages. + :type man: string + """ + #: The options by which this option can be called. + #: (Coincidentally, this is also the list of arguments that + #: will be passed to + #: :meth:`argparse.ArgumentParser.add_argument` when this + #: option is added to a parser.) As a result, ``args`` can be + #: tested to see if this argument can be given on the command + #: line at all, or if it is purely a config file option. + self.args = args + self._kwargs = kwargs + + #: The tuple giving the section and option name for this + #: option in the config file + self.cf = None + + #: The environment variable that this option can take its + #: value from + self.env = None + + #: A detailed description of this option that will be used in + #: man pages. + self.man = None + + #: A list of :class:`Bcfg2.Options.Parser` objects to which + #: this option has been added. (There will be more than one + #: parser if this option is added to a subparser, for + #: instance.) + self.parsers = [] + + #: A dict of :class:`Bcfg2.Options.Parser` -> + #: :class:`argparse.Action` that gives the actions that + #: resulted from adding this option to each parser that it was + #: added to. If this option cannot be specified on the + #: command line (i.e., it only takes its value from the config + #: file), then this will be empty. + self.actions = dict() + + self.type = self._kwargs.get("type") + self.help = self._kwargs.get("help") + self._default = self._kwargs.get("default") + for kwarg in self._local_args: + setattr(self, kwarg, self._kwargs.pop(kwarg, None)) + if self.args: + # cli option + self._dest = None + else: + action_cls = _get_action_class(self._kwargs.get('action', 'store')) + # determine the name of this option. use, in order, the + # 'name' kwarg; the option name; the environment variable + # name. + self._dest = None + if 'dest' in self._kwargs: + self._dest = self._kwargs.pop('dest') + elif self.cf is not None: + self._dest = self.cf[1] + elif self.env is not None: + self._dest = self.env + kwargs = copy.copy(self._kwargs) + kwargs.pop("action", None) + self.actions[None] = action_cls(self._dest, self._dest, **kwargs) + + def __repr__(self): + sources = [] + if self.args: + sources.extend(self.args) + if self.cf: + sources.append("%s.%s" % self.cf) + if self.env: + sources.append("$" + self.env) + spec = ["sources=%s" % sources, "default=%s" % self.default] + spec.append("%d parsers" % (len(self.parsers))) + return 'Option(%s: %s)' % (self.dest, ", ".join(spec)) + + def list_options(self): + """ List options contained in this option. This exists to + provide a consistent interface with + :class:`Bcfg2.Options.OptionGroup` """ + return [self] + + def finalize(self): + """ Finalize the default value for this option. This is used + with actions (such as :class:`Bcfg2.Options.ComponentAction`) + that allow you to specify a default in a different format than + its final storage format; this can be called after it has been + determined that the default will be used (i.e., the option is + not given on the command line or in the config file) to store + the appropriate default value in the appropriate format.""" + for parser, action in self.actions.items(): + if parser is not None: + if hasattr(action, "finalize"): + action.finalize(parser, parser.namespace) + + def from_config(self, cfp): + """ Get the value of this option from the given + :class:`ConfigParser.ConfigParser`. If it is not found in the + config file, the default is returned. (If there is no + default, None is returned.) + + :param cfp: The config parser to get the option value from + :type cfp: ConfigParser.ConfigParser + :returns: The default value + """ + if not self.cf: + return None + if '*' in self.cf[1]: + if cfp.has_section(self.cf[0]): + # build a list of known options in this section, and + # exclude them + exclude = set() + for parser in self.parsers: + exclude.update(o.cf[1] + for o in parser.option_list + if o.cf and o.cf[0] == self.cf[0]) + return dict([(o, cfp.get(self.cf[0], o)) + for o in fnmatch.filter(cfp.options(self.cf[0]), + self.cf[1]) + if o not in exclude]) + else: + return dict() + else: + try: + val = cfp.getboolean(*self.cf) + except ValueError: + val = cfp.get(*self.cf) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return None + if self.type: + return self.type(val) + else: + return val + + def default_from_config(self, cfp): + """ Set the default value of this option from the config file + or from the environment. + + :param cfp: The config parser to get the option value from + :type cfp: ConfigParser.ConfigParser + """ + if self.env and self.env in os.environ: + self.default = os.environ[self.env] + else: + val = self.from_config(cfp) + if val is not None: + self.default = val + + def _get_default(self): + return self._default + + def _set_default(self, value): + self._default = value + for action in self.actions.values(): + action.default = value + + #: The current default value of this option + default = property(_get_default, _set_default) + + def _get_dest(self): + return self._dest + + def _set_dest(self, value): + self._dest = value + for action in self.actions.values(): + action.dest = value + + #: The namespace destination of this option (see `dest + #: <http://docs.python.org/dev/library/argparse.html#dest>`_) + dest = property(_get_dest, _set_dest) + + def add_to_parser(self, parser): + """ Add this option to the given parser. + + :param parser: The parser to add the option to. + :type parser: Bcfg2.Options.Parser + :returns: argparse.Action + """ + self.parsers.append(parser) + if self.args: + # cli option + action = parser.add_argument(*self.args, **self._kwargs) + if not self._dest: + self._dest = action.dest + if self._default: + action.default = self._default + self.actions[parser] = action + # else, config file-only option + + +class PathOption(Option): + """ Shortcut for options that expect a path argument. Uses + :meth:`Bcfg2.Options.Types.path` to transform the argument into a + canonical path. + + The type of a path option can also be overridden to return an + option file-like object. For example: + + .. code-block:: python + + options = [ + Bcfg2.Options.PathOption( + "--input", type=argparse.FileType('r'), + help="The input file")] + """ + + def __init__(self, *args, **kwargs): + kwargs.setdefault('type', Types.path) + kwargs.setdefault('metavar', '<path>') + Option.__init__(self, *args, **kwargs) + + +class BooleanOption(Option): + """ Shortcut for boolean options. The default is False, but this + can easily be overridden: + + .. code-block:: python + + options = [ + Bcfg2.Options.PathOption( + "--dwim", default=True, help="Do What I Mean")] + """ + def __init__(self, *args, **kwargs): + if 'default' in kwargs and kwargs['default']: + kwargs.setdefault('action', 'store_false') + else: + kwargs.setdefault('action', 'store_true') + kwargs.setdefault('default', False) + Option.__init__(self, *args, **kwargs) + + +class PositionalArgument(Option): + """ Shortcut for positional arguments. """ + def __init__(self, *args, **kwargs): + if 'metavar' not in kwargs: + kwargs['metavar'] = '<%s>' % args[0] + Option.__init__(self, *args, **kwargs) diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py new file mode 100644 index 000000000..6414cf98e --- /dev/null +++ b/src/lib/Bcfg2/Options/Parser.py @@ -0,0 +1,282 @@ +""" The option parser """ + +import os +import sys +import argparse +from Bcfg2.version import __version__ +from Bcfg2.Compat import ConfigParser +from Options import Option, PathOption, BooleanOption + +__all__ = ["setup", "OptionParserException", "Parser", "get_parser"] + + +#: The repository option. This is specified here (and imported into +#: :module:`Bcfg2.Options.Common`) rather than vice-versa due to +#: circular imports. +repository = PathOption( + '-Q', '--repository', cf=('server', 'repository'), + default='var/lib/bcfg2', help="Server repository path") + + +#: A module-level :class:`argparse.Namespace` object that stores all +#: configuration for Bcfg2. +setup = argparse.Namespace(version=__version__, + name="Bcfg2", + uri='http://trac.mcs.anl.gov/projects/bcfg2') + + +class OptionParserException(Exception): + """ Base exception raised for generic option parser errors """ + + +class Parser(argparse.ArgumentParser): + """ The Bcfg2 option parser. Most interfaces should not need to + instantiate a parser, but should instead use + :func:`Bcfg2.Options.get_parser` to get the parser that already + exists.""" + + #: Option for specifying the path to the Bcfg2 config file + configfile = PathOption('-C', '--config', + help="Path to configuration file", + default="/etc/bcfg2.conf") + + #: Builtin options that apply to all commands + options = [configfile, + BooleanOption('--version', help="Print the version and exit"), + Option('-E', '--encoding', metavar='<encoding>', + default='UTF-8', help="Encoding of config files", + cf=('components', 'encoding'))] + + def __init__(self, **kwargs): + """ See :class:`argparse.ArgumentParser` for a full list of + accepted parameters. + + In addition to supporting all arguments and keyword arguments + from :class:`argparse.ArgumentParser`, several additional + keyword arguments are allowed. + + :param components: A list of components to add to the parser. + :type components: list + :param namespace: The namespace to store options in. Default + is :attr:`Bcfg2.Options.setup`. + :type namespace: argparse.Namespace + :param add_base_options: Whether or not to add the options in + :attr:`Bcfg2.Options.Parser.options` + to the parser. Setting this to False + is default for subparsers. Default is + True. + :type add_base_options: bool + """ + self._cfp = ConfigParser.ConfigParser() + components = kwargs.pop('components', []) + + #: The namespace options will be stored in. + self.namespace = kwargs.pop('namespace', setup) + add_base_options = kwargs.pop('add_base_options', True) + + if 'add_help' not in kwargs: + kwargs['add_help'] = add_base_options + argparse.ArgumentParser.__init__(self, **kwargs) + + #: Whether or not parsing has completed on all current options. + self.parsed = False + + #: The argument list that was parsed. + self.argv = None + + #: Components that have been added to the parser + self.components = [] + + #: Options that have been added to the parser + self.option_list = [] + self._defaults_set = [] + self._config_files = [] + if add_base_options: + self.add_component(self) + for component in components: + self.add_component(component) + + def add_options(self, options): + """ Add an explicit list of options to the parser. When + possible, prefer :func:`Bcfg2.Options.Parser.add_component` to + add a whole component instead.""" + self.parsed = False + for option in options: + if option not in self.option_list: + self.option_list.extend(option.list_options()) + option.add_to_parser(self) + + def add_component(self, component): + """ Add a component (and all of its options) to the + parser. """ + if component not in self.components: + self.components.append(component) + if hasattr(component, "options"): + self.add_options(getattr(component, "options")) + + def _set_defaults(self): + for opt in self.option_list: + if opt not in self._defaults_set: + opt.default_from_config(self._cfp) + self._defaults_set.append(opt) + + def _parse_config_options(self): + """ populate the namespace with default values for any options + that aren't already in the namespace (i.e., options without + CLI arguments) """ + for opt in self.option_list[:]: + if not opt.args and opt.dest not in self.namespace: + value = opt.default + if value: + for parser, action in opt.actions.items(): + if parser is None: + action(self, self.namespace, value) + else: + action(parser, parser.namespace, value) + else: + setattr(self.namespace, opt.dest, value) + + def _finalize(self): + for opt in self.option_list[:]: + opt.finalize() + + def _reset_namespace(self): + self.parsed = False + for attr in dir(self.namespace): + if (not attr.startswith("_") and + attr not in ['uri', 'version', 'name'] and + attr not in self.config_files): + delattr(self.namespace, attr) + + def add_config_file(self, dest, cfile): + """ Add a config file, which triggers a full reparse of all + options. """ + if dest not in self.config_files: + self._reset_namespace() + self._cfp.read([cfile]) + self._defaults_set = [] + self._set_defaults() + self._parse_config_options() + self.config_files.append(dest) + + def reparse(self, argv=None): + """ Reparse options after they have already been parsed. + + :param argv: The argument list to parse. By default, + :attr:`Bcfg2.Options.Parser.argv` is reused. + (I.e., the argument list that was initially + parsed.) :type argv: list + """ + self._reset_namespace() + self.parse(argv or self.argv) + + def parse(self, argv=None): + """ Parse options. + + :param argv: The argument list to parse. By default, + ``sys.argv[1:]`` is used. This is stored in + :attr:`Bcfg2.Options.Parser.argv` for reuse by + :func:`Bcfg2.Options.Parser.reparse`. :type + argv: list + """ + if argv is None: + argv = sys.argv[1:] + if self.parsed and self.argv == argv: + return self.namespace + self.argv = argv + + # phase 1: get and read config file + bootstrap_parser = argparse.ArgumentParser(add_help=False) + self.configfile.add_to_parser(bootstrap_parser) + bootstrap = bootstrap_parser.parse_known_args(args=self.argv)[0] + + # check whether the specified bcfg2.conf exists + if not os.path.exists(bootstrap.config): + print("Could not read %s" % bootstrap.config) + return 1 + self.add_config_file(self.configfile.dest, bootstrap.config) + + # phase 2: re-parse command line, loading additional + # components, until all components have been loaded. On each + # iteration, set defaults from config file/environment + # variables + remaining = self.argv + while not self.parsed: + self.parsed = True + self._set_defaults() + remaining = self.parse_known_args(args=remaining, + namespace=self.namespace)[1] + self._parse_config_options() + self._finalize() + + # phase 3: parse command line for real, with all components + # loaded and all options known + self._parse_config_options() + + # phase 4: fix up <repository> macros + repo = getattr(self.namespace, "repository", repository.default) + for attr in dir(self.namespace): + value = getattr(self.namespace, attr) + if not attr.startswith("_") and hasattr(value, "replace"): + setattr(self.namespace, attr, + value.replace("<repository>", repo, 1)) + + # phase 5: call post-parsing hooks + for component in self.components: + if hasattr(component, "options_parsed_hook"): + getattr(component, "options_parsed_hook")() + + return self.namespace + + +#: A module-level :class:`Bcfg2.Options.Parser` object that is used +#: for all parsing +_parser = Parser() + +#: Track whether or not the module-level parser has been initialized +#: yet. We track this separately because some things (e.g., modules +#: that add components on import) will use the parser before it has +#: been initialized, so we can't just set +#: :attr:`Bcfg2.Options._parser` to None and wait for +#: :func:`Bcfg2.Options.get_parser` to be called. +_parser_initialized = False + + +def get_parser(description=None, components=None, namespace=None): + """ Get an existing :class:`Bcfg2.Options.Parser` object. (One is + created at the module level when :mod:`Bcfg2.Options` is + imported.) If no arguments are given, then the existing parser is + simply fetched. + + If arguments are given, then one of two things happens: + + * If this is the first ``get_parser`` call with arguments, then + the values given are set accordingly in the parser, and it is + returned. + * If this is not the first such call, then + :class:`Bcfg2.Options.OptionParserException` is raised. + + That is, a ``get_parser`` call with options is considered to + initialize the parser that already exists, and that can only + happen once. + + :param description: Set the parser description + :type description: string + :param components: Load the given components in the parser + :type components: list + :param namespace: Use the given namespace instead of + :attr:`Bcfg2.Options.setup` + :type namespace: argparse.Namespace + :returns: Bcfg2.Options.Parser object + """ + if _parser_initialized and (description or components or namespace): + raise OptionParserException("Parser has already been initialized") + elif (description or components or namespace): + if description: + _parser.description = description + if components is not None: + for component in components: + _parser.add_component(component) + if namespace: + _parser.namespace = namespace + return _parser diff --git a/src/lib/Bcfg2/Options/Subcommands.py b/src/lib/Bcfg2/Options/Subcommands.py new file mode 100644 index 000000000..53c4e563f --- /dev/null +++ b/src/lib/Bcfg2/Options/Subcommands.py @@ -0,0 +1,237 @@ +""" Classes to make it easier to create commands with large numbers of +subcommands (e.g., bcfg2-admin, bcfg2-info). """ + +import re +import cmd +import sys +import copy +import shlex +import logging +from Bcfg2.Compat import StringIO +from OptionGroups import Subparser +from Options import PositionalArgument +from Parser import Parser, setup as master_setup + + +__all__ = ["Subcommand", "HelpCommand", "CommandRegistry", "register_commands"] + + +class Subcommand(object): + """ Base class for subcommands. This must be subclassed to create + commands. + + Specifically, you must override + :func:`Bcfg2.Options.Subcommand.run`. You may want to override: + + * The docstring, which will be used as the short help. + * :attr:`Bcfg2.Options.Subcommand.options` + * :attr:`Bcfg2.Options.Subcommand.help` + * :attr:`Bcfg2.Options.Subcommand.interactive` + * + * :func:`Bcfg2.Options.Subcommand.shutdown` + + You should not need to override + :func:`Bcfg2.Options.Subcommand.__call__` or + :func:`Bcfg2.Options.Subcommand.usage`. + + A ``Subcommand`` subclass constructor must not take any arguments. + """ + + #: Options this command takes + options = [] + + #: Longer help message + help = None + + #: Whether or not to expose this command in an interactive + #: :class:`cmd.Cmd` shell, if one is used. (``bcfg2-info`` uses + #: one, ``bcfg2-admin`` does not.) + interactive = True + + _ws_re = re.compile(r'\s+', flags=re.MULTILINE) + + def __init__(self): + self.core = None + description = "%s: %s" % (self.__class__.__name__.lower(), + self.__class__.__doc__) + + #: The :class:`Bcfg2.Options.Parser` that will be used to + #: parse options if this subcommand is called from an + #: interactive :class:`cmd.Cmd` shell. + self.parser = Parser( + prog=self.__class__.__name__.lower(), + description=description, + components=[self], + add_base_options=False, + epilog=self.help) + self._usage = None + + #: A :class:`logging.Logger` that can be used to produce + #: logging output for this command. + self.logger = logging.getLogger(self.__class__.__name__.lower()) + + def __call__(self, args=None): + """ Perform option parsing and other tasks necessary to + support running ``Subcommand`` objects as part of a + :class:`cmd.Cmd` shell. You should not need to override + ``__call__``. + + :param args: Arguments given in the interactive shell + :type args: list of strings + :returns: The return value of :func:`Bcfg2.Options.Subcommand.run` + """ + if args is not None: + self.parser.namespace = copy.copy(master_setup) + alist = shlex.split(args) + try: + setup = self.parser.parse(alist) + except SystemExit: + return sys.exc_info()[1].code + return self.run(setup) + else: + return self.run(master_setup) + + def usage(self): + """ Get the short usage message. """ + if self._usage is None: + io = StringIO() + self.parser.print_usage(file=io) + usage = self._ws_re.sub(' ', io.getvalue()).strip()[7:] + doc = self._ws_re.sub(' ', getattr(self, "__doc__")).strip() + if doc is None: + self._usage = usage + else: + self._usage = "%s - %s" % (usage, doc) + return self._usage + + def run(self, setup): + """ Run the command. + + :param setup: A namespace giving the options for this command. + This must be used instead of + :attr:`Bcfg2.Options.setup` because this command + may have been called from an interactive + :class:`cmd.Cmd` shell, and thus has its own + option parser and its own (private) namespace. + ``setup`` is guaranteed to contain all of the + options in the global + :attr:`Bcfg2.Options.setup` namespace, in + addition to any local options given to this + command from the interactive shell. + :type setup: argparse.Namespace + """ + raise NotImplementedError + + def shutdown(self): + """ Perform any necessary shtudown tasks for this command This + is called to when the program exits (*not* when this command + is finished executing). """ + pass + + +class HelpCommand(Subcommand): + """ Get help on a specific subcommand. This must be subclassed to + create the actual help command by overriding + :func:`Bcfg2.Options.HelpCommand.command_registry` and giving the + command access to a :class:`Bcfg2.Options.CommandRegistry`. """ + options = [PositionalArgument("command", nargs='?')] + + # the interactive shell has its own help + interactive = False + + def command_registry(self): + """ Return a :class:`Bcfg2.Options.CommandRegistry` class. + All commands registered with the class will be included in the + help message. """ + raise NotImplementedError + + def run(self, setup): + commands = self.command_registry() + if setup.command: + try: + commands[setup.command].parser.print_help() + return 0 + except KeyError: + print("No such command: %s" % setup.command) + for command in sorted(commands.keys()): + print(commands[command].usage()) + + +class CommandRegistry(object): + """ A ``CommandRegistry`` is used to register subcommands and + provides a single interface to run them. It's also used by + :class:`Bcfg2.Options.HelpCommand` to produce help messages for + all available commands. """ + + #: A dict of registered commands. Keys are the class names, + #: lowercased (i.e., the command names), and values are instances + #: of the command objects. + commands = dict() + + options = [] + + def runcommand(self): + """ Run the single command named in + ``Bcfg2.Options.setup.subcommand``, which is where + :class:`Bcfg2.Options.Subparser` groups store the + subcommand. """ + try: + return self.commands[master_setup.subcommand].run(master_setup) + finally: + self.shutdown() + + def shutdown(self): + """ Perform shutdown tasks. This calls the ``shutdown`` + method of all registered subcommands. """ + self.commands[master_setup.subcommand].shutdown() + + @classmethod + def register_command(cls, cmdcls): + """ Register a single command. + + :param cmdcls: The command class to register + :type cmdcls: type + :returns: An instance of ``cmdcls`` + """ + cmd_obj = cmdcls() + name = cmdcls.__name__.lower() + cls.commands[name] = cmd_obj + cls.options.append( + Subparser(*cmdcls.options, name=name, help=cmdcls.__doc__)) + if issubclass(cls, cmd.Cmd) and cmdcls.interactive: + setattr(cls, "do_%s" % name, cmd_obj) + setattr(cls, "help_%s" % name, cmd_obj.parser.print_help) + return cmd_obj + + +def register_commands(registry, candidates, parent=Subcommand): + """ Register all subcommands in ``candidates`` against the + :class:`Bcfg2.Options.CommandRegistry` subclass given in + ``registry``. A command is registered if and only if: + + * It is a subclass of the given ``parent`` (by default, + :class:`Bcfg2.Options.Subcommand`); + * It is not the parent class itself; and + * Its name does not start with an underscore. + + :param registry: The :class:`Bcfg2.Options.CommandRegistry` + subclass against which commands will be + registered. + :type registry: Bcfg2.Options.CommandRegistry + :param candidates: A list of objects that will be considered for + registration. Only objects that meet the + criteria listed above will be registered. + :type candidates: list + :param parent: Specify a parent class other than + :class:`Bcfg2.Options.Subcommand` that all + registered commands must subclass. + :type parent: type + """ + for attr in candidates: + try: + if (issubclass(attr, parent) and + attr != parent and + not attr.__name__.startswith("_")): + registry.register_command(attr) + except TypeError: + pass diff --git a/src/lib/Bcfg2/Options/Types.py b/src/lib/Bcfg2/Options/Types.py new file mode 100644 index 000000000..5769d674a --- /dev/null +++ b/src/lib/Bcfg2/Options/Types.py @@ -0,0 +1,109 @@ +""" :mod:`Bcfg2.Options` provides a number of useful types for use +with the :class:`Bcfg2.Options.Option` constructor. """ + +import os +import re +import pwd +import grp + +_COMMA_SPLIT_RE = re.compile(r'\s*,\s*') + + +def path(value): + """ A generic path. ``~`` will be expanded with + :func:`os.path.expanduser` and the absolute resulting path will be + used. This does *not* ensure that the path exists. """ + return os.path.abspath(os.path.expanduser(value)) + + +def comma_list(value): + """ Split a comma-delimited list, with optional whitespace around + the commas.""" + return _COMMA_SPLIT_RE.split(value) + + +def colon_list(value): + """ Split a colon-delimited list. Whitespace is not allowed + around the colons. """ + return value.split(':') + + +def comma_dict(value): + """ Split an option string on commas, optionally surrounded by + whitespace, and split the resulting items again on equals signs, + returning a dict """ + result = dict() + if value: + items = comma_list(value) + for item in items: + if '=' in item: + key, value = item.split(r'=', 1) + try: + result[key] = bool(value) + except ValueError: + try: + result[key] = int(value) + except ValueError: + result[key] = value + else: + result[item] = True + return result + + +def octal(value): + """ Given an octal string, get an integer representation. """ + return int(value, 8) + + +def username(value): + """ Given a username or numeric UID, get a numeric UID. The user + must exist.""" + try: + return int(value) + except ValueError: + return int(pwd.getpwnam(value)[2]) + + +def groupname(value): + """ Given a group name or numeric GID, get a numeric GID. The + user must exist.""" + try: + return int(value) + except ValueError: + return int(grp.getgrnam(value)[2]) + + +def timeout(value): + """ Convert the value into a float or None. """ + if value is None: + return value + rv = float(value) # pass ValueError up the stack + if rv <= 0: + return None + return rv + + +_bytes_multipliers = dict(k=1, + m=2, + g=3, + t=4) +_suffixes = "".join(_bytes_multipliers.keys()).lower() +_suffixes += _suffixes.upper() +_bytes_re = re.compile(r'(?P<value>\d+)(?P<multiplier>[%s])?' % _suffixes) + + +def size(value): + """ Given a number of bytes in a human-readable format (e.g., + '512m', '2g'), get the absolute number of bytes as an integer. + """ + if value == -1: + return value + mat = _bytes_re.match(value) + if not mat: + raise ValueError("Not a valid size", value) + rvalue = int(mat.group("value")) + mult = mat.group("multiplier") + if mult: + return rvalue * (1024 ** _bytes_multipliers[mult.lower()]) + else: + return rvalue diff --git a/src/lib/Bcfg2/Options/__init__.py b/src/lib/Bcfg2/Options/__init__.py new file mode 100644 index 000000000..546068f1f --- /dev/null +++ b/src/lib/Bcfg2/Options/__init__.py @@ -0,0 +1,10 @@ +""" Bcfg2 options parsing. """ + +# pylint: disable=W0611,W0401,W0403 +import Types +from Common import * +from Parser import * +from Actions import * +from Options import * +from Subcommands import * +from OptionGroups import * diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 3d224432e..a1e6025e3 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -1,8 +1,8 @@ +import sys import atexit import daemon import logging import time -import traceback import threading # pylint: disable=E0611 @@ -14,52 +14,53 @@ except ImportError: # pylint: enable=E0611 import Bcfg2.Logger -from Bcfg2.Reporting.Transport import load_transport_from_config, \ - TransportError, TransportImportError +import Bcfg2.Options +from Bcfg2.Reporting.Transport.base import TransportError from Bcfg2.Reporting.Transport.DirectStore import DirectStore -from Bcfg2.Reporting.Storage import load_storage_from_config, \ - StorageError, StorageImportError +from Bcfg2.Reporting.Storage.base import StorageError + class ReportingError(Exception): """Generic reporting exception""" pass + class ReportingCollector(object): """The collecting process for reports""" + options = [Bcfg2.Options.Common.reporting_storage, + Bcfg2.Options.Common.reporting_transport, + Bcfg2.Options.Common.daemon] - def __init__(self, setup): - """Setup the collector. This may be called by the daemon or though + def __init__(self): + """Setup the collector. This may be called by the daemon or though bcfg2-admin""" - self.setup = setup - self.datastore = setup['repo'] - self.encoding = setup['encoding'] self.terminate = None self.context = None - if setup['debug']: + if Bcfg2.Options.setup.debug: level = logging.DEBUG - elif setup['verbose']: + elif Bcfg2.Options.setup.verbose: level = logging.INFO else: level = logging.WARNING Bcfg2.Logger.setup_logging('bcfg2-report-collector', to_console=logging.INFO, - to_syslog=setup['syslog'], - to_file=setup['logging'], + to_syslog=Bcfg2.Options.setup.syslog, + to_file=Bcfg2.Options.setup.logging, level=level) self.logger = logging.getLogger('bcfg2-report-collector') try: - self.transport = load_transport_from_config(setup) - self.storage = load_storage_from_config(setup) + self.transport = Bcfg2.Options.setup.transport() + self.storage = Bcfg2.Options.setup.reporting_storage() except TransportError: self.logger.error("Failed to load transport: %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) raise ReportingError except StorageError: self.logger.error("Failed to load storage: %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) raise ReportingError if isinstance(self.transport, DirectStore): @@ -69,14 +70,13 @@ class ReportingCollector(object): raise ReportingError try: - self.logger.debug("Validating storage %s" % - self.storage.__class__.__name__) + self.logger.debug("Validating storage %s" % + self.storage.__class__.__name__) self.storage.validate() except: self.logger.error("Storage backed %s failed to validate: %s" % - (self.storage.__class__.__name__, - traceback.format_exc().splitlines()[-1])) - + (self.storage.__class__.__name__, + sys.exc_info()[1])) def run(self): """Startup the processing and go!""" @@ -84,14 +84,14 @@ class ReportingCollector(object): atexit.register(self.shutdown) self.context = daemon.DaemonContext() - if self.setup['daemon']: + if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") try: - self.context.pidfile = PIDLockFile(self.setup['daemon']) + self.context.pidfile = PIDLockFile(Bcfg2.Options.setup.daemon) self.context.open() except PIDFileError: self.logger.error("Error writing pid file: %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) self.shutdown() return self.logger.info("Starting daemon") @@ -107,8 +107,8 @@ class ReportingCollector(object): start = time.time() self.storage.import_interaction(interaction) self.logger.info("Imported interaction for %s in %ss" % - (interaction.get('hostname', '<unknown>'), - time.time() - start)) + (interaction.get('hostname', '<unknown>'), + time.time() - start)) except: #TODO requeue? raise @@ -117,7 +117,7 @@ class ReportingCollector(object): self.shutdown() except: self.logger.error("Unhandled exception in main loop %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) def shutdown(self): """Cleanup and go""" diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index aea5e9d4b..9505682a7 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -11,6 +11,7 @@ from time import strptime os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' from Bcfg2 import settings +import Bcfg2.Options from Bcfg2.Compat import md5 from Bcfg2.Reporting.Storage.base import StorageBase, StorageError from Bcfg2.Server.Plugin.exceptions import PluginExecutionError @@ -27,9 +28,13 @@ from Bcfg2.Reporting.models import * class DjangoORM(StorageBase): - def __init__(self, setup): - super(DjangoORM, self).__init__(setup) - self.size_limit = setup.get('reporting_file_limit') + options = StorageBase.options + [ + Bcfg2.Options.Common.repository, + Bcfg2.Options.Option( + cf=('reporting', 'file_limit'), + type=Bcfg2.Options.Types.size, + help='Reporting file size limit', + default=1024 * 1024)] def _import_default(self, entry, state, entrytype=None, defaults=None, mapping=None, boolean=None, xforms=None): @@ -184,7 +189,7 @@ class DjangoORM(StorageBase): act_dict['detail_type'] = PathEntry.DETAIL_DIFF cdata = entry.get('current_bdiff') if cdata: - if len(cdata) > self.size_limit: + if len(cdata) > Bcfg2.Options.setup.file_limit: act_dict['detail_type'] = PathEntry.DETAIL_SIZE_LIMIT act_dict['details'] = md5(cdata).hexdigest() else: @@ -364,31 +369,31 @@ class DjangoORM(StorageBase): def import_interaction(self, interaction): """Import the data into the backend""" - try: self._import_interaction(interaction) except: self.logger.error("Failed to import interaction: %s" % - traceback.format_exc().splitlines()[-1]) + sys.exc_info()[1]) def validate(self): """Validate backend storage. Should be called once when loaded""" - settings.read_config(repo=self.setup['repo']) + settings.read_config(repo=Bcfg2.Options.setup.repository) # verify our database schema try: - if self.setup['debug']: + if Bcfg2.Options.setup.debug: vrb = 2 - elif self.setup['verbose']: + elif Bcfg2.Options.setup.verbose: vrb = 1 else: vrb = 0 management.call_command("syncdb", verbosity=vrb, interactive=False) - management.call_command("migrate", verbosity=vrb, interactive=False) + management.call_command("migrate", verbosity=vrb, + interactive=False) except: - self.logger.error("Failed to update database schema: %s" % \ - traceback.format_exc().splitlines()[-1]) + self.logger.error("Failed to update database schema: %s" % + sys.exc_info()[1]) raise StorageError def GetExtra(self, client): @@ -451,4 +456,3 @@ class DjangoORM(StorageBase): else: ret.append(None) return ret - diff --git a/src/lib/Bcfg2/Reporting/Storage/__init__.py b/src/lib/Bcfg2/Reporting/Storage/__init__.py index 85356fcfe..953104d4b 100644 --- a/src/lib/Bcfg2/Reporting/Storage/__init__.py +++ b/src/lib/Bcfg2/Reporting/Storage/__init__.py @@ -1,32 +1,3 @@ """ Public storage routines """ - -import traceback - -from Bcfg2.Reporting.Storage.base import StorageError, \ - StorageImportError - -def load_storage(storage_name, setup): - """ - Try to load the storage. Raise StorageImportError on failure - """ - try: - mod_name = "%s.%s" % (__name__, storage_name) - mod = getattr(__import__(mod_name).Reporting.Storage, storage_name) - except ImportError: - try: - mod = __import__(storage_name) - except: - raise StorageImportError("Unavailable") - try: - cls = getattr(mod, storage_name) - return cls(setup) - except: - raise StorageImportError("Storage unavailable: %s" % - traceback.format_exc().splitlines()[-1]) - -def load_storage_from_config(setup): - """Load the storage in the config... eventually""" - return load_storage('DjangoORM', setup) - diff --git a/src/lib/Bcfg2/Reporting/Storage/base.py b/src/lib/Bcfg2/Reporting/Storage/base.py index 92cc3a68b..771f755a1 100644 --- a/src/lib/Bcfg2/Reporting/Storage/base.py +++ b/src/lib/Bcfg2/Reporting/Storage/base.py @@ -2,28 +2,25 @@ The base for all Storage backends """ -import logging +import logging + class StorageError(Exception): """Generic StorageError""" pass -class StorageImportError(StorageError): - """Raised when a storage module fails to import""" - pass - class StorageBase(object): """The base for all storages""" + options = [] + __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] - def __init__(self, setup): + def __init__(self): """Do something here""" clsname = self.__class__.__name__ self.logger = logging.getLogger(clsname) self.logger.debug("Loading %s storage" % clsname) - self.setup = setup - self.encoding = setup['encoding'] def import_interaction(self, interaction): """Import the data into the backend""" @@ -48,4 +45,3 @@ class StorageBase(object): def GetCurrentEntry(self, client, e_type, e_name): """Get the current status of an entry on the client""" raise NotImplementedError - diff --git a/src/lib/Bcfg2/Reporting/Transport/DirectStore.py b/src/lib/Bcfg2/Reporting/Transport/DirectStore.py index 79d1b5aba..b9d17212e 100644 --- a/src/lib/Bcfg2/Reporting/Transport/DirectStore.py +++ b/src/lib/Bcfg2/Reporting/Transport/DirectStore.py @@ -5,18 +5,20 @@ import os import sys import time import threading +import Bcfg2.Options from Bcfg2.Reporting.Transport.base import TransportBase, TransportError -from Bcfg2.Reporting.Storage import load_storage_from_config from Bcfg2.Compat import Queue, Full, Empty, cPickle class DirectStore(TransportBase, threading.Thread): - def __init__(self, setup): - TransportBase.__init__(self, setup) + options = TransportBase.options + [Bcfg2.Options.Common.reporting_storage] + + def __init__(self): + TransportBase.__init__(self) threading.Thread.__init__(self) self.save_file = os.path.join(self.data, ".saved") - self.storage = load_storage_from_config(setup) + self.storage = Bcfg2.Options.setup.reporting_storage() self.storage.validate() self.queue = Queue(100000) @@ -30,10 +32,9 @@ class DirectStore(TransportBase, threading.Thread): def store(self, hostname, metadata, stats): try: - self.queue.put_nowait(dict( - hostname=hostname, - metadata=metadata, - stats=stats)) + self.queue.put_nowait(dict(hostname=hostname, + metadata=metadata, + stats=stats)) except Full: self.logger.warning("Reporting: Queue is full, " "dropping statistics") diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py index c7d5c512a..d901ded56 100644 --- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -9,6 +9,8 @@ import os import select import time import traceback +import Bcfg2.Options +import Bcfg2.CommonOptions import Bcfg2.Server.FileMonitor from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError from Bcfg2.Reporting.Transport.base import TransportBase, TransportError @@ -16,8 +18,10 @@ from Bcfg2.Compat import cPickle class LocalFilesystem(TransportBase): - def __init__(self, setup): - super(LocalFilesystem, self).__init__(setup) + options = TransportBase.options + [Bcfg2.Options.Common.filemonitor] + + def __init__(self): + super(LocalFilesystem, self).__init__() self.work_path = "%s/work" % self.data self.debug_log("LocalFilesystem: work path %s" % self.work_path) @@ -42,24 +46,16 @@ class LocalFilesystem(TransportBase): def start_monitor(self, collector): """Start the file monitor. Most of this comes from BaseCore""" - setup = self.setup - try: - fmon = Bcfg2.Server.FileMonitor.available[setup['filemonitor']] - except KeyError: - self.logger.error("File monitor driver %s not available; " - "forcing to default" % setup['filemonitor']) - fmon = Bcfg2.Server.FileMonitor.available['default'] - if self.debug_flag: - self.fmon.set_debug(self.debug_flag) try: - self.fmon = fmon(debug=self.debug_flag) - self.logger.info("Using the %s file monitor" % - self.fmon.__class__.__name__) + self.fmon = Bcfg2.Server.FileMonitor.get_fam() except IOError: - msg = "Failed to instantiate file monitor %s" % \ - setup['filemonitor'] + msg = "Failed to instantiate fam driver %s" % \ + Bcfg2.Options.setup.filemonitor self.logger.error(msg, exc_info=1) raise TransportError(msg) + + if self.debug_flag: + self.fmon.set_debug(self.debug_flag) self.fmon.start() self.fmon.AddMonitor(self.work_path, self) @@ -154,7 +150,7 @@ class LocalFilesystem(TransportBase): """ try: if not self._phony_collector: - self._phony_collector = ReportingCollector(self.setup) + self._phony_collector = ReportingCollector() except ReportingError: raise TransportError except: @@ -176,4 +172,3 @@ class LocalFilesystem(TransportBase): self.logger.error("RPC method %s failed: %s" % (method, traceback.format_exc().splitlines()[-1])) raise TransportError - diff --git a/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py b/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py index 22d9af57e..7427c2e1d 100644 --- a/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py +++ b/src/lib/Bcfg2/Reporting/Transport/RedisTransport.py @@ -9,9 +9,9 @@ import signal import platform import traceback import threading +import Bcfg2.Options from Bcfg2.Reporting.Transport.base import TransportBase, TransportError from Bcfg2.Compat import cPickle -from Bcfg2.Options import Option try: import redis @@ -34,9 +34,19 @@ class RedisTransport(TransportBase): STATS_KEY = 'bcfg2_statistics' COMMAND_KEY = 'bcfg2_command' - def __init__(self, setup): - super(RedisTransport, self).__init__(setup) - self._redis = None + options = TransportBase.options + [ + Bcfg2.Options.Option( + cf=('reporting', 'redis_host'), dest="reporting_redis_host", + default='127.0.0.1', help='Reporting Redis host'), + Bcfg2.Options.Option( + cf=('reporting', 'redis_port'), dest="reporting_redis_port", + default=6379, type=int, help='Reporting Redis port'), + Bcfg2.Options.Option( + cf=('reporting', 'redis_db'), dest="reporting_redis_db", + default=0, type=int, help='Reporting Redis DB')] + + def __init__(self): + super(RedisTransport, self).__init__() self._commands = None self.logger.error("Warning: RedisTransport is experimental") @@ -45,36 +55,15 @@ class RedisTransport(TransportBase): self.logger.error("redis python module is not available") raise TransportError - setup.update(dict( - reporting_redis_host=Option( - 'Redis Host', - default='127.0.0.1', - cf=('reporting', 'redis_host')), - reporting_redis_port=Option( - 'Redis Port', - default=6379, - cf=('reporting', 'redis_port')), - reporting_redis_db=Option( - 'Redis DB', - default=0, - cf=('reporting', 'redis_db')), - )) - setup.reparse() - - self._redis_host = setup.get('reporting_redis_host', '127.0.0.1') - try: - self._redis_port = int(setup.get('reporting_redis_port', 6379)) - except ValueError: - self.logger.error("Redis port must be an integer") - raise TransportError - self._redis_db = setup.get('reporting_redis_db', 0) - self._redis = redis.Redis(host=self._redis_host, - port=self._redis_port, db=self._redis_db) + self._redis = redis.Redis( + host=Bcfg2.Options.setup.reporting_redis_host, + port=Bcfg2.Options.setup.reporting_redis_port, + db=Bcfg2.Options.setup.reporting_redis_db) def start_monitor(self, collector): """Start the monitor. Eventaully start the command thread""" - self._commands = threading.Thread(target=self.monitor_thread, + self._commands = threading.Thread(target=self.monitor_thread, args=(self._redis, collector)) self._commands.start() @@ -129,7 +118,7 @@ class RedisTransport(TransportBase): channel = "%s%s" % (platform.node(), int(time.time())) pubsub.subscribe(channel) - self._redis.rpush(RedisTransport.COMMAND_KEY, + self._redis.rpush(RedisTransport.COMMAND_KEY, cPickle.dumps(RedisMessage(channel, method, args, kwargs))) resp = pubsub.listen() @@ -160,7 +149,7 @@ class RedisTransport(TransportBase): continue message = cPickle.loads(payload[1]) if not isinstance(message, RedisMessage): - self.logger.error("Message \"%s\" is not a RedisMessage" % + self.logger.error("Message \"%s\" is not a RedisMessage" % message) if not message.method in collector.storage.__class__.__rmi__ or\ @@ -192,5 +181,3 @@ class RedisTransport(TransportBase): self.logger.error("Unhandled exception in command thread: %s" % traceback.format_exc().splitlines()[-1]) self.logger.info("Command thread shutdown") - - diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py index 73bdd0b3a..04b574ed7 100644 --- a/src/lib/Bcfg2/Reporting/Transport/__init__.py +++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py @@ -1,35 +1,3 @@ """ Public transport routines """ - -import sys -from Bcfg2.Reporting.Transport.base import TransportError, \ - TransportImportError - - -def load_transport(transport_name, setup): - """ - Try to load the transport. Raise TransportImportError on failure - """ - try: - mod_name = "%s.%s" % (__name__, transport_name) - mod = getattr(__import__(mod_name).Reporting.Transport, transport_name) - except ImportError: - try: - mod = __import__(transport_name) - except: - raise TransportImportError("Error importing transport %s: %s" % - (transport_name, sys.exc_info()[1])) - try: - return getattr(mod, transport_name)(setup) - except: - raise TransportImportError("Error instantiating transport %s: %s" % - (transport_name, sys.exc_info()[1])) - - -def load_transport_from_config(setup): - """Load the transport in the config... eventually""" - try: - return load_transport(setup['reporting_transport'], setup) - except KeyError: - raise TransportImportError('Transport missing in config') diff --git a/src/lib/Bcfg2/Reporting/Transport/base.py b/src/lib/Bcfg2/Reporting/Transport/base.py index 530011e47..9fbf8c9d5 100644 --- a/src/lib/Bcfg2/Reporting/Transport/base.py +++ b/src/lib/Bcfg2/Reporting/Transport/base.py @@ -4,7 +4,7 @@ The base for all server -> collector Transports import os import sys -from Bcfg2.Server.Plugin import Debuggable +from Bcfg2.Logger import Debuggable class TransportError(Exception): @@ -12,20 +12,18 @@ class TransportError(Exception): pass -class TransportImportError(TransportError): - """Raised when a transport fails to import""" - pass - - class TransportBase(Debuggable): """The base for all transports""" - def __init__(self, setup): + options = Debuggable.options + + def __init__(self): """Do something here""" clsname = self.__class__.__name__ Debuggable.__init__(self, name=clsname) self.debug_log("Loading %s transport" % clsname) - self.data = os.path.join(setup['repo'], 'Reporting', clsname) + self.data = os.path.join(Bcfg2.Options.setup.repository, 'Reporting', + clsname) if not os.path.exists(self.data): self.logger.info("%s does not exist, creating" % self.data) try: @@ -34,7 +32,6 @@ class TransportBase(Debuggable): self.logger.warning("Could not create %s: %s" % (self.data, sys.exc_info()[1])) self.logger.warning("The transport may not function properly") - self.setup = setup self.timeout = 2 def start_monitor(self, collector): diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py new file mode 100644 index 000000000..7c2241f58 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin.py @@ -0,0 +1,1155 @@ +""" Subcommands and helpers for bcfg2-admin """ + +import os +import sys +import time +import glob +import stat +import random +import socket +import string +import getpass +import difflib +import tarfile +import argparse +import lxml.etree +import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.Server.Core +import Bcfg2.Client.Proxy +from Bcfg2.Server.Plugin import PullSource, Generator, MetadataConsistencyError +from Bcfg2.Utils import hostnames2ranges, Executor, safe_input +import Bcfg2.Server.Plugins.Metadata + +try: + import Bcfg2.settings + os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' + from django.core.exceptions import ImproperlyConfigured + from django.core import management + import Bcfg2.Server.models + + HAS_DJANGO = True + try: + import south # pylint: disable=W0611 + HAS_REPORTS = True + except ImportError: + HAS_REPORTS = False +except ImportError: + HAS_DJANGO = False + HAS_REPORTS = False + + +class ccolors: + # pylint: disable=W1401 + ADDED = '\033[92m' + CHANGED = '\033[93m' + REMOVED = '\033[91m' + ENDC = '\033[0m' + # pylint: enable=W1401 + + @staticmethod + def disable(cls): + cls.ADDED = '' + cls.CHANGED = '' + cls.REMOVED = '' + cls.ENDC = '' + + +def gen_password(length): + """Generates a random alphanumeric password with length characters.""" + chars = string.letters + string.digits + return "".join(random.choice(chars) for i in range(length)) + + +def print_table(rows, justify='left', hdr=True, vdelim=" ", padding=1): + """Pretty print a table + + rows - list of rows ([[row 1], [row 2], ..., [row n]]) + hdr - if True the first row is treated as a table header + vdelim - vertical delimiter between columns + padding - # of spaces around the longest element in the column + justify - may be left,center,right + + """ + hdelim = "=" + justify = {'left': str.ljust, + 'center': str.center, + 'right': str.rjust}[justify.lower()] + + # Calculate column widths (longest item in each column + # plus padding on both sides) + cols = list(zip(*rows)) + col_widths = [max([len(str(item)) + 2 * padding + for item in col]) for col in cols] + borderline = vdelim.join([w * hdelim for w in col_widths]) + + # Print out the table + print(borderline) + for row in rows: + print(vdelim.join([justify(str(item), width) + for (item, width) in zip(row, col_widths)])) + if hdr: + print(borderline) + hdr = False + + +class AdminCmd(Bcfg2.Options.Subcommand): + def setup(self): + """ Perform post-init (post-options parsing), pre-run setup + tasks """ + pass + + def errExit(self, emsg): + """ exit with an error """ + print(emsg) + raise SystemExit(1) + + +class _ServerAdminCmd(AdminCmd): + """Base class for admin modes that run a Bcfg2 server.""" + __plugin_whitelist__ = None + __plugin_blacklist__ = None + + options = AdminCmd.options + Bcfg2.Server.Core.Core.options + + def setup(self): + if self.__plugin_whitelist__ is not None: + Bcfg2.Options.setup.plugins = [ + p for p in Bcfg2.Options.setup.plugins + if p.name in self.__plugin_whitelist__] + elif self.__plugin_blacklist__ is not None: + Bcfg2.Options.setup.plugins = [ + p for p in Bcfg2.Options.setup.plugins + if p.name not in self.__plugin_blacklist__] + + try: + self.core = Bcfg2.Server.Core.Core() + except Bcfg2.Server.Core.CoreInitError: + msg = sys.exc_info()[1] + self.errExit("Core load failed: %s" % msg) + self.core.load_plugins() + self.core.fam.handle_event_set() + self.metadata = self.core.metadata + + def shutdown(self): + self.core.shutdown() + + +class _ProxyAdminCmd(AdminCmd): + """ Base class for admin modes that proxy to a running Bcfg2 server """ + + options = AdminCmd.options + Bcfg2.Client.Proxy.ComponentProxy.options + + def __init__(self): + AdminCmd.__init__(self) + self.proxy = None + + def setup(self): + self.proxy = Bcfg2.Client.Proxy.ComponentProxy() + + +class Backup(AdminCmd): + """ Make a backup of the Bcfg2 repository """ + + options = AdminCmd.options + [Bcfg2.Options.Common.repository] + + def run(self, setup): + timestamp = time.strftime('%Y%m%d%H%M%S') + datastore = setup.repository + fmt = 'gz' + mode = 'w:' + fmt + filename = timestamp + '.tar' + '.' + fmt + out = tarfile.open(os.path.join(datastore, filename), mode=mode) + out.add(datastore, os.path.basename(datastore)) + out.close() + print("Archive %s was stored under %s" % (filename, datastore)) + + +class Client(_ServerAdminCmd): + """ Create, delete, or list client entries """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.PositionalArgument( + "mode", + choices=["add", "del", "list"]), + Bcfg2.Options.PositionalArgument("hostname", nargs='?')] + + __plugin_whitelist__ = ["Metadata"] + + def run(self, setup): + if setup.mode != 'list' and not setup.hostname: + self.parser.error("<hostname> is required in %s mode" % setup.mode) + elif setup.mode == 'list' and setup.hostname: + self.logger.warning("<hostname> is not honored in list mode") + + if setup.mode == 'add': + try: + self.metadata.add_client(setup.hostname) + except MetadataConsistencyError: + err = sys.exc_info()[1] + self.errExit("Error adding client %s: %s" % (setup.hostname, + err)) + elif setup.mode == 'del': + try: + self.metadata.remove_client(setup.hostname) + except MetadataConsistencyError: + err = sys.exc_info()[1] + self.errExit("Error deleting client %s: %s" % (setup.hostname, + err)) + elif setup.mode == 'list': + for client in self.metadata.list_clients(): + print(client) + + +class Compare(AdminCmd): + """ Compare two hosts or two versions of a host specification """ + + help = "Given two XML files (as produced by bcfg2-info build or bcfg2 " + \ + "-qnc) or two directories containing XML files (as produced by " + \ + "bcfg2-info buildall or bcfg2-info builddir), output a detailed, " + \ + "Bcfg2-centric diff." + + options = AdminCmd.options + [ + Bcfg2.Options.Option( + "-d", "--diff-lines", type=int, + help="Show only N lines of a diff"), + Bcfg2.Options.BooleanOption( + "-c", "--color", help="Use colors even if not run from a TTY"), + Bcfg2.Options.BooleanOption( + "-q", "--quiet", + help="Only show that entries differ, not how they differ"), + Bcfg2.Options.PathOption("path1", metavar="<file-or-dir>"), + Bcfg2.Options.PathOption("path2", metavar="<file-or-dir>")] + + changes = dict() + + def removed(self, msg, host): + self.record("%sRemoved: %s%s" % (ccolors.REMOVED, msg, ccolors.ENDC), + host) + + def added(self, msg, host): + self.record("%sAdded: %s%s" % (ccolors.ADDED, msg, ccolors.ENDC), host) + + def changed(self, msg, host): + self.record("%sChanged: %s%s" % (ccolors.CHANGED, msg, ccolors.ENDC), + host) + + def record(self, msg, host): + if msg not in self.changes: + self.changes[msg] = [host] + else: + self.changes[msg].append(host) + + def udiff(self, l1, l2, **kwargs): + """ get a unified diff with control lines stripped """ + lines = None + if "lines" in kwargs: + if kwargs['lines'] is not None: + lines = int(kwargs['lines']) + del kwargs['lines'] + if lines == 0: + return [] + kwargs['n'] = 0 + diff = [] + for line in difflib.unified_diff(l1, l2, **kwargs): + if (line.startswith("--- ") or line.startswith("+++ ") or + line.startswith("@@ ")): + continue + if lines is not None and len(diff) > lines: + diff.append(" ...") + break + if line.startswith("+"): + for l in line.splitlines(): + diff.append(" %s%s%s" % (ccolors.ADDED, l, ccolors.ENDC)) + elif line.startswith("-"): + for l in line.splitlines(): + diff.append(" %s%s%s" % (ccolors.REMOVED, l, + ccolors.ENDC)) + return diff + + def _bundletype(self, el): + if el.get("tag") == "Independent": + return "Independent bundle" + else: + return "Bundle" + + def run(self, setup): + if not sys.stdout.isatty() and not setup.color: + ccolors.disable(ccolors) + + files = [] + if os.path.isdir(setup.path1) and os.path.isdir(setup.path1): + for fpath in glob.glob(os.path.join(setup.path1, '*')): + fname = os.path.basename(fpath) + if os.path.exists(os.path.join(setup.path2, fname)): + files.append((os.path.join(setup.path1, fname), + os.path.join(setup.path2, fname))) + else: + if fname.endswith(".xml"): + host = fname[0:-4] + else: + host = fname + self.removed(host, '') + for fpath in glob.glob(os.path.join(setup.path2, '*')): + fname = os.path.basename(fpath) + if not os.path.exists(os.path.join(setup.path1, fname)): + if fname.endswith(".xml"): + host = fname[0:-4] + else: + host = fname + self.added(host, '') + elif os.path.isfile(setup.path1) and os.path.isfile(setup.path2): + files.append((setup.path1, setup.path2)) + else: + self.errExit("Cannot diff a file and a directory") + + for file1, file2 in files: + host = None + if os.path.basename(file1) == os.path.basename(file2): + fname = os.path.basename(file1) + if fname.endswith(".xml"): + host = fname[0:-4] + else: + host = fname + + xdata1 = lxml.etree.parse(file1).getroot() + xdata2 = lxml.etree.parse(file2).getroot() + + elements1 = dict() + elements2 = dict() + bundles1 = [el.get("name") for el in xdata1.iterchildren()] + bundles2 = [el.get("name") for el in xdata2.iterchildren()] + for el in xdata1.iterchildren(): + if el.get("name") not in bundles2: + self.removed("%s %s" % (self._bundletype(el), + el.get("name")), + host) + for el in xdata2.iterchildren(): + if el.get("name") not in bundles1: + self.added("%s %s" % (self._bundletype(el), + el.get("name")), + host) + + for bname in bundles1: + bundle = xdata1.find("*[@name='%s']" % bname) + for el in bundle.getchildren(): + elements1["%s:%s" % (el.tag, el.get("name"))] = el + for bname in bundles2: + bundle = xdata2.find("*[@name='%s']" % bname) + for el in bundle.getchildren(): + elements2["%s:%s" % (el.tag, el.get("name"))] = el + + for el in elements1.values(): + elid = "%s:%s" % (el.tag, el.get("name")) + if elid not in elements2: + self.removed("Element %s" % elid, host) + else: + el2 = elements2[elid] + if (el.getparent().get("name") != + el2.getparent().get("name")): + self.changed( + "Element %s was in bundle %s, " + "now in bundle %s" % (elid, + el.getparent().get("name"), + el2.getparent().get("name")), + host) + attr1 = sorted(["%s=\"%s\"" % (attr, el.get(attr)) + for attr in el.attrib]) + attr2 = sorted(["%s=\"%s\"" % (attr, el.get(attr)) + for attr in el2.attrib]) + if attr1 != attr2: + err = ["Element %s has different attributes" % elid] + if not setup.quiet: + err.extend(self.udiff(attr1, attr2)) + self.changed("\n".join(err), host) + + if el.text != el2.text: + if el.text is None: + self.changed("Element %s content was added" % elid, + host) + elif el2.text is None: + self.changed("Element %s content was removed" % + elid, host) + else: + err = ["Element %s has different content" % + elid] + if not setup.quiet: + err.extend( + self.udiff(el.text.splitlines(), + el2.text.splitlines(), + lines=setup.diff_lines)) + self.changed("\n".join(err), host) + + for el in elements2.values(): + elid = "%s:%s" % (el.tag, el.get("name")) + if elid not in elements2: + self.removed("Element %s" % elid, host) + + for change, hosts in self.changes.items(): + hlist = [h for h in hosts if h is not None] + if len(files) > 1 and len(hlist): + print("===== %s =====" % + "\n ".join(hostnames2ranges(hlist))) + print(change) + if len(files) > 1 and len(hlist): + print("") + + +class Help(AdminCmd, Bcfg2.Options.HelpCommand): + """ Get help on a specific subcommand """ + def command_registry(self): + return CLI.commands + + +class Init(AdminCmd): + """Interactively initialize a new repository.""" + + options = AdminCmd.options + [ + Bcfg2.Options.Common.repository, Bcfg2.Options.Common.plugins] + + # default config file + config = '''[server] +repository = %s +plugins = %s +# Uncomment the following to listen on all interfaces +#listen_all = true + +[database] +#engine = sqlite3 +# 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. +#name = +# Or path to database file if using sqlite3. +#<repository>/etc/bcfg2.sqlite is default path if left empty +#user = +# Not used with sqlite3. +#password = +# Not used with sqlite3. +#host = +# Not used with sqlite3. +#port = + +[reporting] +transport = LocalFilesystem + +[communication] +password = %s +certificate = %s +key = %s +ca = %s + +[components] +bcfg2 = %s +''' + + # Default groups + groups = '''<Groups> + <Group profile='true' public='true' default='true' name='basic'/> +</Groups> +''' + + # Default contents of clients.xml + clients = '''<Clients> + <Client profile="basic" name="%s"/> +</Clients> +''' + + def __init__(self): + AdminCmd.__init__(self) + self.data = dict() + + def _set_defaults(self, setup): + """Set default parameters.""" + self.data['plugins'] = setup.plugins + self.data['configfile'] = setup.config + self.data['repopath'] = setup.repository + self.data['password'] = gen_password(8) + self.data['shostname'] = socket.getfqdn() + self.data['server_uri'] = "https://%s:6789" % self.data['shostname'] + self.data['country'] = 'US' + self.data['state'] = 'Illinois' + self.data['location'] = 'Argonne' + if os.path.exists("/etc/pki/tls"): + self.data['keypath'] = "/etc/pki/tls/private/bcfg2.key" + self.data['certpath'] = "/etc/pki/tls/certs/bcfg2.crt" + elif os.path.exists("/etc/ssl"): + self.data['keypath'] = "/etc/ssl/bcfg2.key" + self.data['certpath'] = "/etc/ssl/bcfg2.crt" + else: + basepath = os.path.dirname(self.data['configfile']) + self.data['keypath'] = os.path.join(basepath, "bcfg2.key") + self.data['certpath'] = os.path.join(basepath, 'bcfg2.crt') + + def input_with_default(self, msg, default_name): + val = safe_input("%s [%s]: " % (msg, self.data[default_name])) + if val: + self.data[default_name] = val + + def run(self, setup): + self._set_defaults(setup) + + # Prompt the user for input + self._prompt_server() + self._prompt_config() + self._prompt_repopath() + self._prompt_password() + self._prompt_keypath() + self._prompt_certificate() + + # Initialize the repository + self.init_repo() + + def _prompt_server(self): + """Ask for the server name and URI.""" + self.input_with_default("What is the server's hostname", 'shostname') + # reset default server URI + self.data['server_uri'] = "https://%s:6789" % self.data['shostname'] + self.input_with_default("Server location", 'server_uri') + + def _prompt_config(self): + """Ask for the configuration file path.""" + self.input_with_default("Path to Bcfg2 configuration", 'configfile') + + def _prompt_repopath(self): + """Ask for the repository path.""" + while True: + self.input_with_default("Location of Bcfg2 repository", 'repopath') + if os.path.isdir(self.data['repopath']): + response = safe_input("Directory %s exists. Overwrite? [y/N]:" + % self.data['repopath']) + if response.lower().strip() == 'y': + break + else: + break + + def _prompt_password(self): + """Ask for a password or generate one if none is provided.""" + newpassword = getpass.getpass( + "Input password used for communication verification " + "(without echoing; leave blank for random): ").strip() + if len(newpassword) != 0: + self.data['password'] = newpassword + + def _prompt_certificate(self): + """Ask for the key details (country, state, and location).""" + print("The following questions affect SSL certificate generation.") + print("If no data is provided, the default values are used.") + self.input_with_default("Country code for certificate", 'country') + self.input_with_default("State or Province Name (full name) for " + "certificate", 'state') + self.input_with_default("Locality Name (e.g., city) for certificate", + 'location') + + def _prompt_keypath(self): + """ Ask for the key pair location. Try to use sensible + defaults depending on the OS """ + self.input_with_default("Path where Bcfg2 server private key will be " + "created", 'keypath') + self.input_with_default("Path where Bcfg2 server cert will be created", + 'certpath') + + def _init_plugins(self): + """Initialize each plugin-specific portion of the repository.""" + for plugin in self.data['plugins']: + kwargs = dict() + if issubclass(plugin, Bcfg2.Server.Plugins.Metadata.Metadata): + kwargs.update( + dict(groups_xml=self.groups, + clients_xml=self.clients % self.data['shostname'])) + plugin.init_repo(self.data['repopath'], **kwargs) + + def create_conf(self): + """ create the config file """ + confdata = self.config % ( + self.data['repopath'], + ','.join(p.__name__ for p in self.data['plugins']), + self.data['password'], + self.data['certpath'], + self.data['keypath'], + self.data['certpath'], + self.data['server_uri']) + + # Don't overwrite existing bcfg2.conf file + if os.path.exists(self.data['configfile']): + result = safe_input("\nWarning: %s already exists. " + "Overwrite? [y/N]: " % self.data['configfile']) + if result not in ['Y', 'y']: + print("Leaving %s unchanged" % self.data['configfile']) + return + try: + open(self.data['configfile'], "w").write(confdata) + os.chmod(self.data['configfile'], + stat.S_IRUSR | stat.S_IWUSR) # 0600 + except: # pylint: disable=W0702 + self.errExit("Error trying to write configuration file '%s': %s" % + (self.data['configfile'], sys.exc_info()[1])) + + def init_repo(self): + """Setup a new repo and create the content of the + configuration file.""" + # Create the repository + path = os.path.join(self.data['repopath'], 'etc') + try: + os.makedirs(path) + self._init_plugins() + print("Repository created successfuly in %s" % + self.data['repopath']) + except OSError: + print("Failed to create %s." % path) + + # Create the configuration file and SSL key + self.create_conf() + self.create_key() + + def create_key(self): + """Creates a bcfg2.key at the directory specifed by keypath.""" + cmd = Executor(timeout=120) + subject = "/C=%s/ST=%s/L=%s/CN=%s'" % ( + self.data['country'], self.data['state'], self.data['location'], + self.data['shostname']) + key = cmd.run(["openssl", "req", "-batch", "-x509", "-nodes", + "-subj", subject, "-days", "1000", + "-newkey", "rsa:2048", + "-keyout", self.data['keypath'], "-noout"]) + if not key.success: + print("Error generating key: %s" % key.error) + return + os.chmod(self.data['keypath'], stat.S_IRUSR | stat.S_IWUSR) # 0600 + csr = cmd.run(["openssl", "req", "-batch", "-new", "-subj", subject, + "-key", self.data['keypath']]) + if not csr.success: + print("Error generating certificate signing request: %s" % + csr.error) + return + cert = cmd.run(["openssl", "x509", "-req", "-days", "1000", + "-signkey", self.data['keypath'], + "-out", self.data['certpath']], + inputdata=csr.stdout) + if not cert.success: + print("Error signing certificate: %s" % cert.error) + return + + +class Minestruct(_ServerAdminCmd): + """ Extract extra entry lists from statistics """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.PathOption( + "-f", "--outfile", type=argparse.FileType('w'), default=sys.stdout, + help="Write to the given file"), + Bcfg2.Options.Option( + "-g", "--groups", help="Only build config for groups", + type=Bcfg2.Options.Types.colon_list, default=[]), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + extra = set() + for source in self.core.plugins_by_type(PullSource): + for item in source.GetExtra(setup.hostname): + extra.add(item) + except: # pylint: disable=W0702 + self.errExit("Failed to find extra entry info for client %s: %s" % + (setup.hostname, sys.exc_info()[1])) + root = lxml.etree.Element("Base") + self.logger.info("Found %d extra entries" % len(extra)) + add_point = root + for grp in setup.groups: + add_point = lxml.etree.SubElement(add_point, "Group", name=grp) + for tag, name in extra: + self.logger.info("%s: %s" % (tag, name)) + lxml.etree.SubElement(add_point, tag, name=name) + + lxml.etree.ElementTree(root).write(setup.outfile, pretty_print=True) + + +class Perf(_ProxyAdminCmd): + """ Get performance data from server """ + + def run(self, setup): + output = [('Name', 'Min', 'Max', 'Mean', 'Count')] + data = self.proxy.get_statistics() + for key in sorted(data.keys()): + output.append( + (key, ) + + tuple(["%.06f" % item + for item in data[key][:-1]] + [data[key][-1]])) + print_table(output) + + +class Pull(_ServerAdminCmd): + """ Retrieves entries from clients and integrates the information + into the repository """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.Common.interactive, + Bcfg2.Options.BooleanOption( + "-s", "--stdin", + help="Read lists of <hostname> <entrytype> <entryname> from stdin " + "instead of the command line"), + Bcfg2.Options.PositionalArgument("hostname", nargs='?'), + Bcfg2.Options.PositionalArgument("entrytype", nargs='?'), + Bcfg2.Options.PositionalArgument("entryname", nargs='?')] + + def __init__(self): + _ServerAdminCmd.__init__(self) + self.interactive = False + + def setup(self): + if (not Bcfg2.Options.setup.stdin and + not (Bcfg2.Options.setup.hostname and + Bcfg2.Options.setup.entrytype and + Bcfg2.Options.setup.entryname)): + print("You must specify either --stdin or a hostname, entry type, " + "and entry name on the command line.") + self.errExit(self.usage()) + _ServerAdminCmd.setup(self) + + def run(self, setup): + self.interactive = setup.interactive + if setup.stdin: + for line in sys.stdin: + try: + self.PullEntry(*line.split(None, 3)) + except SystemExit: + print(" for %s" % line) + except: + print("Bad entry: %s" % line.strip()) + else: + self.PullEntry(setup.hostname, setup.entrytype, setup.entryname) + + def BuildNewEntry(self, client, etype, ename): + """Construct a new full entry for + given client/entry from statistics. + """ + new_entry = {'type': etype, 'name': ename} + pull_sources = self.core.plugins_by_type(PullSource) + for plugin in pull_sources: + try: + (owner, group, mode, contents) = \ + plugin.GetCurrentEntry(client, etype, ename) + break + except Bcfg2.Server.Plugin.PluginExecutionError: + if plugin == pull_sources[-1]: + self.errExit("Pull Source failure; could not fetch " + "current state") + + try: + data = {'owner': owner, + 'group': group, + 'mode': mode, + 'text': contents} + except UnboundLocalError: + self.errExit("Unable to build entry") + for key, val in list(data.items()): + if val: + new_entry[key] = val + return new_entry + + def Choose(self, choices): + """Determine where to put pull data.""" + if self.interactive: + for choice in choices: + print("Plugin returned choice:") + if id(choice) == id(choices[0]): + print("(current entry) ") + if choice.all: + print(" => global entry") + elif choice.group: + print(" => group entry: %s (prio %d)" % + (choice.group, choice.prio)) + else: + print(" => host entry: %s" % (choice.hostname)) + + # flush input buffer + ans = safe_input("Use this entry? [yN]: ") in ['y', 'Y'] + if ans: + return choice + return False + else: + if not choices: + return False + return choices[0] + + def PullEntry(self, client, etype, ename): + """Make currently recorded client state correct for entry.""" + new_entry = self.BuildNewEntry(client, etype, ename) + + meta = self.core.build_metadata(client) + # Find appropriate plugin in core + glist = [gen for gen in self.core.plugins_by_type(Generator) + if ename in gen.Entries.get(etype, {})] + if len(glist) != 1: + self.errExit("Got wrong numbers of matching generators for entry:" + "%s" % ([g.name for g in glist])) + plugin = glist[0] + if not isinstance(plugin, Bcfg2.Server.Plugin.PullTarget): + self.errExit("Configuration upload not supported by plugin %s" % + plugin.name) + try: + choices = plugin.AcceptChoices(new_entry, meta) + specific = self.Choose(choices) + if specific: + plugin.AcceptPullData(specific, new_entry, self.logger) + except Bcfg2.Server.Plugin.PluginExecutionError: + self.errExit("Configuration upload not supported by plugin %s" % + plugin.name) + + # Commit if running under a VCS + for vcsplugin in list(self.core.plugins.values()): + if isinstance(vcsplugin, Bcfg2.Server.Plugin.Version): + files = "%s/%s" % (plugin.data, ename) + comment = 'file "%s" pulled from host %s' % (files, client) + vcsplugin.commit_data([files], comment) + + +class _ReportsCmd(AdminCmd): + def __init__(self): + AdminCmd.__init__(self) + self.reports_entries = () + self.reports_classes = () + + def setup(self): + # this has to be imported after options are parsed, + # because Django finalizes its settings as soon as it's + # loaded, which means that if we import this before + # Bcfg2.settings has been populated, Django gets a null + # configuration, and subsequent updates to Bcfg2.settings + # won't help. + import Bcfg2.Reporting.models + self.reports_entries = (Bcfg2.Reporting.models.Group, + Bcfg2.Reporting.models.Bundle, + Bcfg2.Reporting.models.FailureEntry, + Bcfg2.Reporting.models.ActionEntry, + Bcfg2.Reporting.models.PathEntry, + Bcfg2.Reporting.models.PackageEntry, + Bcfg2.Reporting.models.PathEntry, + Bcfg2.Reporting.models.ServiceEntry) + self.reports_classes = self.reports_entries + ( + Bcfg2.Reporting.models.Client, + Bcfg2.Reporting.models.Interaction, + Bcfg2.Reporting.models.Performance) + + +if HAS_DJANGO: + class _DjangoProxyCmd(AdminCmd): + command = None + args = [] + + def run(self, _): + '''Call a django command''' + if self.command is not None: + command = self.command + else: + command = self.__class__.__name__.lower() + args = [command] + self.args + management.call_command(*args) + + class DBShell(_DjangoProxyCmd): + """ Call the Django 'dbshell' command on the database """ + + class Shell(_DjangoProxyCmd): + """ Call the Django 'shell' command on the database """ + + class ValidateDB(_DjangoProxyCmd): + """ Call the Django 'validate' command on the database """ + command = "validate" + + class Syncdb(AdminCmd): + """ Sync the Django ORM with the configured database """ + + def run(self, setup): + management.setup_environ(Bcfg2.settings) + Bcfg2.Server.models.load_models() + try: + management.call_command("syncdb", interactive=False, + verbosity=setup.verbose + setup.debug) + except ImproperlyConfigured: + err = sys.exc_info()[1] + self.logger.error("Django configuration problem: %s" % err) + raise SystemExit(1) + except: + err = sys.exc_info()[1] + self.logger.error("Database update failed: %s" % err) + raise SystemExit(1) + + +if HAS_REPORTS: + import datetime + + class ScrubReports(_ReportsCmd): + """ Perform a thorough scrub and cleanup of the Reporting + database """ + + def setup(self): + _ReportsCmd.setup(self) + # this has to be imported after options are parsed, + # because Django finalizes its settings as soon as it's + # loaded, which means that if we import this before + # Bcfg2.settings has been populated, Django gets a null + # configuration, and subsequent updates to Bcfg2.settings + # won't help. + from django.db.transaction import commit_on_success + self.run = commit_on_success(self.run) + + def run(self, _): + # Cleanup unused entries + for cls in self.reports_entries: + try: + start_count = cls.objects.count() + cls.prune_orphans() + self.logger.info("Pruned %d %s records" % + (start_count - cls.objects.count(), + cls.__name__)) + except: # pylint: disable=W0702 + print("Failed to prune %s: %s" % + (cls.__name__, sys.exc_info()[1])) + + class InitReports(AdminCmd): + """ Initialize the Reporting database """ + def run(self, setup): + verbose = setup.verbose + setup.debug + try: + management.call_command("syncdb", interactive=False, + verbosity=verbose) + management.call_command("migrate", interactive=False, + verbosity=verbose) + except: # pylint: disable=W0702 + self.errExit("%s failed: %s" % + (self.__class__.__name__.title(), + sys.exc_info()[1])) + + class UpdateReports(InitReports): + """ Apply updates to the reporting database """ + + class ReportsStats(_ReportsCmd): + """ Print Reporting database statistics """ + def run(self, _): + for cls in self.reports_classes: + print("%s has %s records" % (cls.__name__, + cls.objects.count())) + + class PurgeReports(_ReportsCmd): + """ Purge records from the Reporting database """ + + options = AdminCmd.options + [ + Bcfg2.Options.Option("--client", help="Client to operate on"), + Bcfg2.Options.Option("--days", type=int, metavar='N', + help="Records older than N days"), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption("--expired", + help="Expired clients only"), + Bcfg2.Options.Option("--state", help="Purge entries in state", + choices=['dirty', 'clean', 'modified']), + required=False)] + + def run(self, setup): + if setup.days: + maxdate = datetime.datetime.now() - \ + datetime.timedelta(days=setup.days) + else: + maxdate = None + + starts = {} + for cls in self.reports_classes: + starts[cls] = cls.objects.count() + if setup.expired: + self.purge_expired(maxdate) + else: + self.purge(setup.client, maxdate, setup.state) + for cls in self.reports_classes: + self.logger.info("Purged %s %s records" % + (starts[cls] - cls.objects.count(), + cls.__name__)) + + def purge(self, client=None, maxdate=None, state=None): + '''Purge historical data from the database''' + # indicates whether or not a client should be deleted + filtered = False + + if not client and not maxdate and not state: + self.errExit("Refusing to prune all data. Specify an option " + "to %s" % self.__class__.__name__.lower()) + + ipurge = Bcfg2.Reporting.models.Interaction.objects + if client: + try: + cobj = Bcfg2.Reporting.models.Client.objects.get( + name=client) + ipurge = ipurge.filter(client=cobj) + except Bcfg2.Reporting.models.Client.DoesNotExist: + self.errExit("Client %s not in database" % client) + self.logger.debug("Filtering by client: %s" % client) + + if maxdate: + filtered = True + self.logger.debug("Filtering by maxdate: %s" % maxdate) + ipurge = ipurge.filter(timestamp__lt=maxdate) + + if Bcfg2.settings.DATABASES['default']['ENGINE'] == \ + 'django.db.backends.sqlite3': + grp_limit = 100 + else: + grp_limit = 1000 + if state: + filtered = True + self.logger.debug("Filtering by state: %s" % state) + ipurge = ipurge.filter(state=state) + + count = ipurge.count() + rnum = 0 + try: + while rnum < count: + grp = list(ipurge[:grp_limit].values("id")) + # just in case... + if not grp: + break + Bcfg2.Reporting.models.Interaction.objects.filter( + id__in=[x['id'] for x in grp]).delete() + rnum += len(grp) + self.logger.debug("Deleted %s of %s" % (rnum, count)) + except: # pylint: disable=W0702 + self.logger.error("Failed to remove interactions: %s" % + sys.exc_info()[1]) + + # Prune any orphaned ManyToMany relations + for m2m in self.reports_entries: + self.logger.debug("Pruning any orphaned %s objects" % + m2m.__name__) + m2m.prune_orphans() + + if client and not filtered: + # Delete the client, ping data is automatic + try: + self.logger.debug("Purging client %s" % client) + cobj.delete() + except: # pylint: disable=W0702 + self.logger.error("Failed to delete client %s: %s" % + (client, sys.exc_info()[1])) + + def purge_expired(self, maxdate=None): + """ Purge expired clients from the Reporting database """ + + if maxdate: + if not isinstance(maxdate, datetime.datetime): + raise TypeError("maxdate is not a DateTime object") + self.logger.debug("Filtering by maxdate: %s" % maxdate) + clients = Bcfg2.Reporting.models.Client.objects.filter( + expiration__lt=maxdate) + else: + clients = Bcfg2.Reporting.models.Client.objects.filter( + expiration__isnull=False) + + for client in clients: + self.logger.debug("Purging client %s" % client) + Bcfg2.Reporting.models.Interaction.objects.filter( + client=client).delete() + client.delete() + + class ReportsSQLAll(_DjangoProxyCmd): + """ Call the Django 'sqlall' command on the Reporting database """ + args = ["Reporting"] + + +class Viz(_ServerAdminCmd): + """ Produce graphviz diagrams of metadata structures """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.BooleanOption( + "-H", "--includehosts", + help="Include hosts in the viz output"), + Bcfg2.Options.BooleanOption( + "-b", "--includebundles", + help="Include bundles in the viz output"), + Bcfg2.Options.BooleanOption( + "-k", "--includekey", + help="Show a key for different digraph shapes"), + Bcfg2.Options.Option( + "-c", "--only-client", metavar="<hostname>", + help="Show only the groups, bundles for the named client"), + Bcfg2.Options.PathOption( + "-o", "--outfile", + help="Write viz output to an output file")] + + colors = ['steelblue1', 'chartreuse', 'gold', 'magenta', + 'indianred1', 'limegreen', 'orange1', 'lightblue2', + 'green1', 'blue1', 'yellow1', 'darkturquoise', 'gray66'] + + __plugin_blacklist__ = ['DBStats', 'Cfg', 'Pkgmgr', 'Packages', 'Rules', + 'Decisions', 'Deps', 'Git', 'Svn', 'Fossil', 'Bzr', + 'Bundler'] + + def run(self, setup): + if setup.outfile: + fmt = setup.outfile.split('.')[-1] + else: + fmt = 'png' + + exc = Executor() + cmd = ["dot", "-T", fmt] + if setup.outfile: + cmd.extend(["-o", setup.outfile]) + inputlist = ["digraph groups {", + '\trankdir="LR";', + self.metadata.viz(setup.includehosts, + setup.includebundles, + setup.includekey, + setup.only_client, + self.colors)] + if setup.includekey: + inputlist.extend( + ["\tsubgraph cluster_key {", + '\tstyle="filled";', + '\tcolor="lightblue";', + '\tBundle [ shape="septagon" ];', + '\tGroup [shape="ellipse"];', + '\tProfile [style="bold", shape="ellipse"];', + '\tHblock [label="Host1|Host2|Host3",shape="record"];', + '\tlabel="Key";', + "\t}"]) + inputlist.append("}") + idata = "\n".join(inputlist) + try: + result = exc.run(cmd, inputdata=idata) + except OSError: + # on some systems (RHEL 6), you cannot run dot with + # shell=True. on others (Gentoo with Python 2.7), you + # must. In yet others (RHEL 5), either way works. I have + # no idea what the difference is, but it's kind of a PITA. + result = exc.run(cmd, shell=True, inputdata=idata) + if not result.success: + self.errExit("Error running %s: %s" % (cmd, result.error)) + if not setup.outfile: + print(result.stdout) + + +class Xcmd(_ProxyAdminCmd): + """ XML-RPC Command Interface """ + + options = _ProxyAdminCmd.options + [ + Bcfg2.Options.PositionalArgument("command"), + Bcfg2.Options.PositionalArgument("arguments", nargs='*')] + + def run(self, setup): + try: + data = getattr(self.proxy, setup.command)(*setup.arguments) + except Bcfg2.Client.Proxy.ProxyError: + self.errExit("Proxy Error: %s" % sys.exc_info()[1]) + + if data is not None: + print(data) + + +class CLI(Bcfg2.Options.CommandRegistry): + """ CLI class for bcfg2-admin """ + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, globals().values(), + parent=AdminCmd) + parser = Bcfg2.Options.get_parser( + description="Manage a running Bcfg2 server", + components=[self]) + parser.parse() + + def run(self): + self.commands[Bcfg2.Options.setup.subcommand].setup() + return self.runcommand() diff --git a/src/lib/Bcfg2/Server/Admin/Backup.py b/src/lib/Bcfg2/Server/Admin/Backup.py deleted file mode 100644 index 0a04df98b..000000000 --- a/src/lib/Bcfg2/Server/Admin/Backup.py +++ /dev/null @@ -1,22 +0,0 @@ -""" Make a backup of the Bcfg2 repository """ - -import os -import time -import tarfile -import Bcfg2.Server.Admin -import Bcfg2.Options - - -class Backup(Bcfg2.Server.Admin.MetadataCore): - """ Make a backup of the Bcfg2 repository """ - - def __call__(self, args): - datastore = self.setup['repo'] - timestamp = time.strftime('%Y%m%d%H%M%S') - fmt = 'gz' - mode = 'w:' + fmt - filename = timestamp + '.tar' + '.' + fmt - out = tarfile.open(os.path.join(datastore, filename), mode=mode) - out.add(datastore, os.path.basename(datastore)) - out.close() - print("Archive %s was stored under %s" % (filename, datastore)) diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py deleted file mode 100644 index 187ccfd71..000000000 --- a/src/lib/Bcfg2/Server/Admin/Client.py +++ /dev/null @@ -1,32 +0,0 @@ -""" Create, delete, or list client entries """ - -import sys -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugin import MetadataConsistencyError - - -class Client(Bcfg2.Server.Admin.MetadataCore): - """ Create, delete, or list client entries """ - __usage__ = "[options] [add|del|list] [attr=val]" - __plugin_whitelist__ = ["Metadata"] - - def __call__(self, args): - if len(args) == 0: - self.errExit("No argument specified.\n" - "Usage: %s" % self.__usage__) - if args[0] == 'add': - try: - self.metadata.add_client(args[1]) - except MetadataConsistencyError: - self.errExit("Error in adding client: %s" % sys.exc_info()[1]) - elif args[0] in ['delete', 'remove', 'del', 'rm']: - try: - self.metadata.remove_client(args[1]) - except MetadataConsistencyError: - self.errExit("Error in deleting client: %s" % - sys.exc_info()[1]) - elif args[0] in ['list', 'ls']: - for client in self.metadata.list_clients(): - print(client) - else: - self.errExit("No command specified") diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py deleted file mode 100644 index 6bb15cafd..000000000 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ /dev/null @@ -1,147 +0,0 @@ -import lxml.etree -import os -import Bcfg2.Server.Admin - - -class Compare(Bcfg2.Server.Admin.Mode): - """ Determine differences between files or directories of client - specification instances """ - __usage__ = ("<old> <new>\n\n" - " -r\trecursive") - - def __init__(self): - Bcfg2.Server.Admin.Mode.__init__(self) - self.important = {'Path': ['name', 'type', 'owner', 'group', 'mode', - 'important', 'paranoid', 'sensitive', - 'dev_type', 'major', 'minor', 'prune', - 'encoding', 'empty', 'to', 'recursive', - 'vcstype', 'sourceurl', 'revision', - 'secontext'], - 'Package': ['name', 'type', 'version', 'simplefile', - 'verify'], - 'Service': ['name', 'type', 'status', 'mode', - 'target', 'sequence', 'parameters'], - 'Action': ['name', 'timing', 'when', 'status', - 'command'] - } - - def compareStructures(self, new, old): - if new.get("name"): - bundle = new.get('name') - else: - bundle = 'Independent' - - identical = True - - for child in new.getchildren(): - if child.tag not in self.important: - print(" %s in (new) bundle %s:\n tag type not handled!" % - (child.tag, bundle)) - continue - equiv = old.xpath('%s[@name="%s"]' % - (child.tag, child.get('name'))) - if len(equiv) == 0: - print(" %s %s in bundle %s:\n only in new configuration" % - (child.tag, child.get('name'), bundle)) - identical = False - continue - diff = [] - if child.tag == 'Path' and child.get('type') == 'file' and \ - child.text != equiv[0].text: - diff.append('contents') - attrdiff = [field for field in self.important[child.tag] if \ - child.get(field) != equiv[0].get(field)] - if attrdiff: - diff.append('attributes (%s)' % ', '.join(attrdiff)) - if diff: - print(" %s %s in bundle %s:\n %s differ" % (child.tag, \ - child.get('name'), bundle, ' and '.join(diff))) - identical = False - - for child in old.getchildren(): - if child.tag not in self.important: - print(" %s in (old) bundle %s:\n tag type not handled!" % - (child.tag, bundle)) - elif len(new.xpath('%s[@name="%s"]' % - (child.tag, child.get('name')))) == 0: - print(" %s %s in bundle %s:\n only in old configuration" % - (child.tag, child.get('name'), bundle)) - identical = False - - return identical - - def compareSpecifications(self, path1, path2): - try: - new = lxml.etree.parse(path1).getroot() - except IOError: - print("Failed to read %s" % (path1)) - raise SystemExit(1) - - try: - old = lxml.etree.parse(path2).getroot() - except IOError: - print("Failed to read %s" % (path2)) - raise SystemExit(1) - - for src in [new, old]: - for bundle in src.findall('./Bundle'): - if bundle.get('name')[-4:] == '.xml': - bundle.set('name', bundle.get('name')[:-4]) - - identical = True - - for bundle in old.findall('./Bundle'): - if len(new.xpath('Bundle[@name="%s"]' % (bundle.get('name')))) == 0: - print(" Bundle %s only in old configuration" % - bundle.get('name')) - identical = False - for bundle in new.findall('./Bundle'): - equiv = old.xpath('Bundle[@name="%s"]' % (bundle.get('name'))) - if len(equiv) == 0: - print(" Bundle %s only in new configuration" % - bundle.get('name')) - identical = False - elif not self.compareStructures(bundle, equiv[0]): - identical = False - - i1 = lxml.etree.Element('Independent') - i2 = lxml.etree.Element('Independent') - i1.extend(new.findall('./Independent/*')) - i2.extend(old.findall('./Independent/*')) - if not self.compareStructures(i1, i2): - identical = False - - return identical - - def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin compare help for usage.") - if '-r' in args: - args = list(args) - args.remove('-r') - (oldd, newd) = args - (old, new) = [os.listdir(spot) for spot in args] - old_extra = [] - for item in old: - if item not in new: - old_extra.append(item) - continue - print("File: %s" % item) - state = self.__call__([oldd + '/' + item, newd + '/' + item]) - new.remove(item) - if state: - print("File %s is good" % item) - else: - print("File %s is bad" % item) - if new: - print("%s has extra files: %s" % (newd, ', '.join(new))) - if old_extra: - print("%s has extra files: %s" % (oldd, ', '.join(old_extra))) - return - try: - (old, new) = args - return self.compareSpecifications(new, old) - except IndexError: - self.errExit(self.__call__.__doc__) diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py deleted file mode 100644 index ba553c7ef..000000000 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ /dev/null @@ -1,353 +0,0 @@ -""" Interactively initialize a new repository. """ - -import os -import sys -import stat -import select -import random -import socket -import string -import getpass -from Bcfg2.Utils import Executor -import Bcfg2.Server.Admin -import Bcfg2.Server.Plugin -import Bcfg2.Options -import Bcfg2.Server.Plugins.Metadata -from Bcfg2.Compat import input # pylint: disable=W0622 - -# default config file -CONFIG = '''[server] -repository = %s -plugins = %s -# Uncomment the following to listen on all interfaces -#listen_all = true - -[statistics] -sendmailpath = %s -#web_debug = False -#time_zone = - -[database] -#engine = sqlite3 -# 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. -#name = -# Or path to database file if using sqlite3. -#<repository>/etc/bcfg2.sqlite is default path if left empty -#user = -# Not used with sqlite3. -#password = -# Not used with sqlite3. -#host = -# Not used with sqlite3. -#port = - -[reporting] -transport = LocalFilesystem - -[communication] -protocol = %s -password = %s -certificate = %s -key = %s -ca = %s - -[components] -bcfg2 = %s -''' - -# Default groups -GROUPS = '''<Groups version='3.0'> - <Group profile='true' public='true' default='true' name='basic'> - <Group name='%s'/> - </Group> - <Group name='ubuntu'/> - <Group name='debian'/> - <Group name='freebsd'/> - <Group name='gentoo'/> - <Group name='redhat'/> - <Group name='suse'/> - <Group name='mandrake'/> - <Group name='solaris'/> - <Group name='arch'/> -</Groups> -''' - -# Default contents of clients.xml -CLIENTS = '''<Clients version="3.0"> - <Client profile="basic" name="%s"/> -</Clients> -''' - -# Mapping of operating system names to groups -OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/CentOS', 'redhat'), - ('SUSE/SLES', 'suse'), - ('Mandrake', 'mandrake'), - ('Debian', 'debian'), - ('Ubuntu', 'ubuntu'), - ('Gentoo', 'gentoo'), - ('FreeBSD', 'freebsd'), - ('Arch', 'arch')] - - -def safe_input(prompt): - """ input() that flushes the input buffer before accepting input """ - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - return input(prompt) - - -def gen_password(length): - """Generates a random alphanumeric password with length characters.""" - chars = string.letters + string.digits - return "".join(random.choice(chars) for i in range(length)) - - -def create_key(hostname, keypath, certpath, country, state, location): - """Creates a bcfg2.key at the directory specifed by keypath.""" - cmd = Executor(timeout=120) - subject = "/C=%s/ST=%s/L=%s/CN=%s'" % (country, state, location, hostname) - key = cmd.run(["openssl", "req", "-batch", "-x509", "-nodes", - "-subj", subject, "-days", "1000", "-newkey", "rsa:2048", - "-keyout", keypath, "-noout"]) - if not key.success: - print("Error generating key: %s" % key.error) - return - os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 - csr = cmd.run(["openssl", "req", "-batch", "-new", "-subj", subject, - "-key", keypath]) - if not csr.success: - print("Error generating certificate signing request: %s" % csr.error) - return - cert = cmd.run(["openssl", "x509", "-req", "-days", "1000", - "-signkey", keypath, "-out", certpath], - inputdata=csr.stdout) - if not cert.success: - print("Error signing certificate: %s" % cert.error) - return - - -def create_conf(confpath, confdata): - """ create the config file """ - # Don't overwrite existing bcfg2.conf file - if os.path.exists(confpath): - result = safe_input("\nWarning: %s already exists. " - "Overwrite? [y/N]: " % confpath) - if result not in ['Y', 'y']: - print("Leaving %s unchanged" % confpath) - return - try: - open(confpath, "w").write(confdata) - os.chmod(confpath, stat.S_IRUSR | stat.S_IWUSR) # 0600 - except Exception: - err = sys.exc_info()[1] - print("Error trying to write configuration file '%s': %s" % - (confpath, err)) - raise SystemExit(1) - - -class Init(Bcfg2.Server.Admin.Mode): - """Interactively initialize a new repository.""" - - def __init__(self): - Bcfg2.Server.Admin.Mode.__init__(self) - self.data = dict() - self.plugins = Bcfg2.Options.SERVER_PLUGINS.default - - def _set_defaults(self, opts): - """Set default parameters.""" - self.data['configfile'] = opts['configfile'] - self.data['repopath'] = opts['repo'] - self.data['password'] = gen_password(8) - self.data['server_uri'] = "https://%s:6789" % socket.getfqdn() - self.data['sendmail'] = opts['sendmail'] - self.data['proto'] = opts['proto'] - if os.path.exists("/etc/pki/tls"): - self.data['keypath'] = "/etc/pki/tls/private/bcfg2.key" - self.data['certpath'] = "/etc/pki/tls/certs/bcfg2.crt" - elif os.path.exists("/etc/ssl"): - self.data['keypath'] = "/etc/ssl/bcfg2.key" - self.data['certpath'] = "/etc/ssl/bcfg2.crt" - else: - basepath = os.path.dirname(self.configfile) - self.data['keypath'] = os.path.join(basepath, "bcfg2.key") - self.data['certpath'] = os.path.join(basepath, 'bcfg2.crt') - - def __call__(self, args): - # Parse options - setup = Bcfg2.Options.get_option_parser() - setup.add_options(dict(configfile=Bcfg2.Options.CFILE, - plugins=Bcfg2.Options.SERVER_PLUGINS, - proto=Bcfg2.Options.SERVER_PROTOCOL, - repo=Bcfg2.Options.SERVER_REPOSITORY, - sendmail=Bcfg2.Options.SENDMAIL_PATH)) - opts = sys.argv[1:] - opts.remove(self.__class__.__name__.lower()) - setup.reparse(argv=opts) - self._set_defaults(setup) - - # Prompt the user for input - self._prompt_config() - self._prompt_repopath() - self._prompt_password() - self._prompt_hostname() - self._prompt_server() - self._prompt_groups() - self._prompt_keypath() - self._prompt_certificate() - - # Initialize the repository - self.init_repo() - - def _prompt_hostname(self): - """Ask for the server hostname.""" - data = safe_input("What is the server's hostname [%s]: " % - socket.getfqdn()) - if data != '': - self.data['shostname'] = data - else: - self.data['shostname'] = socket.getfqdn() - - def _prompt_config(self): - """Ask for the configuration file path.""" - newconfig = safe_input("Store Bcfg2 configuration in [%s]: " % - self.configfile) - if newconfig != '': - self.data['configfile'] = os.path.abspath(newconfig) - - def _prompt_repopath(self): - """Ask for the repository path.""" - while True: - newrepo = safe_input("Location of Bcfg2 repository [%s]: " % - self.data['repopath']) - if newrepo != '': - self.data['repopath'] = os.path.abspath(newrepo) - if os.path.isdir(self.data['repopath']): - response = safe_input("Directory %s exists. Overwrite? [y/N]:" - % self.data['repopath']) - if response.lower().strip() == 'y': - break - else: - break - - def _prompt_password(self): - """Ask for a password or generate one if none is provided.""" - newpassword = getpass.getpass( - "Input password used for communication verification " - "(without echoing; leave blank for a random): ").strip() - if len(newpassword) != 0: - self.data['password'] = newpassword - - def _prompt_server(self): - """Ask for the server name.""" - newserver = safe_input( - "Input the server location (the server listens on a single " - "interface by default) [%s]: " % self.data['server_uri']) - if newserver != '': - self.data['server_uri'] = newserver - - def _prompt_groups(self): - """Create the groups.xml file.""" - prompt = '''Input base Operating System for clients:\n''' - for entry in OS_LIST: - prompt += "%d: %s\n" % (OS_LIST.index(entry) + 1, entry[0]) - prompt += ': ' - while True: - try: - osidx = int(safe_input(prompt)) - self.data['os_sel'] = OS_LIST[osidx - 1][1] - break - except ValueError: - continue - - def _prompt_certificate(self): - """Ask for the key details (country, state, and location).""" - print("The following questions affect SSL certificate generation.") - print("If no data is provided, the default values are used.") - newcountry = safe_input("Country name (2 letter code) for " - "certificate: ") - if newcountry != '': - if len(newcountry) == 2: - self.data['country'] = newcountry - else: - while len(newcountry) != 2: - newcountry = safe_input("2 letter country code (eg. US): ") - if len(newcountry) == 2: - self.data['country'] = newcountry - break - else: - self.data['country'] = 'US' - - newstate = safe_input("State or Province Name (full name) for " - "certificate: ") - if newstate != '': - self.data['state'] = newstate - else: - self.data['state'] = 'Illinois' - - newlocation = safe_input("Locality Name (eg, city) for certificate: ") - if newlocation != '': - self.data['location'] = newlocation - else: - self.data['location'] = 'Argonne' - - def _prompt_keypath(self): - """ Ask for the key pair location. Try to use sensible - defaults depending on the OS """ - keypath = safe_input("Path where Bcfg2 server private key will be " - "created [%s]: " % self.data['keypath']) - if keypath: - self.data['keypath'] = keypath - certpath = safe_input("Path where Bcfg2 server cert will be created " - "[%s]: " % self.data['certpath']) - if certpath: - self.data['certpath'] = certpath - - def _init_plugins(self): - """Initialize each plugin-specific portion of the repository.""" - for plugin in self.plugins: - if plugin == 'Metadata': - Bcfg2.Server.Plugins.Metadata.Metadata.init_repo( - self.data['repopath'], - groups_xml=GROUPS % self.data['os_sel'], - clients_xml=CLIENTS % socket.getfqdn()) - else: - try: - module = __import__("Bcfg2.Server.Plugins.%s" % plugin, '', - '', ["Bcfg2.Server.Plugins"]) - cls = getattr(module, plugin) - cls.init_repo(self.data['repopath']) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - print("Plugin setup for %s failed: %s\n" - "Check that dependencies are installed" % (plugin, - err)) - - def init_repo(self): - """Setup a new repo and create the content of the - configuration file.""" - # Create the repository - path = os.path.join(self.data['repopath'], 'etc') - try: - os.makedirs(path) - self._init_plugins() - print("Repository created successfuly in %s" % - self.data['repopath']) - except OSError: - print("Failed to create %s." % path) - - confdata = CONFIG % (self.data['repopath'], - ','.join(self.plugins), - self.data['sendmail'], - self.data['proto'], - self.data['password'], - self.data['certpath'], - self.data['keypath'], - self.data['certpath'], - self.data['server_uri']) - - # Create the configuration file and SSL key - create_conf(self.data['configfile'], confdata) - create_key(self.data['shostname'], self.data['keypath'], - self.data['certpath'], self.data['country'], - self.data['state'], self.data['location']) diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py deleted file mode 100644 index 37ca74894..000000000 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ /dev/null @@ -1,56 +0,0 @@ -""" Extract extra entry lists from statistics """ -import getopt -import lxml.etree -import sys -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugin import PullSource - - -class Minestruct(Bcfg2.Server.Admin.StructureMode): - """ Extract extra entry lists from statistics """ - __usage__ = ("[options] <client>\n\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-f <filename>", "build a particular file", - "-g <groups>", "only build config for groups")) - - def __call__(self, args): - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin minestruct help for usage.") - try: - (opts, args) = getopt.getopt(args, 'f:g:h') - except getopt.GetoptError: - self.errExit(self.__doc__) - - client = args[0] - output = sys.stdout - groups = [] - - for (opt, optarg) in opts: - if opt == '-f': - try: - output = open(optarg, 'w') - except IOError: - self.errExit("Failed to open file: %s" % (optarg)) - elif opt == '-g': - groups = optarg.split(':') - - try: - extra = set() - for source in self.bcore.plugins_by_type(PullSource): - for item in source.GetExtra(client): - extra.add(item) - except: # pylint: disable=W0702 - self.errExit("Failed to find extra entry info for client %s" % - client) - root = lxml.etree.Element("Base") - self.log.info("Found %d extra entries" % (len(extra))) - add_point = root - for grp in groups: - add_point = lxml.etree.SubElement(add_point, "Group", name=grp) - for tag, name in extra: - self.log.info("%s: %s" % (tag, name)) - lxml.etree.SubElement(add_point, tag, name=name) - - lxml.etree.ElementTree(root).write(output, pretty_print=True) diff --git a/src/lib/Bcfg2/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py deleted file mode 100644 index 1a772e6fc..000000000 --- a/src/lib/Bcfg2/Server/Admin/Perf.py +++ /dev/null @@ -1,38 +0,0 @@ -""" Get performance data from server """ - -import sys -import Bcfg2.Options -import Bcfg2.Client.Proxy -import Bcfg2.Server.Admin - - -class Perf(Bcfg2.Server.Admin.Mode): - """ Get performance data from server """ - - def __call__(self, args): - output = [('Name', 'Min', 'Max', 'Mean', 'Count')] - setup = Bcfg2.Options.get_option_parser() - setup.add_options(dict(ca=Bcfg2.Options.CLIENT_CA, - certificate=Bcfg2.Options.CLIENT_CERT, - key=Bcfg2.Options.SERVER_KEY, - password=Bcfg2.Options.SERVER_PASSWORD, - server=Bcfg2.Options.SERVER_LOCATION, - user=Bcfg2.Options.CLIENT_USER, - timeout=Bcfg2.Options.CLIENT_TIMEOUT)) - opts = sys.argv[1:] - opts.remove(self.__class__.__name__.lower()) - setup.reparse(argv=opts) - proxy = Bcfg2.Client.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) - data = proxy.get_statistics() - for key in sorted(data.keys()): - output.append( - (key, ) + - tuple(["%.06f" % item - for item in data[key][:-1]] + [data[key][-1]])) - self.print_table(output) diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py deleted file mode 100644 index 8f84cd87d..000000000 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ /dev/null @@ -1,147 +0,0 @@ -""" Retrieves entries from clients and integrates the information into -the repository """ - -import os -import sys -import getopt -import select -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugin import PullSource, Generator -from Bcfg2.Compat import input # pylint: disable=W0622 - - -class Pull(Bcfg2.Server.Admin.MetadataCore): - """ Retrieves entries from clients and integrates the information - into the repository """ - __usage__ = ("[options] <client> <entry type> <entry name>\n\n" - " %-25s%s\n" - " %-25s%s\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-v", "be verbose", - "-f", "force", - "-I", "interactive", - "-s", "stdin")) - - def __init__(self): - Bcfg2.Server.Admin.MetadataCore.__init__(self) - self.log = False - self.mode = 'interactive' - - def __call__(self, args): - use_stdin = False - try: - opts, gargs = getopt.getopt(args, 'vfIs') - except getopt.GetoptError: - self.errExit(self.__doc__) - for opt in opts: - if opt[0] == '-v': - self.log = True - elif opt[0] == '-f': - self.mode = 'force' - elif opt[0] == '-I': - self.mode = 'interactive' - elif opt[0] == '-s': - use_stdin = True - - if use_stdin: - for line in sys.stdin: - try: - self.PullEntry(*line.split(None, 3)) - except SystemExit: - print(" for %s" % line) - except: - print("Bad entry: %s" % line.strip()) - elif len(gargs) < 3: - self.usage() - else: - self.PullEntry(gargs[0], gargs[1], gargs[2]) - - def BuildNewEntry(self, client, etype, ename): - """Construct a new full entry for - given client/entry from statistics. - """ - new_entry = {'type': etype, 'name': ename} - pull_sources = self.bcore.plugins_by_type(PullSource) - for plugin in pull_sources: - try: - (owner, group, mode, contents) = \ - plugin.GetCurrentEntry(client, etype, ename) - break - except Bcfg2.Server.Plugin.PluginExecutionError: - if plugin == pull_sources[-1]: - print("Pull Source failure; could not fetch current state") - raise SystemExit(1) - - try: - data = {'owner': owner, - 'group': group, - 'mode': mode, - 'text': contents} - except UnboundLocalError: - print("Unable to build entry. " - "Do you have a statistics plugin enabled?") - raise SystemExit(1) - for key, val in list(data.items()): - if val: - new_entry[key] = val - return new_entry - - def Choose(self, choices): - """Determine where to put pull data.""" - if self.mode == 'interactive': - for choice in choices: - print("Plugin returned choice:") - if id(choice) == id(choices[0]): - print("(current entry) ") - if choice.all: - print(" => global entry") - elif choice.group: - print(" => group entry: %s (prio %d)" % - (choice.group, choice.prio)) - else: - print(" => host entry: %s" % (choice.hostname)) - - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], - 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - ans = input("Use this entry? [yN]: ") in ['y', 'Y'] - if ans: - return choice - return False - else: - # mode == 'force' - if not choices: - return False - return choices[0] - - def PullEntry(self, client, etype, ename): - """Make currently recorded client state correct for entry.""" - new_entry = self.BuildNewEntry(client, etype, ename) - - meta = self.bcore.build_metadata(client) - # Find appropriate plugin in bcore - glist = [gen for gen in self.bcore.plugins_by_type(Generator) - if ename in gen.Entries.get(etype, {})] - if len(glist) != 1: - self.errExit("Got wrong numbers of matching generators for entry:" - "%s" % ([g.name for g in glist])) - plugin = glist[0] - if not isinstance(plugin, Bcfg2.Server.Plugin.PullTarget): - self.errExit("Configuration upload not supported by plugin %s" % - plugin.name) - try: - choices = plugin.AcceptChoices(new_entry, meta) - specific = self.Choose(choices) - if specific: - plugin.AcceptPullData(specific, new_entry, self.log) - except Bcfg2.Server.Plugin.PluginExecutionError: - self.errExit("Configuration upload not supported by plugin %s" % - plugin.name) - # Commit if running under a VCS - for vcsplugin in list(self.bcore.plugins.values()): - if isinstance(vcsplugin, Bcfg2.Server.Plugin.Version): - files = "%s/%s" % (plugin.data, ename) - comment = 'file "%s" pulled from host %s' % (files, client) - vcsplugin.commit_data([files], comment) diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py deleted file mode 100644 index d21d66a22..000000000 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ /dev/null @@ -1,262 +0,0 @@ -'''Admin interface for dynamic reports''' -import Bcfg2.Logger -import Bcfg2.Server.Admin -import datetime -import os -import sys -import traceback -from Bcfg2 import settings - -# Load django and reports stuff _after_ we know we can load settings -from django.core import management -from Bcfg2.Reporting.utils import * - -project_directory = os.path.dirname(settings.__file__) -project_name = os.path.basename(project_directory) -sys.path.append(os.path.join(project_directory, '..')) -project_module = __import__(project_name, '', '', ['']) -sys.path.pop() - -# Set DJANGO_SETTINGS_MODULE appropriately. -os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name -from django.db import transaction - -from Bcfg2.Reporting.models import Client, Interaction, \ - Performance, Bundle, Group, FailureEntry, PathEntry, \ - PackageEntry, ServiceEntry, ActionEntry - - -def printStats(fn): - """ - Print db stats. - - Decorator for purging. Prints database statistics after a run. - """ - def print_stats(self, *data): - classes = (Client, Interaction, Performance, \ - FailureEntry, ActionEntry, PathEntry, PackageEntry, \ - ServiceEntry, Group, Bundle) - - starts = {} - for cls in classes: - starts[cls] = cls.objects.count() - - fn(self, *data) - - for cls in classes: - print("%s removed: %s" % (cls().__class__.__name__, - starts[cls] - cls.objects.count())) - - return print_stats - - -class Reports(Bcfg2.Server.Admin.Mode): - """ Manage dynamic reports """ - django_commands = ['dbshell', 'shell', 'sqlall', 'validate'] - __usage__ = ("[command] [options]\n" - " Commands:\n" - " init Initialize the database\n" - " purge Purge records\n" - " --client [n] Client to operate on\n" - " --days [n] Records older then n days\n" - " --expired Expired clients only\n" - " scrub Scrub the database for duplicate " - "reasons and orphaned entries\n" - " stats print database statistics\n" - " update Apply any updates to the reporting " - "database\n" - "\n" - " Django commands:\n " \ - + "\n ".join(django_commands)) - - def __init__(self): - Bcfg2.Server.Admin.Mode.__init__(self) - try: - import south - except ImportError: - print("Django south is required for Reporting") - raise SystemExit(-3) - - def __call__(self, args): - if len(args) == 0 or args[0] == '-h': - self.errExit(self.__usage__) - - # FIXME - dry run - - if args[0] in self.django_commands: - self.django_command_proxy(args[0]) - elif args[0] == 'scrub': - self.scrub() - elif args[0] == 'stats': - self.stats() - elif args[0] in ['init', 'update', 'syncdb']: - if self.setup['debug']: - vrb = 2 - elif self.setup['verbose']: - vrb = 1 - else: - vrb = 0 - try: - management.call_command("syncdb", verbosity=vrb) - management.call_command("migrate", verbosity=vrb) - except: - self.errExit("Update failed: %s" % sys.exc_info()[1]) - elif args[0] == 'purge': - expired = False - client = None - maxdate = None - state = None - i = 1 - while i < len(args): - if args[i] == '-c' or args[i] == '--client': - if client: - self.errExit("Only one client per run") - client = args[i + 1] - print(client) - i = i + 1 - elif args[i] == '--days': - if maxdate: - self.errExit("Max date specified multiple times") - try: - maxdate = datetime.datetime.now() - \ - datetime.timedelta(days=int(args[i + 1])) - except: - self.errExit("Invalid number of days: %s" % - args[i + 1]) - i = i + 1 - elif args[i] == '--expired': - expired = True - i = i + 1 - if expired: - if state: - self.errExit("--state is not valid with --expired") - self.purge_expired(maxdate) - else: - self.purge(client, maxdate, state) - else: - self.errExit("Unknown command: %s" % args[0]) - - @transaction.commit_on_success - def scrub(self): - ''' Perform a thorough scrub and cleanup of the database ''' - - # Cleanup unused entries - for cls in (Group, Bundle, FailureEntry, ActionEntry, PathEntry, - PackageEntry, PathEntry): - try: - start_count = cls.objects.count() - cls.prune_orphans() - self.log.info("Pruned %d %s records" % \ - (start_count - cls.objects.count(), cls.__class__.__name__)) - except: - print("Failed to prune %s: %s" % - (cls.__class__.__name__, sys.exc_info()[1])) - - def django_command_proxy(self, command): - '''Call a django command''' - if command == 'sqlall': - management.call_command(command, 'Reporting') - else: - management.call_command(command) - - @printStats - def purge(self, client=None, maxdate=None, state=None): - '''Purge historical data from the database''' - - filtered = False # indicates whether or not a client should be deleted - - if not client and not maxdate and not state: - self.errExit("Reports.prune: Refusing to prune all data") - - ipurge = Interaction.objects - if client: - try: - cobj = Client.objects.get(name=client) - ipurge = ipurge.filter(client=cobj) - except Client.DoesNotExist: - self.errExit("Client %s not in database" % client) - self.log.debug("Filtering by client: %s" % client) - - if maxdate: - filtered = True - if not isinstance(maxdate, datetime.datetime): - raise TypeError("maxdate is not a DateTime object") - self.log.debug("Filtering by maxdate: %s" % maxdate) - ipurge = ipurge.filter(timestamp__lt=maxdate) - - if settings.DATABASES['default']['ENGINE'] == \ - 'django.db.backends.sqlite3': - grp_limit = 100 - else: - grp_limit = 1000 - if state: - filtered = True - if state not in ('dirty', 'clean', 'modified'): - raise TypeError("state is not one of the following values: " - "dirty, clean, modified") - self.log.debug("Filtering by state: %s" % state) - ipurge = ipurge.filter(state=state) - - count = ipurge.count() - rnum = 0 - try: - while rnum < count: - grp = list(ipurge[:grp_limit].values("id")) - # just in case... - if not grp: - break - Interaction.objects.filter(id__in=[x['id'] - for x in grp]).delete() - rnum += len(grp) - self.log.debug("Deleted %s of %s" % (rnum, count)) - except: - self.log.error("Failed to remove interactions") - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - del a, b, c - self.log.error(msg) - - # Prune any orphaned ManyToMany relations - for m2m in (ActionEntry, PackageEntry, PathEntry, ServiceEntry, \ - FailureEntry, Group, Bundle): - self.log.debug("Pruning any orphaned %s objects" % \ - m2m().__class__.__name__) - m2m.prune_orphans() - - if client and not filtered: - # Delete the client, ping data is automatic - try: - self.log.debug("Purging client %s" % client) - cobj.delete() - except: - self.log.error("Failed to delete client %s" % client) - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - del a, b, c - self.log.error(msg) - - @printStats - def purge_expired(self, maxdate=None): - '''Purge expired clients from the database''' - - if maxdate: - if not isinstance(maxdate, datetime.datetime): - raise TypeError("maxdate is not a DateTime object") - self.log.debug("Filtering by maxdate: %s" % maxdate) - clients = Client.objects.filter(expiration__lt=maxdate) - else: - clients = Client.objects.filter(expiration__isnull=False) - - for client in clients: - self.log.debug("Purging client %s" % client) - Interaction.objects.filter(client=client).delete() - client.delete() - - def stats(self): - classes = (Client, Interaction, Performance, \ - FailureEntry, ActionEntry, PathEntry, PackageEntry, \ - ServiceEntry, Group, Bundle) - - for cls in classes: - print("%s has %s records" % (cls().__class__.__name__, - cls.objects.count())) diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py deleted file mode 100644 index 2722364f7..000000000 --- a/src/lib/Bcfg2/Server/Admin/Syncdb.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys -import Bcfg2.settings -import Bcfg2.Options -import Bcfg2.Server.Admin -import Bcfg2.Server.models -from django.core.exceptions import ImproperlyConfigured -from django.core.management import setup_environ, call_command - - -class Syncdb(Bcfg2.Server.Admin.Mode): - """ Sync the Django ORM with the configured database """ - - def __call__(self, args): - # Parse options - setup = Bcfg2.Options.get_option_parser() - setup.add_option("web_configfile", Bcfg2.Options.WEB_CFILE) - opts = sys.argv[1:] - opts.remove(self.__class__.__name__.lower()) - setup.reparse(argv=opts) - - setup_environ(Bcfg2.settings) - Bcfg2.Server.models.load_models(cfile=setup['web_configfile']) - - try: - call_command("syncdb", interactive=False, verbosity=0) - self._database_available = True - except ImproperlyConfigured: - self.errExit("Django configuration problem: %s" % - sys.exc_info()[1]) - except: - self.errExit("Database update failed: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Admin/Viz.py b/src/lib/Bcfg2/Server/Admin/Viz.py deleted file mode 100644 index a29fdaceb..000000000 --- a/src/lib/Bcfg2/Server/Admin/Viz.py +++ /dev/null @@ -1,104 +0,0 @@ -""" Produce graphviz diagrams of metadata structures """ - -import getopt -import Bcfg2.Server.Admin -from Bcfg2.Utils import Executor - - -class Viz(Bcfg2.Server.Admin.MetadataCore): - """ Produce graphviz diagrams of metadata structures """ - __usage__ = ("[options]\n\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" % - ("-H, --includehosts", - "include hosts in the viz output", - "-b, --includebundles", - "include bundles in the viz output", - "-k, --includekey", - "show a key for different digraph shapes", - "-c, --only-client <clientname>", - "show only the groups, bundles for the named client", - "-o, --outfile <file>", - "write viz output to an output file")) - - colors = ['steelblue1', 'chartreuse', 'gold', 'magenta', - 'indianred1', 'limegreen', 'orange1', 'lightblue2', - 'green1', 'blue1', 'yellow1', 'darkturquoise', 'gray66'] - - __plugin_blacklist__ = ['DBStats', 'Cfg', 'Pkgmgr', - 'Packages', 'Rules', 'Decisions', - 'Deps', 'Git', 'Svn', 'Fossil', 'Bzr', 'Bundler'] - - def __call__(self, args): - # First get options to the 'viz' subcommand - try: - opts, args = getopt.getopt(args, 'Hbkc:o:', - ['includehosts', 'includebundles', - 'includekey', 'only-client=', - 'outfile=']) - except getopt.GetoptError: - self.usage() - - hset = False - bset = False - kset = False - only_client = None - outputfile = False - for opt, arg in opts: - if opt in ("-H", "--includehosts"): - hset = True - elif opt in ("-b", "--includebundles"): - bset = True - elif opt in ("-k", "--includekey"): - kset = True - elif opt in ("-c", "--only-client"): - only_client = arg - elif opt in ("-o", "--outfile"): - outputfile = arg - - data = self.Visualize(hset, bset, kset, only_client, outputfile) - if data: - print(data) - - def Visualize(self, hosts=False, bundles=False, key=False, - only_client=None, output=None): - """Build visualization of groups file.""" - if output: - fmt = output.split('.')[-1] - else: - fmt = 'png' - - exc = Executor() - cmd = ["dot", "-T", fmt] - if output: - cmd.extend(["-o", output]) - idata = ["digraph groups {", - '\trankdir="LR";', - self.metadata.viz(hosts, bundles, - key, only_client, self.colors)] - if key: - idata.extend( - ["\tsubgraph cluster_key {", - '\tstyle="filled";', - '\tcolor="lightblue";', - '\tBundle [ shape="septagon" ];', - '\tGroup [shape="ellipse"];', - '\tProfile [style="bold", shape="ellipse"];', - '\tHblock [label="Host1|Host2|Host3",shape="record"];', - '\tlabel="Key";', - "\t}"]) - idata.append("}") - try: - result = exc.run(cmd, inputdata=idata) - except OSError: - # on some systems (RHEL 6), you cannot run dot with - # shell=True. on others (Gentoo with Python 2.7), you - # must. In yet others (RHEL 5), either way works. I have - # no idea what the difference is, but it's kind of a PITA. - result = exc.run(cmd, shell=True, inputdata=idata) - if not result.success: - print("Error running %s: %s" % (cmd, result.error)) - raise SystemExit(result.retval) diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py deleted file mode 100644 index f7f30fd80..000000000 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ /dev/null @@ -1,54 +0,0 @@ -""" XML-RPC Command Interface for bcfg2-admin""" - -import sys -import Bcfg2.Options -import Bcfg2.Client.Proxy -import Bcfg2.Server.Admin -from Bcfg2.Compat import xmlrpclib - - -class Xcmd(Bcfg2.Server.Admin.Mode): - """ XML-RPC Command Interface """ - __usage__ = "<command>" - - def __call__(self, args): - setup = Bcfg2.Options.get_option_parser() - setup.add_options(dict(ca=Bcfg2.Options.CLIENT_CA, - certificate=Bcfg2.Options.CLIENT_CERT, - key=Bcfg2.Options.SERVER_KEY, - password=Bcfg2.Options.SERVER_PASSWORD, - server=Bcfg2.Options.SERVER_LOCATION, - user=Bcfg2.Options.CLIENT_USER, - timeout=Bcfg2.Options.CLIENT_TIMEOUT)) - opts = sys.argv[1:] - opts.remove(self.__class__.__name__.lower()) - setup.reparse(argv=opts) - Bcfg2.Client.Proxy.RetryMethod.max_retries = 1 - proxy = Bcfg2.Client.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) - if len(setup['args']) == 0 or len(args) == 0: - self.errExit("Usage: xcmd <xmlrpc method> <optional arguments>") - cmd = args[0] - try: - data = getattr(proxy, cmd)(*args[1:]) - except xmlrpclib.Fault: - flt = sys.exc_info()[1] - if flt.faultCode == 7: - print("Unknown method %s" % cmd) - return - elif flt.faultCode == 20: - return - else: - raise - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - print("Proxy Error: %s" % err) - return - - if data is not None: - print(data) diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py deleted file mode 100644 index 06a419354..000000000 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ /dev/null @@ -1,142 +0,0 @@ -""" Base classes for admin modes """ - -import re -import sys -import logging -import lxml.etree -import Bcfg2.Server.Core -import Bcfg2.Options -from Bcfg2.Compat import ConfigParser, walk_packages - -__all__ = [m[1] for m in walk_packages(path=__path__)] - - -class Mode(object): - """ Base object for admin modes. Docstrings are used as help - messages, so if you are seeing this, a help message has not yet - been added for this mode. """ - __usage__ = None - __args__ = [] - - def __init__(self): - self.setup = Bcfg2.Options.get_option_parser() - self.configfile = self.setup['configfile'] - self.__cfp = False - self.log = logging.getLogger('Bcfg2.Server.Admin.Mode') - usage = "bcfg2-admin %s" % self.__class__.__name__.lower() - if self.__usage__ is not None: - usage += " " + self.__usage__ - self.setup.hm = usage - - def getCFP(self): - """ get a config parser for the Bcfg2 config file """ - if not self.__cfp: - self.__cfp = ConfigParser.ConfigParser() - self.__cfp.read(self.configfile) - return self.__cfp - - cfp = property(getCFP) - - def __call__(self, args): - raise NotImplementedError - - @classmethod - def usage(cls, rv=1): - """ Exit with a long usage message """ - print(re.sub(r'\s{2,}', ' ', cls.__doc__.strip())) - print("") - print("Usage:") - usage = "bcfg2-admin %s" % cls.__name__.lower() - if cls.__usage__ is not None: - usage += " " + cls.__usage__ - print(" %s" % usage) - raise SystemExit(rv) - - def shutdown(self): - """ Perform any necessary shtudown tasks for this mode """ - pass - - def errExit(self, emsg): - """ exit with an error """ - print(emsg) - raise SystemExit(1) - - def load_stats(self, client): - """ Load static statistics from the repository """ - stats = lxml.etree.parse("%s/etc/statistics.xml" % self.setup['repo']) - hostent = stats.xpath('//Node[@name="%s"]' % client) - if not hostent: - self.errExit("Could not find stats for client %s" % (client)) - return hostent[0] - - def print_table(self, rows, justify='left', hdr=True, vdelim=" ", - padding=1): - """Pretty print a table - - rows - list of rows ([[row 1], [row 2], ..., [row n]]) - hdr - if True the first row is treated as a table header - vdelim - vertical delimiter between columns - padding - # of spaces around the longest element in the column - justify - may be left,center,right - - """ - hdelim = "=" - justify = {'left': str.ljust, - 'center': str.center, - 'right': str.rjust}[justify.lower()] - - # Calculate column widths (longest item in each column - # plus padding on both sides) - cols = list(zip(*rows)) - col_widths = [max([len(str(item)) + 2 * padding - for item in col]) for col in cols] - borderline = vdelim.join([w * hdelim for w in col_widths]) - - # Print out the table - print(borderline) - for row in rows: - print(vdelim.join([justify(str(item), width) - for (item, width) in zip(row, col_widths)])) - if hdr: - print(borderline) - hdr = False - - -# pylint wants MetadataCore and StructureMode to be concrete classes -# and implement __call__, but they aren't and they don't, so we -# disable that warning -# pylint: disable=W0223 - -class MetadataCore(Mode): - """Base class for admin-modes that handle metadata.""" - __plugin_whitelist__ = None - __plugin_blacklist__ = None - - def __init__(self): - Mode.__init__(self) - if self.__plugin_whitelist__ is not None: - self.setup['plugins'] = [p for p in self.setup['plugins'] - if p in self.__plugin_whitelist__] - elif self.__plugin_blacklist__ is not None: - self.setup['plugins'] = [p for p in self.setup['plugins'] - if p not in self.__plugin_blacklist__] - - # admin modes don't need to watch for changes. one shot is fine here. - self.setup['filemonitor'] = 'pseudo' - try: - self.bcore = Bcfg2.Server.Core.BaseCore() - except Bcfg2.Server.Core.CoreInitError: - msg = sys.exc_info()[1] - self.errExit("Core load failed: %s" % msg) - self.bcore.load_plugins() - self.bcore.fam.handle_event_set() - self.metadata = self.bcore.metadata - - def shutdown(self): - if hasattr(self, 'bcore'): - self.bcore.shutdown() - - -class StructureMode(MetadataCore): # pylint: disable=W0223 - """ Base class for admin modes that handle structure plugins """ - pass diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index ea1d97e83..179a6aa9f 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -4,8 +4,9 @@ import sys import time import socket import daemon +import Bcfg2.Options import Bcfg2.Server.Statistics -from Bcfg2.Server.Core import BaseCore, NoExposedMethod +from Bcfg2.Server.Core import NetworkCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse from Bcfg2.Server.SSLServer import XMLRPCServer @@ -18,29 +19,29 @@ except ImportError: # pylint: enable=E0611 -class Core(BaseCore): +class BuiltinCore(NetworkCore): """ The built-in server core """ name = 'bcfg2-server' def __init__(self): - BaseCore.__init__(self) + NetworkCore.__init__(self) #: The :class:`Bcfg2.Server.SSLServer.XMLRPCServer` instance #: powering this server core self.server = None - daemon_args = dict(uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid'], - umask=int(self.setup['umask'], 8), + daemon_args = dict(uid=Bcfg2.Options.setup.daemon_uid, + gid=Bcfg2.Options.setup.daemon_gid, + umask=int(Bcfg2.Options.setup.umask, 8), detach_process=True) - if self.setup['daemon']: - daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'], - acquire_timeout=5) + if Bcfg2.Options.setup.daemon: + daemon_args['pidfile'] = TimeoutPIDLockFile( + Bcfg2.Options.setup.daemon, acquire_timeout=5) #: The :class:`daemon.DaemonContext` used to drop #: privileges, write the PID file (with :class:`PidFile`), #: and daemonize this core. self.context = daemon.DaemonContext(**daemon_args) - __init__.__doc__ = BaseCore.__init__.__doc__.split('.. -----')[0] + __init__.__doc__ = NetworkCore.__init__.__doc__.split('.. -----')[0] def _dispatch(self, method, args, dispatch_dict): """ Dispatch XML-RPC method calls @@ -95,25 +96,25 @@ class Core(BaseCore): except LockTimeout: err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: Failed to acquire lock " - "on %s" % (self.name, self.setup['daemon'])) + "on %s" % (self.name, + Bcfg2.Options.setup.daemon)) return False def _run(self): """ Create :attr:`server` to start the server listening. """ - hostname, port = urlparse(self.setup['location'])[1].split(':') + hostname, port = urlparse(Bcfg2.Options.setup.server)[1].split(':') server_address = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM)[0][4] try: - self.server = XMLRPCServer(self.setup['listen_all'], + self.server = XMLRPCServer(Bcfg2.Options.setup.listen_all, server_address, - keyfile=self.setup['key'], - certfile=self.setup['cert'], + keyfile=Bcfg2.Options.setup.key, + certfile=Bcfg2.Options.setup.cert, register=False, timeout=1, - ca=self.setup['ca'], - protocol=self.setup['protocol']) + ca=Bcfg2.Options.setup.ca) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Server startup failed: %s" % err) diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherrypyCore.py index bf3be72f9..dbfe260f7 100644 --- a/src/lib/Bcfg2/Server/CherryPyCore.py +++ b/src/lib/Bcfg2/Server/CherrypyCore.py @@ -5,7 +5,7 @@ import sys import time import Bcfg2.Server.Statistics from Bcfg2.Compat import urlparse, xmlrpclib, b64decode -from Bcfg2.Server.Core import BaseCore +from Bcfg2.Server.Core import NetworkCore import cherrypy from cherrypy.lib import xmlrpcutil from cherrypy._cptools import ErrorTool @@ -27,7 +27,7 @@ def on_error(*args, **kwargs): # pylint: disable=W0613 cherrypy.tools.xmlrpc_error = ErrorTool(on_error) -class Core(BaseCore): +class CherrypyCore(NetworkCore): """ The CherryPy-based server core. """ #: Base CherryPy config for this class. We enable the @@ -37,7 +37,7 @@ class Core(BaseCore): 'tools.bcfg2_authn.on': True} def __init__(self): - BaseCore.__init__(self) + NetworkCore.__init__(self) cherrypy.tools.bcfg2_authn = cherrypy.Tool('on_start_resource', self.do_authn) @@ -45,11 +45,11 @@ class Core(BaseCore): #: List of exposed plugin RMI self.rmi = self._get_rmi() cherrypy.engine.subscribe('stop', self.shutdown) - __init__.__doc__ = BaseCore.__init__.__doc__.split('.. -----')[0] + __init__.__doc__ = NetworkCore.__init__.__doc__.split('.. -----')[0] def do_authn(self): """ Perform authentication by calling - :func:`Bcfg2.Server.Core.BaseCore.authenticate`. This is + :func:`Bcfg2.Server.Core.NetworkCore.authenticate`. This is implemented as a CherryPy tool.""" try: header = cherrypy.request.headers['Authorization'] @@ -115,36 +115,36 @@ class Core(BaseCore): with :class:`cherrypy.process.plugins.Daemonizer`, and write a PID file with :class:`cherrypy.process.plugins.PIDFile`. """ DropPrivileges(cherrypy.engine, - uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid'], - umask=int(self.setup['umask'], 8)).subscribe() + uid=Bcfg2.Options.setup.daemon_uid, + gid=Bcfg2.Options.setup.daemon_gid, + umask=int(Bcfg2.Options.setup.umask, 8)).subscribe() Daemonizer(cherrypy.engine).subscribe() - PIDFile(cherrypy.engine, self.setup['daemon']).subscribe() + PIDFile(cherrypy.engine, Bcfg2.Options.setup.daemon).subscribe() return True def _run(self): """ Start the server listening. """ - hostname, port = urlparse(self.setup['location'])[1].split(':') - if self.setup['listen_all']: + hostname, port = urlparse(Bcfg2.Options.setup.server)[1].split(':') + if Bcfg2.Options.setup.listen_all: hostname = '0.0.0.0' config = {'engine.autoreload.on': False, 'server.socket_port': int(port), 'server.socket_host': hostname} - if self.setup['cert'] and self.setup['key']: + if Bcfg2.Options.setup.cert and Bcfg2.Options.setup.key: config.update({'server.ssl_module': 'pyopenssl', - 'server.ssl_certificate': self.setup['cert'], - 'server.ssl_private_key': self.setup['key']}) - if self.setup['debug']: + 'server.ssl_certificate': Bcfg2.Options.setup.cert, + 'server.ssl_private_key': Bcfg2.Options.setup.key}) + if Bcfg2.Options.setup.debug: config['log.screen'] = True cherrypy.config.update(config) - cherrypy.tree.mount(self, '/', {'/': self.setup}) + cherrypy.tree.mount(self, '/', {'/': Bcfg2.Options.setup}) cherrypy.engine.start() return True def _block(self): """ Enter the blocking infinite server - loop. :func:`Bcfg2.Server.Core.BaseCore.shutdown` is called on + loop. :func:`Bcfg2.Server.Core.NetworkCore.shutdown` is called on exit by a :meth:`subscription <cherrypy.process.wspbus.Bus.subscribe>` on the top-level CherryPy engine.""" diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 698703457..360b7868d 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -13,12 +13,12 @@ import inspect import lxml.etree import Bcfg2.Server import Bcfg2.Logger +import Bcfg2.Options import Bcfg2.settings import Bcfg2.Server.Statistics import Bcfg2.Server.FileMonitor from itertools import chain from Bcfg2.Server.Cache import Cache -from Bcfg2.Options import get_option_parser, SERVER_FAM_IGNORE from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 @@ -82,43 +82,40 @@ class NoExposedMethod (Exception): # in core we frequently want to catch all exceptions, regardless of # type, so disable the pylint rule that catches that. - -class BaseCore(object): +class Core(object): """ The server core is the container for all Bcfg2 server logic and modules. All core implementations must inherit from - ``BaseCore``. """ + ``Core``. """ + + options = [ + Bcfg2.Options.Common.plugins, + Bcfg2.Options.Common.repository, + Bcfg2.Options.Common.filemonitor, + Bcfg2.Options.BooleanOption( + cf=('server', 'fam_blocking'), default=False, + help='FAM blocks on startup until all events are processed'), + Bcfg2.Options.BooleanOption( + cf=('logging', 'performance'), dest="perflog", + help="Periodically log performance statistics"), + Bcfg2.Options.Option( + cf=('logging', 'performance_interval'), default=300.0, + type=Bcfg2.Options.Types.timeout, + help="Performance statistics logging interval in seconds"), + Bcfg2.Options.Option( + cf=('caching', 'client_metadata'), dest='client_metadata_cache', + default='off', + choices=['off', 'on', 'initial', 'cautious', 'aggressive'])] def __init__(self): # pylint: disable=R0912,R0915 """ - .. automethod:: _daemonize .. automethod:: _run .. automethod:: _block .. ----- .. automethod:: _file_monitor_thread .. automethod:: _perflog_thread """ - #: The Bcfg2 options dict - self.setup = get_option_parser() - #: The Bcfg2 repository directory - self.datastore = self.setup['repo'] - - if self.setup['debug']: - level = logging.DEBUG - elif self.setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - # we set a higher log level for the console by default. we - # assume that if someone is running bcfg2-server in such a way - # that it _can_ log to console, they want more output. if - # level is set to DEBUG, that will get handled by - # setup_logging and the console will get DEBUG output. - Bcfg2.Logger.setup_logging('bcfg2-server', - to_console=logging.INFO, - to_syslog=self.setup['syslog'], - to_file=self.setup['logging'], - level=level) + self.datastore = Bcfg2.Options.setup.repository #: A :class:`logging.Logger` object for use by the core self.logger = logging.getLogger('bcfg2-server') @@ -130,43 +127,32 @@ class BaseCore(object): #: special, and will be used for any log handlers whose name #: does not appear elsewhere in the dict. At a minimum, #: ``default`` must be provided. - self._loglevels = {True: dict(default=logging.DEBUG), - False: dict(console=logging.INFO, - default=level)} + self._loglevels = { + True: dict(default=logging.DEBUG), + False: dict(console=logging.INFO, + default=Bcfg2.Logger.default_log_level())} #: Used to keep track of the current debug state of the core. self.debug_flag = False # enable debugging on the core now. debugging is enabled on # everything else later - if self.setup['debug']: - self.set_core_debug(None, self.setup['debug']) - - if 'ignore' not in self.setup: - self.setup.add_option('ignore', SERVER_FAM_IGNORE) - self.setup.reparse() - - famargs = dict(filemonitor=self.setup['filemonitor'], - debug=self.setup['debug'], - ignore=self.setup['ignore']) - if self.setup['filemonitor'] not in Bcfg2.Server.FileMonitor.available: - self.logger.error("File monitor driver %s not available; " - "forcing to default" % self.setup['filemonitor']) - famargs['filemonitor'] = 'default' + if Bcfg2.Options.setup.debug: + self.set_core_debug(None, Bcfg2.Options.setup.debug) try: #: The :class:`Bcfg2.Server.FileMonitor.FileMonitor` #: object used by the core to monitor for Bcfg2 data #: changes. - self.fam = Bcfg2.Server.FileMonitor.load_fam(**famargs) + self.fam = Bcfg2.Server.FileMonitor.get_fam() except IOError: msg = "Failed to instantiate fam driver %s" % \ - self.setup['filemonitor'] + Bcfg2.Options.setup.filemonitor self.logger.error(msg, exc_info=1) raise CoreInitError(msg) #: Path to bcfg2.conf - self.cfile = self.setup['configfile'] + self.cfile = Bcfg2.Options.setup.config #: Dict of plugins that are enabled. Keys are the plugin #: names (just the plugin name, in the correct case; e.g., @@ -198,59 +184,19 @@ class BaseCore(object): # generate Django ORM settings. this must be done _before_ we # load plugins - Bcfg2.settings.read_config(repo=self.datastore) - - #: Whether or not it's possible to use the Django database - #: backend for plugins that have that capability - self._database_available = False - if Bcfg2.settings.HAS_DJANGO: - db_settings = Bcfg2.settings.DATABASES['default'] - if ('daemon' in self.setup and 'daemon_uid' in self.setup and - self.setup['daemon'] and self.setup['daemon_uid'] and - db_settings['ENGINE'].endswith(".sqlite3") and - not os.path.exists(db_settings['NAME'])): - # syncdb will create the sqlite database, and we're - # going to daemonize, dropping privs to a non-root - # user, so we need to chown the database after - # creating it - do_chown = True - else: - do_chown = False - - from django.core.exceptions import ImproperlyConfigured - from django.core import management - try: - management.call_command("syncdb", interactive=False, - verbosity=0) - self._database_available = True - except ImproperlyConfigured: - err = sys.exc_info()[1] - self.logger.error("Django configuration problem: %s" % err) - except: - err = sys.exc_info()[1] - self.logger.error("Database update failed: %s" % err) - - if do_chown and self._database_available: - try: - os.chown(db_settings['NAME'], - self.setup['daemon_uid'], - self.setup['daemon_gid']) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Failed to set ownership of database " - "at %s: %s" % (db_settings['NAME'], err)) - - #: The CA that signed the server cert - self.ca = self.setup['ca'] + Bcfg2.settings.read_config() #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ - threading.Thread(name="%sFAMThread" % self.setup['filemonitor'], + threading.Thread(name="%sFAMThread" % + Bcfg2.Options.setup.filemonitor.__name__, target=self._file_monitor_thread) + #: The :class:`threading.Thread` that reports performance + #: statistics to syslog. self.perflog_thread = None - if self.setup['perflog']: + if Bcfg2.Options.setup.perflog: self.perflog_thread = \ threading.Thread(name="PerformanceLoggingThread", target=self._perflog_thread) @@ -263,6 +209,24 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() + #: Whether or not it's possible to use the Django database + #: backend for plugins that have that capability + self._database_available = False + if Bcfg2.settings.HAS_DJANGO: + from django.core.exceptions import ImproperlyConfigured + from django.core import management + try: + management.call_command("syncdb", interactive=False, + verbosity=0) + self._database_available = True + except ImproperlyConfigured: + err = sys.exc_info()[1] + self.logger.error("Django configuration problem: %s" % err) + except: + err = sys.exc_info()[1] + self.logger.error("Updating database %s failed: %s" % + (Bcfg2.Options.setup.db_name, err)) + def expire_caches_by_type(self, base_cls, key=None): """ Expire caches for all :class:`Bcfg2.Server.Plugin.interfaces.Caching` plugins that @@ -302,7 +266,7 @@ class BaseCore(object): to syslog. """ self.logger.debug("Performance logging thread starting") while not self.terminate.isSet(): - self.terminate.wait(self.setup['perflog_interval']) + self.terminate.wait(Bcfg2.Options.setup.performance_interval) if not self.terminate.isSet(): for name, stats in self.get_statistics(None).items(): self.logger.info("Performance statistics: " @@ -354,10 +318,7 @@ class BaseCore(object): :attr:`Bcfg2.Server.Core.BaseCore.metadata` as side effects. This does not start plugin threads; that is done later, in :func:`Bcfg2.Server.Core.BaseCore.run` """ - while '' in self.setup['plugins']: - self.setup['plugins'].remove('') - - for plugin in self.setup['plugins']: + for plugin in Bcfg2.Options.setup.plugins: if not plugin in self.plugins: self.init_plugin(plugin) @@ -397,10 +358,6 @@ class BaseCore(object): "failed to instantiate Core") raise CoreInitError("No Metadata Plugin") - if self.debug_flag: - # enable debugging on plugins - self.plugins[plugin].set_debug(self.debug_flag) - def init_plugin(self, plugin): """ Import and instantiate a single plugin. The plugin is stored to :attr:`plugins`. @@ -411,29 +368,13 @@ class BaseCore(object): :type plugin: string :returns: None """ - self.logger.debug("Loading plugin %s" % plugin) - try: - mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % - (plugin)).Server.Plugins, plugin) - except ImportError: - try: - mod = __import__(plugin, globals(), locals(), - [plugin.split('.')[-1]]) - except: - self.logger.error("Failed to load plugin %s" % plugin) - return - try: - plug = getattr(mod, plugin.split('.')[-1]) - except AttributeError: - self.logger.error("Failed to load plugin %s: %s" % - (plugin, sys.exc_info()[1])) - return + self.logger.debug("Loading plugin %s" % plugin.name) # Blacklist conflicting plugins - cplugs = [conflict for conflict in plug.conflicts + cplugs = [conflict for conflict in plugin.conflicts if conflict in self.plugins] - self.plugin_blacklist[plug.name] = cplugs + self.plugin_blacklist[plugin.name] = cplugs try: - self.plugins[plugin] = plug(self, self.datastore) + self.plugins[plugin.name] = plugin(self, self.datastore) except PluginInitError: self.logger.error("Failed to instantiate plugin %s" % plugin, exc_info=1) @@ -461,10 +402,7 @@ class BaseCore(object): """ Get the client :attr:`metadata_cache` mode. Options are off, initial, cautious, aggressive, on (synonym for cautious). See :ref:`server-caching` for more details. """ - # pylint: disable=E1103 - mode = self.setup.cfp.get("caching", "client_metadata", - default="off").lower() - # pylint: enable=E1103 + mode = Bcfg2.Options.setup.client_metadata_cache if mode == "on": return "cautious" else: @@ -648,10 +586,9 @@ class BaseCore(object): del entry.attrib['realname'] return ret except: - entry.set('name', oldname) self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, entry.get('name'), - entry.get('altsrc'))) + (entry.tag, oldname, entry.get('name'))) + entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) @@ -745,7 +682,7 @@ class BaseCore(object): return if event.code2str() == 'deleted': return - self.setup.reparse() + Bcfg2.Options.get_parser().reparse() self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def block_for_fam_events(self, handle_events=False): @@ -758,7 +695,7 @@ class BaseCore(object): if handle_events: self.fam.handle_events_in_interval(1) slept += 1 - if self.setup['fam_blocking']: + if Bcfg2.Options.setup.fam_blocking: time.sleep(1) slept += 1 while self.fam.pending() != 0: @@ -769,35 +706,12 @@ class BaseCore(object): self.logger.debug("Slept %s seconds while handling FAM events" % slept) def run(self): - """ Run the server core. This calls :func:`_daemonize`, - :func:`_run`, starts the :attr:`fam_thread`, and calls - :func:`_block`, but note that it is the responsibility of the - server core implementation to call :func:`shutdown` under - normal operation. This also handles creation of the directory - containing the pidfile, if necessary. """ - if self.setup['daemon']: - # if we're dropping privs, then the pidfile is likely - # /var/run/bcfg2-server/bcfg2-server.pid or similar. - # since some OSes clean directories out of /var/run on - # reboot, we need to ensure that the directory containing - # the pidfile exists and has the appropriate permissions - piddir = os.path.dirname(self.setup['daemon']) - if not os.path.exists(piddir): - os.makedirs(piddir) - os.chown(piddir, - self.setup['daemon_uid'], - self.setup['daemon_gid']) - os.chmod(piddir, 493) # 0775 - if not self._daemonize(): - return False - - # rewrite $HOME. pulp stores its auth creds in ~/.pulp, so - # this is necessary to make that work when privileges are - # dropped - os.environ['HOME'] = pwd.getpwuid(self.setup['daemon_uid'])[5] - else: - os.umask(int(self.setup['umask'], 8)) - + """ Run the server core. This calls :func:`_run`, starts the + :attr:`fam_thread`, and calls :func:`_block`, but note that it + is the responsibility of the server core implementation to + call :func:`shutdown` under normal operation. This also + handles creation of the directory containing the pidfile, if + necessary.""" if not self._run(): self.shutdown() return False @@ -817,16 +731,9 @@ class BaseCore(object): self.shutdown() raise - if self.debug_flag: - self.set_debug(None, self.debug_flag) self.block_for_fam_events() self._block() - def _daemonize(self): - """ Daemonize the server and write the pidfile. This must be - overridden by a core implementation. """ - raise NotImplementedError - def _run(self): """ Start up the server; this method should return immediately. This must be overridden by a core @@ -884,9 +791,13 @@ class BaseCore(object): if all(ip_checks): # if all ACL plugins return True (allow), then allow + self.logger.debug("Client %s passed IP-based ACL checks for %s" % + (address[0], rmi)) return True elif False in ip_checks: # if any ACL plugin returned False (deny), then deny + self.logger.warning("Client %s failed IP-based ACL checks for %s" % + (address[0], rmi)) return False # else, no plugins returned False, but not all plugins # returned True, so some plugin returned None (defer), so @@ -894,7 +805,16 @@ class BaseCore(object): client, metadata = self.resolve_client(address) try: - return all(p.check_acl_metadata(metadata, rmi) for p in plugins) + rv = all(p.check_acl_metadata(metadata, rmi) for p in plugins) + if rv: + self.logger.debug( + "Client %s passed metadata ACL checks for %s" % + (metadata.hostname, rmi)) + else: + self.logger.warning( + "Client %s failed metadata ACL checks for %s" % + (metadata.hostname, rmi)) + return rv except: self.logger.error("Unexpected error checking ACLs for %s for %s: " "%s" % (client, rmi, sys.exc_info()[1])) @@ -1226,28 +1146,6 @@ class BaseCore(object): self.process_statistics(client, sdata) return True - def authenticate(self, cert, user, password, address): - """ Authenticate a client connection with - :func:`Bcfg2.Server.Plugin.interfaces.Metadata.AuthenticateConnection`. - - :param cert: an x509 certificate - :type cert: dict - :param user: The username of the user trying to authenticate - :type user: string - :param password: The password supplied by the client - :type password: string - :param address: An address pair of ``(<ip address>, <port>)`` - :type address: tuple - :return: bool - True if the authenticate succeeds, False otherwise - """ - if self.ca: - acert = cert - else: - # No ca, so no cert validation can be done - acert = None - return self.metadata.AuthenticateConnection(acert, user, password, - address) - @exposed def GetDecisionList(self, address, mode): """ Get the decision list for the client with :func:`GetDecisions`. @@ -1369,3 +1267,110 @@ class BaseCore(object): address[0]) return "This method is deprecated and will be removed in a future " + \ "release\n%s" % self.fam.set_debug(debug) + + +class NetworkCore(Core): + """ A server core that actually listens on the network, can be + daemonized, etc.""" + options = Core.options + [ + Bcfg2.Options.Common.daemon, Bcfg2.Options.Common.syslog, + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_key, + Bcfg2.Options.Common.ssl_cert, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.BooleanOption( + '--listen-all', cf=('server', 'listen_all'), default=False, + help="Listen on all interfaces"), + Bcfg2.Options.Option( + cf=('server', 'umask'), default='0077', help='Server umask', + type=Bcfg2.Options.Types.octal), + Bcfg2.Options.Option( + cf=('server', 'user'), default=0, dest='daemon_uid', + type=Bcfg2.Options.Types.username, + help="User to run the server daemon as"), + Bcfg2.Options.Option( + cf=('server', 'group'), default=0, dest='daemon_gid', + type=Bcfg2.Options.Types.groupname, + help="Group to run the server daemon as")] + + def __init__(self): + Core.__init__(self) + + #: The CA that signed the server cert + self.ca = Bcfg2.Options.setup.ca + + if self._database_available: + db_settings = Bcfg2.settings.DATABASES['default'] + if (Bcfg2.Options.setup.daemon and + Bcfg2.Options.setup.daemon_uid and + db_settings['ENGINE'].endswith(".sqlite3") and + not os.path.exists(db_settings['NAME'])): + # syncdb will create the sqlite database, and we're + # going to daemonize, dropping privs to a non-root + # user, so we need to chown the database after + # creating it + try: + os.chown(db_settings['NAME'], + Bcfg2.Options.setup.daemon_uid, + Bcfg2.Options.setup.daemon_gid) + except OSError: + err = sys.exc_info()[1] + self.logger.error("Failed to set ownership of database " + "at %s: %s" % (db_settings['NAME'], err)) + __init__.__doc__ = Core.__init__.__doc__.split(".. -----")[0] + \ + "\n.. automethod:: _daemonize\n" + + def run(self): + """ Run the server core. This calls :func:`_daemonize` before + calling :func:`Bcfg2.Server.Core.Core.run` to run the server + core. """ + if Bcfg2.Options.setup.daemon: + # if we're dropping privs, then the pidfile is likely + # /var/run/bcfg2-server/bcfg2-server.pid or similar. + # since some OSes clean directories out of /var/run on + # reboot, we need to ensure that the directory containing + # the pidfile exists and has the appropriate permissions + piddir = os.path.dirname(Bcfg2.Options.setup.daemon) + if not os.path.exists(piddir): + os.makedirs(piddir) + os.chown(piddir, + Bcfg2.Options.setup.daemon_uid, + Bcfg2.Options.setup.daemon_gid) + os.chmod(piddir, 493) # 0775 + if not self._daemonize(): + return False + + # rewrite $HOME. pulp stores its auth creds in ~/.pulp, so + # this is necessary to make that work when privileges are + # dropped + os.environ['HOME'] = \ + pwd.getpwuid(Bcfg2.Options.setup.daemon_uid)[5] + else: + os.umask(int(Bcfg2.Options.setup.umask, 8)) + + Core.run(self) + + def authenticate(self, cert, user, password, address): + """ Authenticate a client connection with + :func:`Bcfg2.Server.Plugin.interfaces.Metadata.AuthenticateConnection`. + + :param cert: an x509 certificate + :type cert: dict + :param user: The username of the user trying to authenticate + :type user: string + :param password: The password supplied by the client + :type password: string + :param address: An address pair of ``(<ip address>, <port>)`` + :type address: tuple + :return: bool - True if the authenticate succeeds, False otherwise + """ + if self.ca: + acert = cert + else: + # No ca, so no cert validation can be done + acert = None + return self.metadata.AuthenticateConnection(acert, user, password, + address) + + def _daemonize(self): + """ Daemonize the server and write the pidfile. This must be + overridden by a core implementation. """ + raise NotImplementedError diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py index 797b44ab9..7e1294587 100755 --- a/src/lib/Bcfg2/Server/Encryption.py +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -3,10 +3,17 @@ for handling encryption in Bcfg2. See :ref:`server-encryption` for more details. """ import os +import sys +import copy +import logging +import lxml.etree +import Bcfg2.Logger import Bcfg2.Options from M2Crypto import Rand from M2Crypto.EVP import Cipher, EVPError -from Bcfg2.Compat import StringIO, md5, b64encode, b64decode +from Bcfg2.Utils import safe_input +from Bcfg2.Server import XMLParser +from Bcfg2.Compat import md5, b64encode, b64decode, StringIO #: Constant representing the encryption operation for #: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This @@ -23,26 +30,22 @@ DECRYPT = 0 #: automated fashion. IV = r'\0' * 16 -#: The config file section encryption options and passphrases are -#: stored in -CFG_SECTION = "encryption" -#: The config option used to store the algorithm -CFG_ALGORITHM = "algorithm" +class _OptionContainer(object): + options = [ + Bcfg2.Options.BooleanOption( + cf=("encryption", "lax_decryption"), + help="Decryption failures should cause warnings, not errors"), + Bcfg2.Options.Option( + cf=("encryption", "algorithm"), default="aes_256_cbc", + type=lambda v: v.lower().replace("-", "_"), + help="The encryption algorithm to use"), + Bcfg2.Options.Option( + cf=("encryption", "*"), dest='passphrases', default=dict(), + help="Encryption passphrases")] -#: The config option used to store the decryption strictness -CFG_DECRYPT = "decrypt" - -#: Default cipher algorithm. To get a full list of valid algorithms, -#: you can run:: -#: -#: openssl list-cipher-algorithms | grep -v ' => ' | \ -#: tr 'A-Z-' 'a-z_' | sort -u -ALGORITHM = Bcfg2.Options.get_option_parser().cfp.get( # pylint: disable=E1103 - CFG_SECTION, - CFG_ALGORITHM, - default="aes_256_cbc").lower().replace("-", "_") +Bcfg2.Options.get_parser().add_component(_OptionContainer) Rand.rand_seed(os.urandom(1024)) @@ -64,7 +67,7 @@ def _cipher_filter(cipher, instr): return rv -def str_encrypt(plaintext, key, iv=IV, algorithm=ALGORITHM, salt=None): +def str_encrypt(plaintext, key, iv=IV, algorithm=None, salt=None): """ Encrypt a string with a key. For a higher-level encryption interface, see :func:`ssl_encrypt`. @@ -80,11 +83,13 @@ def str_encrypt(plaintext, key, iv=IV, algorithm=ALGORITHM, salt=None): :type salt: string :returns: string - The decrypted data """ + if algorithm is None: + algorithm = Bcfg2.Options.setup.algorithm cipher = Cipher(alg=algorithm, key=key, iv=iv, op=ENCRYPT, salt=salt) return _cipher_filter(cipher, plaintext) -def str_decrypt(crypted, key, iv=IV, algorithm=ALGORITHM): +def str_decrypt(crypted, key, iv=IV, algorithm=None): """ Decrypt a string with a key. For a higher-level decryption interface, see :func:`ssl_decrypt`. @@ -98,11 +103,13 @@ def str_decrypt(crypted, key, iv=IV, algorithm=ALGORITHM): :type algorithm: string :returns: string - The decrypted data """ + if algorithm is None: + algorithm = Bcfg2.Options.setup.algorithm cipher = Cipher(alg=algorithm, key=key, iv=iv, op=DECRYPT) return _cipher_filter(cipher, crypted) -def ssl_decrypt(data, passwd, algorithm=ALGORITHM): +def ssl_decrypt(data, passwd, algorithm=None): """ Decrypt openssl-encrypted data. This can decrypt data encrypted by :func:`ssl_encrypt`, or ``openssl enc``. It performs a base64 decode first if the data is base64 encoded, and @@ -132,7 +139,7 @@ def ssl_decrypt(data, passwd, algorithm=ALGORITHM): return str_decrypt(data[16:], key=key, iv=iv, algorithm=algorithm) -def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): +def ssl_encrypt(plaintext, passwd, algorithm=None, salt=None): """ Encrypt data in a format that is openssl compatible. :param plaintext: The plaintext data to encrypt @@ -164,25 +171,10 @@ def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): return b64encode("Salted__" + salt + crypted) + "\n" -def get_passphrases(): - """ Get all candidate encryption passphrases from the config file. - - :returns: dict - a dict of ``<passphrase name>``: ``<passphrase>`` - """ - setup = Bcfg2.Options.get_option_parser() - if setup.cfp.has_section(CFG_SECTION): - return dict([(o, setup.cfp.get(CFG_SECTION, o)) - for o in setup.cfp.options(CFG_SECTION) - if o not in [CFG_ALGORITHM, CFG_DECRYPT]]) - else: - return dict() - - -def bruteforce_decrypt(crypted, passphrases=None, algorithm=ALGORITHM): +def bruteforce_decrypt(crypted, passphrases=None, algorithm=None): """ Convenience method to decrypt the given encrypted string by - trying the given passphrases or all passphrases (as returned by - :func:`get_passphrases`) sequentially until one is found that - works. + trying the given passphrases or all passphrases sequentially until + one is found that works. :param crypted: The data to decrypt :type crypted: string @@ -194,10 +186,413 @@ def bruteforce_decrypt(crypted, passphrases=None, algorithm=ALGORITHM): :raises: :class:`M2Crypto.EVP.EVPError`, if the data cannot be decrypted """ if passphrases is None: - passphrases = get_passphrases().values() + passphrases = Bcfg2.Options.setup.passphrases.values() for passwd in passphrases: try: return ssl_decrypt(crypted, passwd, algorithm=algorithm) except EVPError: pass raise EVPError("Failed to decrypt") + + +class PassphraseError(Exception): + """ Exception raised when there's a problem determining the + passphrase to encrypt or decrypt with """ + + +class CryptoTool(object): + """ Generic decryption/encryption interface base object """ + + def __init__(self, filename): + self.logger = logging.getLogger(self.__class__.__name__) + self.filename = filename + self.data = open(self.filename).read() + self.pname, self.passphrase = self._get_passphrase() + + def _get_passphrase(self): + """ get the passphrase for the current file """ + if not Bcfg2.Options.setup.passphrases: + raise PassphraseError("No passphrases available in %s" % + Bcfg2.Options.setup.configfile) + + pname = None + if Bcfg2.Options.setup.passphrase: + pname = Bcfg2.Options.setup.passphrase + + if pname: + try: + passphrase = Bcfg2.Options.setup.passphrases[pname] + self.logger.debug("Using passphrase %s specified on command " + "line" % pname) + return (pname, passphrase) + except KeyError: + raise PassphraseError("Could not find passphrase %s in %s" % + (pname, Bcfg2.Options.setup.configfile)) + else: + if len(Bcfg2.Options.setup.passphrases) == 1: + pname, passphrase = Bcfg2.Options.setup.passphrases.items()[0] + self.logger.info("Using passphrase %s" % pname) + return (pname, passphrase) + elif len(Bcfg2.Options.setup.passphrases) > 1: + return (None, None) + raise PassphraseError("No passphrase could be determined") + + def get_destination_filename(self, original_filename): + """ Get the filename where data should be written """ + return original_filename + + def write(self, data): + """ write data to disk """ + new_fname = self.get_destination_filename(self.filename) + try: + self._write(new_fname, data) + self.logger.info("Wrote data to %s" % new_fname) + return True + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error writing data from %s to %s: %s" % + (self.filename, new_fname, err)) + return False + + def _write(self, filename, data): + """ Perform the actual write of data. This is separate from + :func:`CryptoTool.write` so it can be easily + overridden. """ + open(filename, "wb").write(data) + + +class Decryptor(CryptoTool): + """ Decryptor interface """ + def decrypt(self): + """ decrypt the file, returning the encrypted data """ + raise NotImplementedError + + +class Encryptor(CryptoTool): + """ encryptor interface """ + def encrypt(self): + """ encrypt the file, returning the encrypted data """ + raise NotImplementedError + + +class CfgEncryptor(Encryptor): + """ encryptor class for Cfg files """ + + def __init__(self, filename): + Encryptor.__init__(self, filename) + if self.passphrase is None: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + Bcfg2.Options.setup.configfile) + + def encrypt(self): + return ssl_encrypt(self.data, self.passphrase) + + def get_destination_filename(self, original_filename): + return original_filename + ".crypt" + + +class CfgDecryptor(Decryptor): + """ Decrypt Cfg files """ + + def decrypt(self): + """ decrypt the given file, returning the plaintext data """ + if self.passphrase: + try: + return ssl_decrypt(self.data, self.passphrase) + except EVPError: + self.logger.info("Could not decrypt %s with the " + "specified passphrase" % self.filename) + return False + except: + err = sys.exc_info()[1] + self.logger.error("Error decrypting %s: %s" % + (self.filename, err)) + return False + else: # no passphrase given, brute force + try: + return bruteforce_decrypt(self.data) + except EVPError: + self.logger.info("Could not decrypt %s with any passphrase" % + self.filename) + return False + + def get_destination_filename(self, original_filename): + if original_filename.endswith(".crypt"): + return original_filename[:-6] + else: + return Decryptor.get_destination_filename(self, original_filename) + + +class PropertiesCryptoMixin(object): + """ Mixin to provide some common methods for Properties crypto """ + default_xpath = '//*' + + def _get_elements(self, xdata): + """ Get the list of elements to encrypt or decrypt """ + if Bcfg2.Options.setup.xpath: + elements = xdata.xpath(Bcfg2.Options.setup.xpath) + if not elements: + self.logger.warning("XPath expression %s matched no elements" % + Bcfg2.Options.setup.xpath) + else: + elements = xdata.xpath(self.default_xpath) + if not elements: + elements = list(xdata.getiterator(tag=lxml.etree.Element)) + + # filter out elements without text data + for el in elements[:]: + if not el.text: + elements.remove(el) + + if Bcfg2.Options.setup.interactive: + for element in elements[:]: + if len(element): + elt = copy.copy(element) + for child in elt.iterchildren(): + elt.remove(child) + else: + elt = element + print(lxml.etree.tostring( + elt, + xml_declaration=False).decode("UTF-8").strip()) + ans = safe_input("Encrypt this element? [y/N] ") + if not ans.lower().startswith("y"): + elements.remove(element) + return elements + + def _get_element_passphrase(self, element): + """ Get the passphrase to use to encrypt or decrypt a given + element """ + pname = element.get("encrypted") + if pname in self.passphrases: + passphrase = self.passphrases[pname] + elif self.passphrase: + if pname: + self.logger.warning("Passphrase %s not found in %s, " + "using passphrase given on command line" % + (pname, Bcfg2.Option.setup.configfile)) + passphrase = self.passphrase + pname = self.pname + else: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + Bcfg2.Options.setup.configfile) + return (pname, passphrase) + + def _write(self, filename, data): + """ Write the data """ + data.getroottree().write(filename, + xml_declaration=False, + pretty_print=True) + + +class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): + """ encryptor class for Properties files """ + + def encrypt(self): + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + return False + elt.text = ssl_encrypt(elt.text, passphrase).strip() + elt.set("encrypted", pname) + return xdata + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) + + +class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): + """ decryptor class for Properties files """ + default_xpath = '//*[@encrypted]' + + def decrypt(self): + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + return False + decrypted = ssl_decrypt(elt.text, passphrase).strip() + try: + elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') + elt.set("encrypted", pname) + except UnicodeDecodeError: + # we managed to decrypt the value, but it contains + # content that can't even be encoded into xml + # entities. what probably happened here is that we + # coincidentally could decrypt a value encrypted with + # a different key, and wound up with gibberish. + self.logger.warning("Decrypted %s to gibberish, skipping" % + elt.tag) + return xdata + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) + + +class CLI(object): + """ The bcfg2-crypt CLI """ + + options = [ + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption( + "--encrypt", help='Encrypt the specified file'), + Bcfg2.Options.BooleanOption( + "--decrypt", help='Decrypt the specified file')), + Bcfg2.Options.BooleanOption( + "--stdout", + help='Decrypt or encrypt the specified file to stdout'), + Bcfg2.Options.Option( + "-p", "--passphrase", metavar="NAME", + help='Encryption passphrase name'), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption( + "--properties", + help='Encrypt the specified file as a Properties file'), + Bcfg2.Options.BooleanOption( + "--cfg", help='Encrypt the specified file as a Cfg file')), + Bcfg2.Options.OptionGroup( + Bcfg2.Options.Common.interactive, + Bcfg2.Options.Option( + "--xpath", + help='XPath expression to select elements to encrypt'), + title="Options for handling Properties files"), + Bcfg2.Options.OptionGroup( + Bcfg2.Options.BooleanOption( + "--remove", help='Remove the plaintext file after encrypting'), + title="Options for handling Cfg files"), + Bcfg2.Options.PathOption( + "files", help="File(s) to encrypt or decrypt", nargs='+')] + + def __init__(self): + parser = Bcfg2.Options.get_parser( + description="Encrypt and decrypt Bcfg2 data", + components=[self, _OptionContainer]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + if Bcfg2.Options.setup.decrypt: + if Bcfg2.Options.setup.remove: + self.logger.error("--remove cannot be used with --decrypt, " + "ignoring --remove") + Bcfg2.Options.setup.remove = False + elif Bcfg2.Options.setup.interactive: + self.logger.error("Cannot decrypt interactively") + Bcfg2.Options.setup.interactive = False + + def _is_properties(self, filename): + """ Determine if a given file is a Properties file or not """ + if Bcfg2.Options.setup.properties: + return True + elif Bcfg2.Options.setup.cfg: + return False + elif filename.endswith(".xml"): + try: + xroot = lxml.etree.parse(filename).getroot() + return xroot.tag == "Properties" + except lxml.etree.XMLSyntaxError: + return False + else: + return False + + def run(self): # pylint: disable=R0912,R0915 + for fname in Bcfg2.Options.setup.files: + if not os.path.exists(fname): + self.logger.error("%s does not exist, skipping" % fname) + continue + + # figure out if we need to encrypt this as a Properties file + # or as a Cfg file + try: + props = self._is_properties(fname) + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + + if props: + if Bcfg2.Options.setup.remove: + self.logger.info("Cannot use --remove with Properties " + "file %s, ignoring for this file" % fname) + try: + tools = (PropertiesEncryptor(fname), + PropertiesDecryptor(fname)) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + continue + except IOError: + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + else: + if Bcfg2.Options.setup.xpath: + self.logger.error("Specifying --xpath with --cfg is " + "nonsensical, ignoring --xpath") + Bcfg2.Options.setup.xpath = None + if Bcfg2.Options.setup.interactive: + self.logger.error("Cannot use interactive mode with " + "--cfg, ignoring --interactive") + Bcfg2.Options.setup.interactive = False + try: + tools = (CfgEncryptor(fname), CfgDecryptor(fname)) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + continue + except IOError: + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + + data = None + mode = None + if Bcfg2.Options.setup.encrypt: + tool = tools[0] + mode = "encrypt" + elif Bcfg2.Options.setup.decrypt: + tool = tools[1] + mode = "decrypt" + else: + self.logger.info("Neither --encrypt nor --decrypt specified, " + "determining mode") + tool = tools[1] + try: + data = tool.decrypt() + mode = "decrypt" + except: # pylint: disable=W0702 + pass + if data is False: + data = None + self.logger.info("Failed to decrypt %s, trying encryption" + % fname) + tool = tools[0] + mode = "encrypt" + + if data is None: + data = getattr(tool, mode)() + if not data: + self.logger.error("Failed to %s %s, skipping" % (mode, fname)) + continue + if Bcfg2.Options.setup.stdout: + if len(Bcfg2.Options.setup.files) > 1: + print("----- %s -----" % fname) + print(data) + if len(Bcfg2.Options.setup.files) > 1: + print("") + else: + tool.write(data) + + if (Bcfg2.Options.setup.remove and + tool.get_destination_filename(fname) != fname): + try: + os.unlink(fname) + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error removing %s: %s" % (fname, err)) + continue diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 9134758b8..69463ab4c 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -33,8 +33,8 @@ class Gamin(FileMonitor): #: releases, so it has a fairly high priority. __priority__ = 90 - def __init__(self, ignore=None, debug=False): - FileMonitor.__init__(self, ignore=ignore, debug=debug) + def __init__(self): + FileMonitor.__init__(self) #: The :class:`Gamin.WatchMonitor` object for this monitor. self.mon = None diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index 2cdf27ed8..39d062604 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -34,8 +34,8 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): #: listed in :attr:`action_map` mask = reduce(lambda x, y: x | y, action_map.keys()) - def __init__(self, ignore=None, debug=False): - Pseudo.__init__(self, ignore=ignore, debug=debug) + def __init__(self): + Pseudo.__init__(self) pyinotify.ProcessEvent.__init__(self) #: inotify can't set useful monitors directly on files, only diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 522ddb705..ae42a3429 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -48,11 +48,9 @@ Base Classes import os import sys import fnmatch -import logging +import Bcfg2.Options from time import sleep, time -from Bcfg2.Server.Plugin import Debuggable - -LOGGER = logging.getLogger(__name__) +from Bcfg2.Logger import Debuggable class Event(object): @@ -112,6 +110,14 @@ class FileMonitor(Debuggable): monitor objects to :attr:`handles` and received events to :attr:`events`; the basic interface will handle the rest. """ + options = [ + Bcfg2.Options.Option( + cf=('server', 'ignore_files'), + help='File globs to ignore', + type=Bcfg2.Options.Types.comma_list, + default=['*~', '*#', '.#*', '*.swp', '*.swpx', '.*.swx', + 'SCCS', '.svn', '4913', '.gitignore'])] + #: The relative priority of this FAM backend. Better backends #: should have higher priorities. __priority__ = -1 @@ -119,7 +125,7 @@ class FileMonitor(Debuggable): #: List of names of methods to be exposed as XML-RPC functions __rmi__ = Debuggable.__rmi__ + ["list_event_handlers"] - def __init__(self, ignore=None, debug=False): + def __init__(self): """ :param ignore: A list of filename globs describing events that should be ignored (i.e., not processed by any @@ -133,7 +139,6 @@ class FileMonitor(Debuggable): .. autoattribute:: __priority__ """ Debuggable.__init__(self) - self.debug_flag = debug #: A dict that records which objects handle which events. #: Keys are monitor handle IDs and values are objects whose @@ -143,12 +148,10 @@ class FileMonitor(Debuggable): #: Queue of events to handle self.events = [] - if ignore is None: - ignore = [] #: List of filename globs to ignore events for. For events #: that include the full path, both the full path and the bare #: filename will be checked against ``ignore``. - self.ignore = ignore + self.ignore = Bcfg2.Options.setup.ignore_files #: Whether or not the FAM has been started. See :func:`start`. self.started = False @@ -226,8 +229,8 @@ class FileMonitor(Debuggable): if self.should_ignore(event): return if event.requestID not in self.handles: - LOGGER.info("Got event for unexpected id %s, file %s" % - (event.requestID, event.filename)) + self.logger.info("Got event for unexpected id %s, file %s" % + (event.requestID, event.filename)) return self.debug_log("Dispatching event %s %s to obj %s" % (event.code2str(), event.filename, @@ -236,8 +239,8 @@ class FileMonitor(Debuggable): self.handles[event.requestID].HandleEvent(event) except: # pylint: disable=W0702 err = sys.exc_info()[1] - LOGGER.error("Error in handling of event %s for %s: %s" % - (event.code2str(), event.filename, err)) + self.logger.error("Error in handling of event %s for %s: %s" % + (event.code2str(), event.filename, err)) def handle_event_set(self, lock=None): """ Handle all pending events. @@ -263,7 +266,8 @@ class FileMonitor(Debuggable): lock.release() end = time() if count > 0: - LOGGER.info("Handled %d events in %.03fs" % (count, (end - start))) + self.logger.info("Handled %d events in %.03fs" % (count, + (end - start))) def handle_events_in_interval(self, interval): """ Handle events for the specified period of time (in @@ -330,37 +334,17 @@ class FileMonitor(Debuggable): _FAM = None -def load_fam(filemonitor='default', ignore=None, debug=False): - """ Load a new :class:`Bcfg2.Server.FileMonitor.FileMonitor` - object, caching it in :attr:`_FAM` for later retrieval via - :func:`get_fam`. - - :param filemonitor: Which filemonitor backend to use - :type filemonitor: string - :param ignore: A list of filenames to ignore - :type ignore: list of strings (filename globs) - :param debug: Produce debugging information - :type debug: bool - :returns: :class:`Bcfg2.Server.FileMonitor.FileMonitor` - """ - global _FAM # pylint: disable=W0603 - if _FAM is None: - if ignore is None: - ignore = [] - _FAM = available[filemonitor](ignore=ignore, debug=debug) - return _FAM - - def get_fam(): - """ Get an already-created + """ Get a :class:`Bcfg2.Server.FileMonitor.FileMonitor` object. If :attr:`_FAM` has not been populated, then a new default FileMonitor will be created. :returns: :class:`Bcfg2.Server.FileMonitor.FileMonitor` """ + global _FAM # pylint: disable=W0603 if _FAM is None: - return load_fam('default') + _FAM = Bcfg2.Options.setup.filemonitor() return _FAM diff --git a/src/lib/Bcfg2/Server/Info.py b/src/lib/Bcfg2/Server/Info.py new file mode 100644 index 000000000..24d7cc637 --- /dev/null +++ b/src/lib/Bcfg2/Server/Info.py @@ -0,0 +1,870 @@ +""" Subcommands and helpers for bcfg2-info """ +# -*- coding: utf-8 -*- + +import os +import sys +import cmd +import math +import time +import copy +import pipes +import fnmatch +import argparse +import operator +import lxml.etree +from code import InteractiveConsole +import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.Server.Core +import Bcfg2.Server.Plugin +import Bcfg2.Client.Tools.POSIX +from Bcfg2.Compat import any # pylint: disable=W0622 + +try: + try: + import cProfile as profile + except ImportError: + import profile + import pstats + HAS_PROFILE = True +except ImportError: + HAS_PROFILE = False + + +def print_tabular(rows): + """Print data in tabular format.""" + cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 + for index in range(len(rows[0]))]) + fstring = (" %%-%ss |" * len(cmax)) % cmax + fstring = ('|'.join([" %%-%ss "] * len(cmax))) % cmax + print(fstring % rows[0]) + print((sum(cmax) + (len(cmax) * 2) + (len(cmax) - 1)) * '=') + for row in rows[1:]: + print(fstring % row) + + +def display_trace(trace): + """ display statistics from a profile trace """ + stats = pstats.Stats(trace) + stats.sort_stats('cumulative', 'calls', 'time') + stats.print_stats(200) + + +def load_interpreters(): + """ Load a dict of available Python interpreters """ + interpreters = dict(python=lambda v: InteractiveConsole(v).interact()) + default = "python" + try: + import bpython.cli + interpreters["bpython"] = lambda v: bpython.cli.main(args=[], + locals_=v) + default = "bpython" + except ImportError: + pass + + try: + # whether ipython is actually better than bpython is + # up for debate, but this is the behavior that existed + # before --interpreter was added, so we call IPython + # better + import IPython + # pylint: disable=E1101 + if hasattr(IPython, "Shell"): + interpreters["ipython"] = lambda v: \ + IPython.Shell.IPShell(argv=[], user_ns=v).mainloop() + default = "ipython" + elif hasattr(IPython, "embed"): + interpreters["ipython"] = lambda v: IPython.embed(user_ns=v) + default = "ipython" + else: + print("Unknown IPython API version") + # pylint: enable=E1101 + except ImportError: + pass + + return (interpreters, default) + + +class InfoCmd(Bcfg2.Options.Subcommand): + """ Base class for bcfg2-info subcommands """ + + def _expand_globs(self, globs, candidates): + # special cases to speed things up: + if globs is None or '*' in globs: + return candidates + has_wildcards = False + for glob in globs: + # check if any wildcard characters are in the string + if set('*?[]') & set(glob): + has_wildcards = True + break + if not has_wildcards: + return globs + + rv = set() + cset = set(candidates) + for glob in globs: + rv.update(c for c in cset if fnmatch.fnmatch(c, glob)) + cset.difference_update(rv) + return list(rv) + + def get_client_list(self, globs): + """ given a list of host globs, get a list of clients that + match them """ + return self._expand_globs(globs, self.core.metadata.clients) + + def get_group_list(self, globs): + """ given a list of group glob, get a list of groups that + match them""" + # special cases to speed things up: + return self._expand_globs(globs, + list(self.core.metadata.groups.keys())) + + +class Help(InfoCmd, Bcfg2.Options.HelpCommand): + """ Get help on a specific subcommand """ + def command_registry(self): + return self.core.commands + + +class Debug(InfoCmd): + """ Shell out to a Python interpreter """ + interpreters, default_interpreter = load_interpreters() + options = [ + Bcfg2.Options.BooleanOption( + "-n", "--non-interactive", + help="Do not enter the interactive debugger"), + Bcfg2.Options.PathOption( + "-f", dest="cmd_list", type=argparse.FileType('r'), + help="File containing commands to run"), + Bcfg2.Options.Option( + "--interpreter", cf=("bcfg2-info", "interpreter"), + env="BCFG2_INFO_INTERPRETER", + choices=interpreters.keys(), default=default_interpreter)] + + def run(self, setup): + if setup.cmd_list: + console = InteractiveConsole(locals()) + for command in setup.cmd_list.readlines(): + command = command.strip() + if command: + console.push(command) + if not setup.non_interactive: + print("Dropping to interpreter; press ^D to resume") + self.interpreters[setup.interpreter](self.core.get_locals()) + + +class Build(InfoCmd): + """ Build config for hostname, writing to filename """ + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("filename", nargs='?', + default=sys.stdout, + type=argparse.FileType('w'))] + + def run(self, setup): + etree = lxml.etree.ElementTree( + self.core.BuildConfiguration(setup.hostname)) + try: + etree.write( + setup.filename, + encoding='UTF-8', xml_declaration=True, + pretty_print=True) + except IOError: + err = sys.exc_info()[1] + print("Failed to write %s: %s" % (setup.filename, err)) + + +class Builddir(InfoCmd): + """ Build config for hostname, writing separate files to directory + """ + + # don't try to isntall these types of entries + blacklisted_types = ["nonexistent", "permissions"] + + options = Bcfg2.Client.Tools.POSIX.POSIX.options + [ + Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PathOption("directory")] + + help = """Generates a config for client <hostname> and writes the +individual configuration files out separately in a tree under <output +dir>. This only handles file entries, and does not respect 'owner' or +'group' attributes unless run as root. """ + + def run(self, setup): + setup.paranoid = False + client_config = self.core.BuildConfiguration(setup.hostname) + if client_config.tag == 'error': + print("Building client configuration failed.") + return 1 + + entries = [] + for struct in client_config: + for entry in struct: + if (entry.tag == 'Path' and + entry.get("type") not in self.blacklisted_types): + failure = entry.get("failure") + if failure is not None: + print("Skipping entry %s:%s with bind failure: %s" % + (entry.tag, entry.get("name"), failure)) + continue + entry.set('name', + os.path.join(setup.directory, + entry.get('name').lstrip("/"))) + entries.append(entry) + + Bcfg2.Client.Tools.POSIX.POSIX(client_config).Install(entries) + + +class Buildfile(InfoCmd): + """ Build config file for hostname """ + + options = [ + Bcfg2.Options.Option("-f", "--outfile", metavar="<path>", + type=argparse.FileType('w'), default=sys.stdout), + Bcfg2.Options.PathOption("--altsrc"), + Bcfg2.Options.PathOption("filename"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + entry = lxml.etree.Element('Path', name=setup.filename) + if setup.altsrc: + entry.set("altsrc", setup.altsrc) + try: + self.core.Bind(entry, self.core.build_metadata(setup.hostname)) + except: # pylint: disable=W0702 + print("Failed to build entry %s for host %s" % (setup.filename, + setup.hostname)) + raise + try: + setup.outfile.write( + lxml.etree.tostring(entry, + xml_declaration=False).decode('UTF-8')) + setup.outfile.write("\n") + except IOError: + err = sys.exc_info()[1] + print("Failed to write %s: %s" % (setup.outfile.name, err)) + + +class BuildAllMixin(object): + """ InfoCmd mixin to make a version of an existing command that + applies to multiple hosts""" + + directory_arg = Bcfg2.Options.PathOption("directory") + hostname_arg = Bcfg2.Options.PositionalArgument("hostname", nargs='*', + default=[]) + options = [directory_arg, hostname_arg] + + @property + def _parent(self): + """ the parent command """ + for cls in self.__class__.__mro__: + if (cls != InfoCmd and cls != self.__class__ and + issubclass(cls, InfoCmd)): + return cls + + def run(self, setup): + try: + os.makedirs(setup.directory) + except OSError: + err = sys.exc_info()[1] + if err.errno != 17: + print("Could not create %s: %s" % (setup.directory, err)) + return 1 + clients = self.get_client_list(setup.hostname) + for client in clients: + csetup = self._get_setup(client, copy.copy(setup)) + csetup.hostname = client + self._parent.run(self, csetup) # pylint: disable=E1101 + + def _get_setup(self, client, setup): + raise NotImplementedError + + +class Buildallfile(Buildfile, BuildAllMixin): + """ Build config file for all clients in directory """ + + options = [BuildAllMixin.directory_arg, + Bcfg2.Options.PathOption("--altsrc"), + Bcfg2.Options.PathOption("filename"), + BuildAllMixin.hostname_arg] + + def run(self, setup): + BuildAllMixin.run(self, setup) + + def _get_setup(self, client, setup): + setup.outfile = open(os.path.join(setup.directory, client), 'w') + return setup + + +class Buildall(Build, BuildAllMixin): + """ Build configs for all clients in directory """ + + options = BuildAllMixin.options + + def run(self, setup): + BuildAllMixin.run(self, setup) + + def _get_setup(self, client, setup): + setup.filename = os.path.join(setup.directory, client + ".xml") + return setup + + +class Buildbundle(InfoCmd): + """ Render a templated bundle for hostname """ + + options = [Bcfg2.Options.PositionalArgument("bundle"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + bundler = self.core.plugins['Bundler'] + bundle = None + if setup.bundle in bundler.entries: + bundle = bundler.entries[setup.bundle] + elif not setup.bundle.endswith(".xml"): + fname = setup.bundle + ".xml" + if fname in bundler.entries: + bundle = bundler.entries[bundle] + if not bundle: + print("No such bundle %s" % setup.bundle) + return 1 + try: + metadata = self.core.build_metadata(setup.hostname) + print(lxml.etree.tostring(bundle.XMLMatch(metadata), + xml_declaration=False, + pretty_print=True).decode('UTF-8')) + except: # pylint: disable=W0702 + print("Failed to render bundle %s for host %s: %s" % + (setup.bundle, client, sys.exc_info()[1])) + raise + + +class Automatch(InfoCmd): + """ Perform automatch on a Properties file """ + + options = [ + Bcfg2.Options.BooleanOption( + "-f", "--force", + help="Force automatch even if it's disabled"), + Bcfg2.Options.PositionalArgument("propertyfile"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + props = self.core.plugins['Properties'] + except KeyError: + print("Properties plugin not enabled") + return 1 + + pfile = props.entries[setup.propertyfile] + if (not Bcfg2.Options.setup.force and + not Bcfg2.Options.setup.automatch and + pfile.xdata.get("automatch", "false").lower() != "true"): + print("Automatch not enabled on %s" % setup.propertyfile) + else: + metadata = self.core.build_metadata(setup.hostname) + print(lxml.etree.tostring(pfile.XMLMatch(metadata), + xml_declaration=False, + pretty_print=True).decode('UTF-8')) + + +class ExpireCache(InfoCmd): + """ Expire the metadata cache """ + + options = [ + Bcfg2.Options.PositionalArgument( + "hostname", nargs="*", default=[], + help="Expire cache for the given host(s)")] + + def run(self, setup): + if setup.clients: + for client in self.get_client_list(setup.clients): + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) + else: + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + + +class Bundles(InfoCmd): + """ Print out group/bundle info """ + + options = [Bcfg2.Options.PositionalArgument("group", nargs='*')] + + def run(self, setup): + data = [('Group', 'Bundles')] + groups = self.get_group_list(setup.group) + groups.sort() + for group in groups: + data.append((group, + ','.join(self.core.metadata.groups[group][0]))) + print_tabular(data) + + +class Clients(InfoCmd): + """ Print out client/profile info """ + + options = [Bcfg2.Options.PositionalArgument("hostname", nargs='*', + default=[])] + + def run(self, setup): + data = [('Client', 'Profile')] + for client in sorted(self.get_client_list(setup.hostname)): + imd = self.core.metadata.get_initial_metadata(client) + data.append((client, imd.profile)) + print_tabular(data) + + +class Config(InfoCmd): + """ Print out the current configuration of Bcfg2""" + + options = [ + Bcfg2.Options.BooleanOption( + "--raw", + help="Produce more accurate but less readable raw output")] + + def run(self, setup): + parser = Bcfg2.Options.get_parser() + data = [('Description', 'Value')] + for option in parser.option_list: + if hasattr(setup, option.dest): + value = getattr(setup, option.dest) + if any(issubclass(a.__class__, + Bcfg2.Options.ComponentAction) + for a in option.actions.values()): + if not setup.raw: + try: + if option.action.islist: + value = [v.__name__ for v in value] + else: + value = value.__name__ + except AttributeError: + # just use the value as-is + pass + if setup.raw: + value = repr(value) + data.append((getattr(option, "help", option.dest), value)) + print_tabular(data) + + +class Probes(InfoCmd): + """ Get probes for the given host """ + + options = [ + Bcfg2.Options.BooleanOption("-p", "--pretty", + help="Human-readable output"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + if setup.pretty: + probes = [] + else: + probes = lxml.etree.Element('probes') + metadata = self.core.build_metadata(setup.hostname) + for plugin in self.core.plugins_by_type(Bcfg2.Server.Plugin.Probing): + for probe in plugin.GetProbes(metadata): + probes.append(probe) + if setup.pretty: + for probe in probes: + pname = probe.get("name") + print("=" * (len(pname) + 2)) + print(" %s" % pname) + print("=" * (len(pname) + 2)) + print("") + print(probe.text) + print("") + else: + print(lxml.etree.tostring(probes, xml_declaration=False, + pretty_print=True).decode('UTF-8')) + + +class Showentries(InfoCmd): + """ Show abstract configuration entries for a given host """ + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("type", nargs='?')] + + def run(self, setup): + try: + metadata = self.core.build_metadata(setup.hostname) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Unable to build metadata for %s: %s" % (setup.hostname, + sys.exc_info()[1])) + structures = self.core.GetStructures(metadata) + output = [('Entry Type', 'Name')] + etypes = None + if setup.type: + etypes = [setup.type, "Bound%s" % setup.type] + for item in structures: + output.extend((child.tag, child.get('name')) + for child in item.getchildren() + if not etypes or child.tag in etypes) + print_tabular(output) + + +class Groups(InfoCmd): + """ Print out group info """ + options = [Bcfg2.Options.PositionalArgument("group", nargs='*')] + + def _profile_flag(self, group): + if self.core.metadata.groups[group].is_profile: + return 'yes' + else: + return 'no' + + def run(self, setup): + data = [("Groups", "Profile", "Category")] + groups = self.get_group_list(setup.group) + groups.sort() + for group in groups: + data.append((group, + self._profile_flag(group), + self.core.metadata.groups[group].category)) + print_tabular(data) + + +class Showclient(InfoCmd): + """ Show metadata for the given hosts """ + + options = [Bcfg2.Options.PositionalArgument("hostname", nargs='*')] + + def run(self, setup): + for client in self.get_client_list(setup.hostname): + try: + metadata = self.core.build_metadata(client) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Could not build metadata for %s: %s" % + (client, sys.exc_info()[1])) + continue + fmt = "%-10s %s" + print(fmt % ("Hostname:", metadata.hostname)) + print(fmt % ("Profile:", metadata.profile)) + + group_fmt = "%-10s %-30s %s" + header = False + for group in list(metadata.groups): + category = "" + for cat, grp in metadata.categories.items(): + if grp == group: + category = "Category: %s" % cat + break + if not header: + print(group_fmt % ("Groups:", group, category)) + header = True + else: + print(group_fmt % ("", group, category)) + + if metadata.bundles: + print(fmt % ("Bundles:", list(metadata.bundles)[0])) + for bnd in list(metadata.bundles)[1:]: + print(fmt % ("", bnd)) + if metadata.connectors: + print("Connector data") + print("=" * 80) + for conn in metadata.connectors: + if getattr(metadata, conn): + print(fmt % (conn + ":", getattr(metadata, conn))) + print("=" * 80) + + +class Mappings(InfoCmd): + """ Print generator mappings for optional type and name """ + + options = [Bcfg2.Options.PositionalArgument("type", nargs='?'), + Bcfg2.Options.PositionalArgument("name", nargs='?')] + + def run(self, setup): + data = [('Plugin', 'Type', 'Name')] + for generator in self.core.plugins_by_type( + Bcfg2.Server.Plugin.Generator): + etypes = setup.type or list(generator.Entries.keys()) + if setup.name: + interested = [(etype, [setup.name]) for etype in etypes] + else: + interested = [(etype, generator.Entries[etype]) + for etype in etypes + if etype in generator.Entries] + for etype, names in interested: + data.extend((generator.name, etype, name) + for name in names + if name in generator.Entries.get(etype, {})) + print_tabular(data) + + +class PackageResolve(InfoCmd): + """ Resolve packages for the given host""" + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("package", nargs="*")] + + def run(self, setup): + try: + pkgs = self.core.plugins['Packages'] + except KeyError: + print("Packages plugin not enabled") + return 1 + + metadata = self.core.build_metadata(setup.hostname) + + indep = lxml.etree.Element("Independent", + name=self.__class__.__name__.lower()) + if setup.package: + structures = [lxml.etree.Element("Bundle", name="packages")] + for package in setup.package: + lxml.etree.SubElement(structures[0], "Package", name=package) + else: + structures = self.core.GetStructures(metadata) + + pkgs._build_packages(metadata, indep, # pylint: disable=W0212 + structures) + print("%d new packages added" % len(indep.getchildren())) + if len(indep.getchildren()): + print(" %s" % "\n ".join(lxml.etree.tostring(p) + for p in indep.getchildren())) + + +class Packagesources(InfoCmd): + """ Show package sources """ + + options = [Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + pkgs = self.core.plugins['Packages'] + except KeyError: + print("Packages plugin not enabled") + return 1 + try: + metadata = self.core.build_metadata(setup.hostname) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Unable to build metadata for %s: %s" % (setup.hostname, + sys.exc_info()[1])) + return 1 + print(pkgs.get_collection(metadata).sourcelist()) + + +class Query(InfoCmd): + """ Query clients """ + + options = [ + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.Option( + "-g", "--group", metavar="<group>", dest="querygroups", + type=Bcfg2.Options.Types.comma_list), + Bcfg2.Options.Option( + "-p", "--profile", metavar="<profile>", dest="queryprofiles", + type=Bcfg2.Options.Types.comma_list), + Bcfg2.Options.Option( + "-b", "--bundle", metavar="<bundle>", dest="querybundles", + type=Bcfg2.Options.Types.comma_list), + required=True)] + + def run(self, setup): + if setup.queryprofiles: + res = self.core.metadata.get_client_names_by_profiles( + setup.queryprofiles) + elif setup.querygroups: + res = self.core.metadata.get_client_names_by_groups( + setup.querygroups) + elif setup.querybundles: + res = self.core.metadata.get_client_names_by_bundles( + setup.querybundles) + print("\n".join(res)) + + +class Shell(InfoCmd): + """ Open an interactive shell to run multiple bcfg2-info commands """ + interactive = False + + def run(self, setup): + try: + self.core.cmdloop('Welcome to bcfg2-info\n' + 'Type "help" for more information') + except KeyboardInterrupt: + print("Ctrl-C pressed, exiting...") + loop = False + + +class ProfileTemplates(InfoCmd): + """ Benchmark template rendering times """ + + options = [ + Bcfg2.Options.Option( + "--clients", type=Bcfg2.Options.Types.comma_list, + help="Benchmark templates for the named clients"), + Bcfg2.Options.Option( + "--runs", help="Number of rendering passes per template", + default=5, type=int), + Bcfg2.Options.PositionalArgument( + "templates", nargs="*", default=[], + help="Profile the named templates instead of all templates")] + + def profile_entry(self, entry, metadata, runs=5): + times = [] + for i in range(runs): # pylint: disable=W0612 + start = time.time() + try: + self.core.Bind(entry, metadata) + times.append(time.time() - start) + except: # pylint: disable=W0702 + break + if times: + avg = sum(times) / len(times) + if avg: + self.logger.debug(" %s: %.02f sec" % + (metadata.hostname, avg)) + return times + + def profile_struct(self, struct, metadata, templates=None, runs=5): + times = dict() + entries = struct.xpath("//Path") + entry_count = 0 + for entry in entries: + entry_count += 1 + if templates is None or entry.get("name") in templates: + self.logger.info("Rendering Path:%s (%s/%s)..." % + (entry.get("name"), entry_count, + len(entries))) + times.setdefault(entry.get("name"), + self.profile_entry(entry, metadata, + runs=runs)) + return times + + def profile_client(self, metadata, templates=None, runs=5): + structs = self.core.GetStructures(metadata) + struct_count = 0 + times = dict() + for struct in structs: + struct_count += 1 + self.logger.info("Rendering templates from structure %s:%s " + "(%s/%s)" % + (struct.tag, struct.get("name"), struct_count, + len(structs))) + times.update(self.profile_struct(struct, metadata, + templates=templates, runs=runs)) + return times + + def stdev(self, nums): + mean = float(sum(nums)) / len(nums) + return math.sqrt(sum((n - mean) ** 2 for n in nums) / float(len(nums))) + + def run(self, setup): + clients = self.get_client_list(setup.clients) + + times = dict() + client_count = 0 + for client in clients: + client_count += 1 + self.logger.info("Rendering templates for client %s (%s/%s)" % + (client, client_count, len(clients))) + times.update(self.profile_client(self.core.build_metadata(client), + templates=setup.templates, + runs=setup.runs)) + + # print out per-file results + tmpltimes = [] + for tmpl, ptimes in times.items(): + try: + mean = float(sum(ptimes)) / len(ptimes) + except ZeroDivisionError: + continue + ptimes.sort() + median = ptimes[len(ptimes) / 2] + std = self.stdev(ptimes) + if mean > 0.01 or median > 0.01 or std > 1 or setup.templates: + tmpltimes.append((tmpl, mean, median, std)) + print("%-50s %-9s %-11s %6s" % + ("Template", "Mean Time", "Median Time", "σ")) + for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): + print("%-50s %9.02f %11.02f %6.02f" % info) + + +if HAS_PROFILE: + class Profile(InfoCmd): + """ Profile a single bcfg2-info command """ + + options = [Bcfg2.Options.PositionalArgument("command"), + Bcfg2.Options.PositionalArgument("args", nargs="*")] + + def run(self, setup): + prof = profile.Profile() + cls = self.core.commands[setup.command] + prof.runcall(cls, " ".join(pipes.quote(a) for a in setup.args)) + display_trace(prof) + + +class InfoCore(cmd.Cmd, + Bcfg2.Server.Core.Core, + Bcfg2.Options.CommandRegistry): + """Main class for bcfg2-info.""" + + def __init__(self): + cmd.Cmd.__init__(self) + Bcfg2.Server.Core.Core.__init__(self) + Bcfg2.Options.CommandRegistry.__init__(self) + self.prompt = 'bcfg2-info> ' + + def get_locals(self): + return locals() + + def do_quit(self, _): + """ quit|exit - Exit program """ + raise SystemExit(0) + + do_EOF = do_quit + do_exit = do_quit + + def do_eventdebug(self, _): + """ eventdebug - Enable debugging output for FAM events """ + self.fam.set_debug(True) + + do_event_debug = do_eventdebug + + def do_update(self, _): + """ update - Process pending filesystem events """ + self.fam.handle_events_in_interval(0.1) + + def run(self): + self.load_plugins() + self.block_for_fam_events(handle_events=True) + + def _daemonize(self): + pass + + def _run(self): + pass + + def _block(self): + pass + + def shutdown(self): + Bcfg2.Options.CommandRegistry.shutdown(self) + Bcfg2.Server.Core.Core.shutdown(self) + + +class CLI(object): + options = [Bcfg2.Options.BooleanOption("-p", "--profile", help="Profile")] + + def __init__(self): + Bcfg2.Options.register_commands(InfoCore, globals().values(), + parent=InfoCmd) + parser = Bcfg2.Options.get_parser( + description="Inspect a running Bcfg2 server", + components=[self, InfoCore]) + parser.parse() + + if Bcfg2.Options.setup.profile and HAS_PROFILE: + prof = profile.Profile() + self.core = prof.runcall(InfoCore) + display_trace(prof) + else: + if Bcfg2.Options.setup.profile: + print("Profiling functionality not available.") + self.core = InfoCore() + + for command in self.core.commands.values(): + command.core = self.core + + def run(self): + if Bcfg2.Options.setup.subcommand != 'help': + self.core.run() + return self.core.runcommand() diff --git a/src/lib/Bcfg2/Server/Lint/Bundler.py b/src/lib/Bcfg2/Server/Lint/Bundler.py new file mode 100644 index 000000000..b41313349 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Bundler.py @@ -0,0 +1,55 @@ +from Bcfg2.Server.Lint import ServerPlugin + + +class Bundler(ServerPlugin): + """ Perform various :ref:`Bundler + <server-plugins-structures-bundler-index>` checks. """ + + def Run(self): + self.missing_bundles() + for bundle in self.core.plugins['Bundler'].entries.values(): + if self.HandlesFile(bundle.name): + self.bundle_names(bundle) + + @classmethod + def Errors(cls): + return {"bundle-not-found": "error", + "unused-bundle": "warning", + "explicit-bundle-name": "error", + "genshi-extension-bundle": "error"} + + def missing_bundles(self): + """ Find bundles listed in Metadata but not implemented in + Bundler. """ + if self.files is None: + # when given a list of files on stdin, this check is + # useless, so skip it + groupdata = self.metadata.groups_xml.xdata + ref_bundles = set([b.get("name") + for b in groupdata.findall("//Bundle")]) + + allbundles = self.core.plugins['Bundler'].bundles.keys() + for bundle in ref_bundles: + if bundle not in allbundles: + self.LintError("bundle-not-found", + "Bundle %s referenced, but does not exist" % + bundle) + + for bundle in allbundles: + if bundle not in ref_bundles: + self.LintError("unused-bundle", + "Bundle %s defined, but is not referenced " + "in Metadata" % bundle) + + def bundle_names(self, bundle): + """ Verify that deprecated bundle .genshi bundles and explicit + bundle names aren't used """ + if bundle.xdata.get('name'): + self.LintError("explicit-bundle-name", + "Deprecated explicit bundle name in %s" % + bundle.name) + + if bundle.name.endswith(".genshi"): + self.LintError("genshi-extension-bundle", + "Bundle %s uses deprecated .genshi extension" % + bundle.name) diff --git a/src/lib/Bcfg2/Server/Lint/Cfg.py b/src/lib/Bcfg2/Server/Lint/Cfg.py new file mode 100644 index 000000000..4cdf5c48a --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Cfg.py @@ -0,0 +1,95 @@ +import os +from Bcfg2.Server.Lint import ServerPlugin + + +class Cfg(ServerPlugin): + """ warn about Cfg issues """ + + def Run(self): + for basename, entry in list(self.core.plugins['Cfg'].entries.items()): + self.check_pubkey(basename, entry) + self.check_missing_files() + + @classmethod + def Errors(cls): + return {"no-pubkey-xml": "warning", + "unknown-cfg-files": "error", + "extra-cfg-files": "error"} + + def check_pubkey(self, basename, entry): + """ check that privkey.xml files have corresponding pubkey.xml + files """ + if "privkey.xml" not in entry.entries: + return + privkey = entry.entries["privkey.xml"] + if not self.HandlesFile(privkey.name): + return + + pubkey = basename + ".pub" + if pubkey not in self.core.plugins['Cfg'].entries: + self.LintError("no-pubkey-xml", + "%s has no corresponding pubkey.xml at %s" % + (basename, pubkey)) + else: + pubset = self.core.plugins['Cfg'].entries[pubkey] + if "pubkey.xml" not in pubset.entries: + self.LintError("no-pubkey-xml", + "%s has no corresponding pubkey.xml at %s" % + (basename, pubkey)) + + def _list_path_components(self, path): + """ Get a list of all components of a path. E.g., + ``self._list_path_components("/foo/bar/foobaz")`` would return + ``["foo", "bar", "foo", "baz"]``. The list is not guaranteed + to be in order.""" + rv = [] + remaining, component = os.path.split(path) + while component != '': + rv.append(component) + remaining, component = os.path.split(remaining) + return rv + + def check_missing_files(self): + """ check that all files on the filesystem are known to Cfg """ + cfg = self.core.plugins['Cfg'] + + # first, collect ignore patterns from handlers + ignore = set() + for hdlr in handlers(): + ignore.update(hdlr.__ignore__) + + # next, get a list of all non-ignored files on the filesystem + all_files = set() + for root, _, files in os.walk(cfg.data): + for fname in files: + fpath = os.path.join(root, fname) + # check against the handler ignore patterns and the + # global FAM ignore list + if (not any(fname.endswith("." + i) for i in ignore) and + not any(fnmatch(fpath, p) + for p in self.config['ignore']) and + not any(fnmatch(c, p) + for p in self.config['ignore'] + for c in self._list_path_components(fpath))): + all_files.add(fpath) + + # next, get a list of all files known to Cfg + cfg_files = set() + for root, eset in cfg.entries.items(): + cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) + for fname in eset.entries.keys()) + + # finally, compare the two + unknown_files = all_files - cfg_files + extra_files = cfg_files - all_files + if unknown_files: + self.LintError( + "unknown-cfg-files", + "Files on the filesystem could not be understood by Cfg: %s" % + "; ".join(unknown_files)) + if extra_files: + self.LintError( + "extra-cfg-files", + "Cfg has entries for files that do not exist on the " + "filesystem: %s\nThis is probably a bug." % + "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index f028e225e..e2d1ec597 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -2,6 +2,7 @@ import os import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Lint from Bcfg2.Server import XI_NAMESPACE from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ @@ -16,6 +17,81 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): give information about the files. For instance, you can require SVN keywords in a comment, or require the name of the maintainer of a Genshi template, and so on. """ + + options = Bcfg2.Server.Lint.ServerPlugin.options + [ + Bcfg2.Options.Option( + cf=("Comments", "global_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for all file types"), + Bcfg2.Options.Option( + cf=("Comments", "global_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for all file types"), + Bcfg2.Options.Option( + cf=("Comments", "bundler_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for non-templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "bundler_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for non-templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "genshibundler_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "genshibundler_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "properties_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Properties files"), + Bcfg2.Options.Option( + cf=("Comments", "properties_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Properties files"), + Bcfg2.Options.Option( + cf=("Comments", "cfg_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for non-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "cfg_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for non-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "genshi_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Genshi-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "genshi_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Genshi-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "cheetah_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Cheetah-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "cheetah_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Cheetah-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "infoxml_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for info.xml files"), + Bcfg2.Options.Option( + cf=("Comments", "infoxml_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for info.xml files"), + Bcfg2.Options.Option( + cf=("Comments", "probe_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for probes"), + Bcfg2.Options.Option( + cf=("Comments", "probe_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for probes")] + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.config_cache = {} @@ -73,17 +149,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): if rtype not in self.config_cache[itype]: rv = [] - global_item = "global_%ss" % itype - if global_item in self.config: - rv.extend(self.config[global_item].split(",")) - - item = "%s_%ss" % (rtype.lower(), itype) - if item in self.config: - if self.config[item]: - rv.extend(self.config[item].split(",")) - else: - # config explicitly specifies nothing - rv = [] + rv.extend(getattr(Bcfg2.Options.setup, "global_%ss" % itype)) + local_reqs = getattr(Bcfg2.Options.setup, + "%s_%ss" % (rtype.lower(), itype)) + if local_reqs == ['']: + # explicitly specified as empty + rv = [] + else: + rv.extend(local_reqs) self.config_cache[itype][rtype] = rv return self.config_cache[itype][rtype] diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index da8da1aa4..76e1986f9 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -18,7 +18,20 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"genshi-syntax-error": "error"} + return {"genshi-syntax-error": "error", + "unknown-genshi-error": "error"} + + def check_template(self, loader, fname, cls=None): + try: + loader.load(fname, cls=cls) + except TemplateSyntaxError: + err = sys.exc_info()[1] + self.LintError("genshi-syntax-error", + "Genshi syntax error in %s: %s" % (fname, err)) + except: + err = sys.exc_info()[1] + self.LintError("unknown-genshi-error", + "Unknown Genshi error in %s: %s" % (fname, err)) def check_cfg(self): """ Check genshi templates in Cfg for syntax errors. """ @@ -27,30 +40,13 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): if (self.HandlesFile(entry.name) and isinstance(entry, CfgGenshiGenerator) and not entry.template): - try: - entry.loader.load(entry.name, - cls=NewTextTemplate) - except TemplateSyntaxError: - err = sys.exc_info()[1] - self.LintError("genshi-syntax-error", - "Genshi syntax error: %s" % err) - except: - etype, err = sys.exc_info()[:2] - self.LintError( - "genshi-syntax-error", - "Unexpected Genshi error on %s: %s: %s" % - (entry.name, etype.__name__, err)) + self.check_template(entry.loader, entry.name, + cls=NewTextTemplate) def check_bundler(self): """ Check templates in Bundler for syntax errors. """ loader = TemplateLoader() - for entry in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(entry.name) and entry.template is not None): - try: - loader.load(entry.name, cls=MarkupTemplate) - except TemplateSyntaxError: - err = sys.exc_info()[1] - self.LintError("genshi-syntax-error", - "Genshi syntax error: %s" % err) + self.check_template(loader, entry.name, cls=MarkupTemplate) diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py index 730f32750..e28080300 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupNames.py +++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py @@ -39,7 +39,8 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): continue xdata = rules.pnode.data self.check_entries(xdata.xpath("//Group"), - os.path.join(self.config['repo'], rules.name)) + os.path.join(Bcfg2.Options.setup.repository, + rules.name)) def check_bundles(self): """ Check groups used in the Bundler plugin for validity. """ @@ -52,7 +53,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): """ Check groups used or declared in the Metadata plugin for validity. """ self.check_entries(self.metadata.groups_xml.xdata.xpath("//Group"), - os.path.join(self.config['repo'], + os.path.join(Bcfg2.Options.setup.repository, self.metadata.groups_xml.name)) def check_grouppatterns(self): diff --git a/src/lib/Bcfg2/Server/Lint/GroupPatterns.py b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py new file mode 100644 index 000000000..8a0ab4f18 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py @@ -0,0 +1,40 @@ +import sys +from Bcfg2.Server.Lint import ServerPlugin +from Bcfg2.Server.Plugins.GroupPatterns import PatternMap + + +class GroupPatterns(ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns + <server-plugins-grouping-grouppatterns>` patterns for validity. + This is simply done by trying to create a + :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for + each pattern, and catching exceptions and presenting them as + ``bcfg2-lint`` errors.""" + + def Run(self): + cfg = self.core.plugins['GroupPatterns'].config + for entry in cfg.xdata.xpath('//GroupPattern'): + groups = [g.text for g in entry.findall('Group')] + self.check(entry, groups, ptype='NamePattern') + self.check(entry, groups, ptype='NameRange') + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, groups, ptype="NamePattern"): + """ Check a single pattern for validity """ + if ptype == "NamePattern": + pmap = lambda p: PatternMap(p, None, groups) + else: + pmap = lambda p: PatternMap(None, p, groups) + + for el in entry.findall(ptype): + pat = el.text + try: + pmap(pat) + except: # pylint: disable=W0702 + err = sys.exc_info()[1] + self.LintError("pattern-fails-to-initialize", + "Failed to initialize %s %s for %s: %s" % + (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index 184f657b7..4b1513a11 100644 --- a/src/lib/Bcfg2/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -15,6 +15,15 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): * Paranoid mode disabled in an ``info.xml`` file; * Required attributes missing from ``info.xml`` """ + + options = Bcfg2.Server.Lint.ServerPlugin.options + [ + Bcfg2.Options.Common.default_paranoid, + Bcfg2.Options.Option( + cf=("InfoXML", "required_attrs"), + type=Bcfg2.Options.Types.comma_list, + default=["owner", "group", "mode"], + help="Attributes to require on <Info> tags")] + def Run(self): if 'Cfg' not in self.core.plugins: return @@ -26,7 +35,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): for entry in entryset.entries.values(): if isinstance(entry, CfgInfoXML): self.check_infoxml(infoxml_fname, - entry.infoxml.pnode.data) + entry.infoxml.xdata) found = True if not found: self.LintError("no-infoxml", @@ -42,8 +51,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): """ Verify that info.xml contains everything it should. """ for info in xdata.getroottree().findall("//Info"): required = [] - if "required_attrs" in self.config: - required = self.config["required_attrs"].split(",") + required = Bcfg2.Options.setup.required_attrs missing = [attr for attr in required if info.get(attr) is None] if missing: @@ -52,10 +60,10 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): (",".join(missing), fname, self.RenderXML(info))) - if ((Bcfg2.Options.MDATA_PARANOID.value and + if ((Bcfg2.Options.setup.default_paranoid == "true" and info.get("paranoid") is not None and info.get("paranoid").lower() == "false") or - (not Bcfg2.Options.MDATA_PARANOID.value and + (Bcfg2.Options.setup.default_paranoid == "false" and (info.get("paranoid") is None or info.get("paranoid").lower() != "true"))): self.LintError("paranoid-false", diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index 2419c3d43..dff95fbf3 100644 --- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -8,9 +8,24 @@ import Bcfg2.Server.Lint from Bcfg2.Server.Plugins.Cfg import CfgGenerator +def threshold(val): + """ Option type processor to accept either a percentage (e.g., + "threshold=75") or a ratio (e.g., "threshold=.75") """ + threshold = float(val) + if threshold > 1: + threshold /= 100 + return threshold + + class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): """ find Probes or Cfg files with multiple similar files that might be merged into one """ + + options = Bcfg2.Server.Lint.ServerPlugin.options + [ + Bcfg2.Options.Option( + cf=("MergeFiles", "threshold"), default="0.75", type=threshold, + help="The threshold at which to suggest merging files and probes")] + def Run(self): if 'Cfg' in self.core.plugins: self.check_cfg() @@ -48,19 +63,10 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): """ Get a list of similar files from the entry dict. Return value is a list of lists, each of which gives the filenames of similar files """ - if "threshold" in self.config: - # accept threshold either as a percent (e.g., "threshold=75") or - # as a ratio (e.g., "threshold=.75") - threshold = float(self.config['threshold']) - if threshold > 1: - threshold /= 100 - else: - threshold = 0.75 rv = [] elist = list(entries.items()) while elist: - result = self._find_similar(elist.pop(0), copy.copy(elist), - threshold) + result = self._find_similar(elist.pop(0), copy.copy(elist)) if len(result) > 1: elist = [(fname, fdata) for fname, fdata in elist @@ -68,7 +74,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): rv.append(result) return rv - def _find_similar(self, ftuple, others, threshold): + def _find_similar(self, ftuple, others): """ Find files similar to the one described by ftupe in the list of other files. ftuple is a tuple of (filename, data); others is a list of such tuples. threshold is a float between @@ -80,9 +86,9 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): cname, cdata = others.pop(0) seqmatch = SequenceMatcher(None, fdata.data, cdata.data) # perform progressively more expensive comparisons - if (seqmatch.real_quick_ratio() > threshold and - seqmatch.quick_ratio() > threshold and - seqmatch.ratio() > threshold): - rv.extend(self._find_similar((cname, cdata), copy.copy(others), - threshold)) + if (seqmatch.real_quick_ratio() > Bcfg2.Options.setup.threshold and + seqmatch.quick_ratio() > Bcfg2.Options.setup.threshold and + seqmatch.ratio() > Bcfg2.Options.setup.threshold): + rv.extend( + self._find_similar((cname, cdata), copy.copy(others))) return rv diff --git a/src/lib/Bcfg2/Server/Lint/Metadata.py b/src/lib/Bcfg2/Server/Lint/Metadata.py new file mode 100644 index 000000000..a349805fd --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Metadata.py @@ -0,0 +1,148 @@ +from Bcfg2.Server.Lint import ServerPlugin + + +class Metadata(ServerPlugin): + """ ``bcfg2-lint`` plugin for :ref:`Metadata + <server-plugins-grouping-metadata>`. This checks for several things: + + * ``<Client>`` tags nested inside other ``<Client>`` tags; + * Deprecated options (like ``location="floating"``); + * Profiles that don't exist, or that aren't profile groups; + * Groups or clients that are defined multiple times; + * Multiple default groups or a default group that isn't a profile + group. + """ + + def Run(self): + self.nested_clients() + self.deprecated_options() + self.bogus_profiles() + self.duplicate_groups() + self.duplicate_default_groups() + self.duplicate_clients() + self.default_is_profile() + + @classmethod + def Errors(cls): + return {"nested-client-tags": "warning", + "deprecated-clients-options": "warning", + "nonexistent-profile-group": "error", + "non-profile-set-as-profile": "error", + "duplicate-group": "error", + "duplicate-client": "error", + "multiple-default-groups": "error", + "default-is-not-profile": "error"} + + def deprecated_options(self): + """ Check for the ``location='floating'`` option, which has + been deprecated in favor of ``floating='true'``. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return + clientdata = self.metadata.clients_xml.xdata + for el in clientdata.xpath("//Client"): + loc = el.get("location") + if loc: + if loc == "floating": + floating = True + else: + floating = False + self.LintError("deprecated-clients-options", + "The location='%s' option is deprecated. " + "Please use floating='%s' instead:\n%s" % + (loc, floating, self.RenderXML(el))) + + def nested_clients(self): + """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, + which is either redundant or will never match. """ + groupdata = self.metadata.groups_xml.xdata + for el in groupdata.xpath("//Client//Client"): + self.LintError("nested-client-tags", + "Client %s nested within Client tag: %s" % + (el.get("name"), self.RenderXML(el))) + + def bogus_profiles(self): + """ Check for clients that have profiles that are either not + flagged as profile groups in ``groups.xml``, or don't exist. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return + for client in self.metadata.clients_xml.xdata.findall('.//Client'): + profile = client.get("profile") + if profile not in self.metadata.groups: + self.LintError("nonexistent-profile-group", + "%s has nonexistent profile group %s:\n%s" % + (client.get("name"), profile, + self.RenderXML(client))) + elif not self.metadata.groups[profile].is_profile: + self.LintError("non-profile-set-as-profile", + "%s is set as profile for %s, but %s is not a " + "profile group:\n%s" % + (profile, client.get("name"), profile, + self.RenderXML(client))) + + def duplicate_default_groups(self): + """ Check for multiple default groups. """ + defaults = [] + for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): + if grp.get("default", "false").lower() == "true": + defaults.append(self.RenderXML(grp)) + if len(defaults) > 1: + self.LintError("multiple-default-groups", + "Multiple default groups defined:\n%s" % + "\n".join(defaults)) + + def duplicate_clients(self): + """ Check for clients that are defined more than once. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return + self.duplicate_entries( + self.metadata.clients_xml.xdata.xpath("//Client"), + "client") + + def duplicate_groups(self): + """ Check for groups that are defined more than once. We + count a group tag as a definition if it a) has profile or + public set; or b) has any children.""" + allgroups = [ + g + for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") + if g.get("profile") or g.get("public") or g.getchildren()] + self.duplicate_entries(allgroups, "group") + + def duplicate_entries(self, allentries, etype): + """ Generic duplicate entry finder. + + :param allentries: A list of all entries to check for + duplicates. + :type allentries: list of lxml.etree._Element + :param etype: The entry type. This will be used to determine + the error name (``duplicate-<etype>``) and for + display to the end user. + :type etype: string + """ + entries = dict() + for el in allentries: + if el.get("name") in entries: + entries[el.get("name")].append(self.RenderXML(el)) + else: + entries[el.get("name")] = [self.RenderXML(el)] + for ename, els in entries.items(): + if len(els) > 1: + self.LintError("duplicate-%s" % etype, + "%s %s is defined multiple times:\n%s" % + (etype.title(), ename, "\n".join(els))) + + def default_is_profile(self): + """ Ensure that the default group is a profile group. """ + if (self.metadata.default and + not self.metadata.groups[self.metadata.default].is_profile): + xdata = \ + self.metadata.groups_xml.xdata.xpath("//Group[@name='%s']" % + self.metadata.default)[0] + self.LintError("default-is-not-profile", + "Default group is not a profile group:\n%s" % + self.RenderXML(xdata)) diff --git a/src/lib/Bcfg2/Server/Lint/Pkgmgr.py b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py new file mode 100644 index 000000000..54f6f07d1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py @@ -0,0 +1,46 @@ +import os +import glob +import lxml.etree +import Bcfg2.Options +from Bcfg2.Server.Lint import ServerlessPlugin + + +class Pkgmgr(ServerlessPlugin): + """ Find duplicate :ref:`Pkgmgr + <server-plugins-generators-pkgmgr>` entries with the same + priority. """ + + def Run(self): + pset = set() + for pfile in glob.glob(os.path.join(Bcfg2.Options.setup.repository, + 'Pkgmgr', '*.xml')): + if self.HandlesFile(pfile): + xdata = lxml.etree.parse(pfile).getroot() + # get priority, type, group + priority = xdata.get('priority') + ptype = xdata.get('type') + for pkg in xdata.xpath("//Package"): + if pkg.getparent().tag == 'Group': + grp = pkg.getparent().get('name') + if (type(grp) is not str and + grp.getparent().tag == 'Group'): + pgrp = grp.getparent().get('name') + else: + pgrp = 'none' + else: + grp = 'none' + pgrp = 'none' + ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) + # check if package is already listed with same + # priority, type, grp + if ptuple in pset: + self.LintError( + "duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) + else: + pset.add(ptuple) + + @classmethod + def Errors(cls): + return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 3bf76765b..cf7b51ecc 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -167,8 +167,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): for rules in self.core.plugins['Rules'].entries.values(): xdata = rules.pnode.data for path in xdata.xpath("//Path"): - self.check_entry(path, os.path.join(self.config['repo'], - rules.name)) + self.check_entry(path, + os.path.join(Bcfg2.Options.setup.repository, + rules.name)) def check_bundles(self): """ Check bundles for BoundPath entries with missing diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py new file mode 100644 index 000000000..a24d70cab --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -0,0 +1,77 @@ +import sys +import imp +from Bcfg2.Server.Lint import ServerPlugin +from Bcfg2.Server.Plugins.TemplateHelper import HelperModule, MODULE_RE, \ + safe_module_name + + +class TemplateHelperLint(ServerPlugin): + """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper + <server-plugins-connectors-templatehelper>` modules are valid. + This can check for: + + * A TemplateHelper module that cannot be imported due to syntax or + other compile-time errors; + * A TemplateHelper module that does not have an ``__export__`` + attribute, or whose ``__export__`` is not a list; + * Bogus symbols listed in ``__export__``, including symbols that + don't exist, that are reserved, or that start with underscores. + """ + + def __init__(self, *args, **kwargs): + ServerPlugin.__init__(self, *args, **kwargs) + self.reserved_keywords = dir(HelperModule("foo.py")) + + def Run(self): + for helper in self.core.plugins['TemplateHelper'].entries.values(): + if self.HandlesFile(helper.name): + self.check_helper(helper.name) + + def check_helper(self, helper): + """ Check a single helper module. + + :param helper: The filename of the helper module + :type helper: string + """ + module_name = MODULE_RE.search(helper).group(1) + + try: + module = imp.load_source(safe_module_name(module_name), helper) + except: # pylint: disable=W0702 + err = sys.exc_info()[1] + self.LintError("templatehelper-import-error", + "Failed to import %s: %s" % + (helper, err)) + return + + if not hasattr(module, "__export__"): + self.LintError("templatehelper-no-export", + "%s has no __export__ list" % helper) + return + elif not isinstance(module.__export__, list): + self.LintError("templatehelper-nonlist-export", + "__export__ is not a list in %s" % helper) + return + + for sym in module.__export__: + if not hasattr(module, sym): + self.LintError("templatehelper-nonexistent-export", + "%s: exported symbol %s does not exist" % + (helper, sym)) + elif sym in self.reserved_keywords: + self.LintError("templatehelper-reserved-export", + "%s: exported symbol %s is reserved" % + (helper, sym)) + elif sym.startswith("_"): + self.LintError("templatehelper-underscore-export", + "%s: exported symbol %s starts with underscore" + % (helper, sym)) + + @classmethod + def Errors(cls): + return {"templatehelper-import-error": "error", + "templatehelper-no-export": "error", + "templatehelper-nonlist-export": "error", + "templatehelper-nonexistent-export": "error", + "templatehelper-reserved-export": "error", + "templatehelper-underscore-export": "warning"} diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index ca9f138ef..2f245561b 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -6,6 +6,7 @@ import sys import glob import fnmatch import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Lint from Bcfg2.Utils import Executor @@ -14,6 +15,12 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): """ Ensure that all XML files in the Bcfg2 repository validate according to their respective schemas. """ + options = Bcfg2.Server.Lint.ServerlessPlugin.options + [ + Bcfg2.Options.PathOption( + "--schema", cf=("Validate", "schema"), + default="/usr/share/bcfg2/schema", + help="The full path to the XML schema files")] + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) @@ -58,7 +65,6 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.cmd = Executor() def Run(self): - schemadir = self.config['schema'] for path, schemaname in self.filesets.items(): try: @@ -68,7 +74,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if filelist: # avoid loading schemas for empty file lists - schemafile = os.path.join(schemadir, schemaname) + schemafile = os.path.join(Bcfg2.Options.setup.schema, + schemaname) schema = self._load_schema(schemafile) if schema: for filename in filelist: @@ -165,8 +172,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): listfiles = lambda p: fnmatch.filter(self.files, os.path.join('*', p)) else: - listfiles = lambda p: glob.glob(os.path.join(self.config['repo'], - p)) + listfiles = lambda p: \ + glob.glob(os.path.join(Bcfg2.Options.setup.repository, p)) for path in self.filesets.keys(): if '/**/' in path: @@ -175,9 +182,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): else: # self.files is None fpath, fname = path.split('/**/') self.filelists[path] = [] - for root, _, files in \ - os.walk(os.path.join(self.config['repo'], - fpath)): + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, fpath)): self.filelists[path].extend([os.path.join(root, f) for f in files if f == fname]) diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 28644263f..4f64fd006 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -2,16 +2,17 @@ import os import sys +import time +import copy +import fcntl +import struct +import termios import logging -from copy import copy import textwrap import lxml.etree -import fcntl -import termios -import struct -from Bcfg2.Compat import walk_packages - -plugins = [m[1] for m in walk_packages(path=__path__)] # pylint: disable=C0103 +import Bcfg2.Options +import Bcfg2.Server.Core +import Bcfg2.Server.Plugins def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -46,10 +47,10 @@ def get_termsize(): class Plugin(object): """ Base class for all bcfg2-lint plugins """ - def __init__(self, config, errorhandler=None, files=None): + options = [Bcfg2.Options.Common.repository] + + def __init__(self, errorhandler=None, files=None): """ - :param config: A :mod:`Bcfg2.Options` setup dict - :type config: dict :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` that will be used to handle lint errors. If one is not provided, a new one will be @@ -63,9 +64,6 @@ class Plugin(object): #: The list of files that bcfg2-lint should be run against self.files = files - #: The Bcfg2.Options setup dict - self.config = config - self.logger = logging.getLogger('bcfg2-lint') if errorhandler is None: #: The error handler @@ -96,9 +94,10 @@ class Plugin(object): False otherwise. """ return (self.files is None or fname in self.files or - os.path.join(self.config['repo'], fname) in self.files or + os.path.join(Bcfg2.Options.setup.repository, + fname) in self.files or os.path.abspath(fname) in self.files or - os.path.abspath(os.path.join(self.config['repo'], + os.path.abspath(os.path.join(Bcfg2.Options.setup.repository, fname)) in self.files) def LintError(self, err, msg): @@ -125,7 +124,7 @@ class Plugin(object): """ xml = None if len(element) or element.text: - el = copy(element) + el = copy.copy(element) if el.text and not keep_text: el.text = '...' for child in el.iterchildren(): @@ -145,8 +144,8 @@ class ErrorHandler(object): def __init__(self, errors=None): """ - :param config: An initial dict of errors to register - :type config: dict + :param errors: An initial dict of errors to register + :type errors: dict """ #: The number of errors passed to this error handler self.errors = 0 @@ -267,12 +266,10 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 """ Base class for bcfg2-lint plugins that check things that require the running Bcfg2 server. """ - def __init__(self, core, config, errorhandler=None, files=None): + def __init__(self, core, errorhandler=None, files=None): """ :param core: The Bcfg2 server core :type core: Bcfg2.Server.Core.BaseCore - :param config: A :mod:`Bcfg2.Options` setup dict - :type config: dict :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` that will be used to handle lint errors. If one is not provided, a new one will be @@ -282,7 +279,7 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 the bcfg2-lint ``--stdin`` option.) :type files: list of strings """ - Plugin.__init__(self, config, errorhandler=errorhandler, files=files) + Plugin.__init__(self, errorhandler=errorhandler, files=files) #: The server core self.core = core @@ -290,3 +287,166 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 #: The metadata plugin self.metadata = self.core.metadata + + +class LintPluginAction(Bcfg2.Options.ComponentAction): + """ We want to load all lint plugins that pertain to server + plugins. In order to do this, we hijack the __call__() method of + this action and add all of the server plugins on the fly """ + + bases = ['Bcfg2.Server.Lint'] + + def __call__(self, parser, namespace, values, option_string=None): + for plugin in getattr(Bcfg2.Options.setup, "plugins", []): + module = sys.modules[plugin.__module__] + if hasattr(module, "%sLint" % plugin.name): + print("Adding lint plugin %s" % plugin) + values.append(plugin) + Bcfg2.Options.ComponentAction.__call__(self, parser, namespace, values, + option_string) + + +class CLI(object): + """ The bcfg2-lint CLI """ + options = Bcfg2.Server.Core.Core.options + [ + Bcfg2.Options.PathOption( + '--lint-config', default='/etc/bcfg2-lint.conf', + action=Bcfg2.Options.ConfigFileAction, + help='Specify bcfg2-lint configuration file'), + Bcfg2.Options.Option( + "--lint-plugins", cf=('lint', 'plugins'), + type=Bcfg2.Options.Types.comma_list, action=LintPluginAction, + help='bcfg2-lint plugin list'), + Bcfg2.Options.BooleanOption( + '--list-errors', help='Show error handling'), + Bcfg2.Options.BooleanOption( + '--stdin', help='Operate on a list of files supplied on stdin'), + Bcfg2.Options.Option( + cf=("errors", '*'), dest="lint_errors", + help="How to handle bcfg2-lint errors")] + + def __init__(self): + parser = Bcfg2.Options.get_parser( + description="Manage a running Bcfg2 server", + components=[self]) + parser.parse() + + self.logger = logging.getLogger(parser.prog) + + # automatically add Lint plugins for loaded server plugins + for plugin in Bcfg2.Options.setup.plugins: + try: + Bcfg2.Options.setup.lint_plugins.append( + getattr( + __import__("Bcfg2.Server.Lint.%s" % plugin.__name__, + fromlist=[plugin.__name__]), + plugin.__name__)) + self.logger.debug("Automatically adding lint plugin %s" % + plugin.__name__) + except ImportError: + # no lint plugin for this server plugin + self.logger.debug("No lint plugin for %s" % plugin.__name__) + pass + except AttributeError: + self.logger.error("Failed to load plugin %s: %s" % + (plugin.__name__, sys.exc_info()[1])) + + self.logger.debug("Running lint with plugins: %s" % + [p.__name__ + for p in Bcfg2.Options.setup.lint_plugins]) + + if Bcfg2.Options.setup.stdin: + self.files = [s.strip() for s in sys.stdin.readlines()] + else: + self.files = None + self.errorhandler = self.get_errorhandler() + self.serverlessplugins = [] + self.serverplugins = [] + for plugin in Bcfg2.Options.setup.lint_plugins: + if issubclass(plugin, ServerPlugin): + self.serverplugins.append(plugin) + else: + self.serverlessplugins.append(plugin) + + def run(self): + if Bcfg2.Options.setup.list_errors: + for plugin in self.serverplugins + self.serverlessplugins: + self.errorhandler.RegisterErrors(getattr(plugin, 'Errors')()) + + print("%-35s %-35s" % ("Error name", "Handler")) + for err, handler in self.errorhandler.errortypes.items(): + print("%-35s %-35s" % (err, handler.__name__)) + return 0 + + if not self.serverplugins and not self.serverlessplugins: + self.logger.error("No lint plugins loaded!") + return 1 + + self.run_serverless_plugins() + + if self.serverplugins: + if self.errorhandler.errors: + # it would be swell if we could try to start the server + # even if there were errors with the serverless plugins, + # but since XML parsing errors occur in the FAM thread + # (not in the core server thread), there's no way we can + # start the server and try to catch exceptions -- + # bcfg2-lint isn't in the same stack as the exceptions. + # so we're forced to assume that a serverless plugin error + # will prevent the server from starting + print("Serverless plugins encountered errors, skipping server " + "plugins") + else: + self.run_server_plugins() + + if (self.errorhandler.errors or + self.errorhandler.warnings or + Bcfg2.Options.setup.verbose): + print("%d errors" % self.errorhandler.errors) + print("%d warnings" % self.errorhandler.warnings) + + if self.errorhandler.errors: + return 2 + elif self.errorhandler.warnings: + return 3 + else: + return 0 + + def get_errorhandler(self): + """ get a Bcfg2.Server.Lint.ErrorHandler object """ + return Bcfg2.Server.Lint.ErrorHandler( + errors=Bcfg2.Options.setup.lint_errors) + + def run_serverless_plugins(self): + """ Run serverless plugins """ + self.logger.debug("Running serverless plugins: %s" % + [p.__name__ for p in self.serverlessplugins]) + for plugin in self.serverlessplugins: + self.logger.debug(" Running %s" % plugin.__name__) + plugin(files=self.files, errorhandler=self.errorhandler).Run() + + def run_server_plugins(self): + """ run plugins that require a running server to run """ + core = Bcfg2.Server.Core.Core() + core.load_plugins() + core.block_for_fam_events(handle_events=True) + try: + self.logger.debug("Running server plugins: %s" % + [p.__name__ for p in self.serverplugins]) + for plugin in self.serverplugins: + self.logger.debug(" Running %s" % plugin.__name__) + plugin(core, + files=self.files, errorhandler=self.errorhandler).Run() + finally: + core.shutdown() + + def _run_plugin(self, plugin, args=None): + if args is None: + args = [] + start = time.time() + # python 2.5 doesn't support mixing *magic and keyword arguments + kwargs = dict(files=self.files, errorhandler=self.errorhandler) + rv = plugin(*args, **kwargs).Run() + self.logger.debug(" Ran %s in %0.2f seconds" % (plugin.__name__, + time.time() - start)) + return rv diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index e79207291..678a1c95d 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -15,12 +15,13 @@ import time import threading import lxml.etree import multiprocessing +import Bcfg2.Options import Bcfg2.Server.Plugin from itertools import cycle from Bcfg2.Cache import Cache from Bcfg2.Compat import Queue, Empty, wraps -from Bcfg2.Server.Core import BaseCore, exposed -from Bcfg2.Server.BuiltinCore import Core as BuiltinCore +from Bcfg2.Server.Core import Core, exposed +from Bcfg2.Server.BuiltinCore import BuiltinCore from multiprocessing.connection import Listener, Client @@ -167,7 +168,7 @@ class DualEvent(object): return self._threading_event.wait(timeout=timeout) -class ChildCore(BaseCore): +class ChildCore(Core): """ A child process for :class:`Bcfg2.MultiprocessingCore.Core`. This core builds configurations from a given :class:`multiprocessing.Pipe`. Note that this is a full-fledged @@ -186,12 +187,10 @@ class ChildCore(BaseCore): #: every ``poll_wait`` seconds. poll_wait = 3.0 - def __init__(self, name, setup, rpc_q, terminate): + def __init__(self, name, rpc_q, terminate): """ :param name: The name of this child :type name: string - :param setup: A Bcfg2 options dict - :type setup: Bcfg2.Options.OptionParser :param read_q: The queue the child will read from for RPC communications from the parent process. :type read_q: multiprocessing.Queue @@ -202,7 +201,7 @@ class ChildCore(BaseCore): themselves down. :type terminate: multiprocessing.Event """ - BaseCore.__init__(self, setup) + Core.__init__(self) #: The name of this child self.name = name @@ -216,7 +215,7 @@ class ChildCore(BaseCore): # override this setting so that the child doesn't try to write # the pidfile - self.setup['daemon'] = False + Bcfg2.Options.setup.daemon = False # ensure that the child doesn't start a perflog thread self.perflog_thread = None @@ -283,7 +282,7 @@ class ChildCore(BaseCore): self.shutdown() def shutdown(self): - BaseCore.shutdown(self) + Core.shutdown(self) self.logger.info("%s: Closing RPC command queue" % self.name) self.rpc_q.close() @@ -328,7 +327,7 @@ class ChildCore(BaseCore): return lxml.etree.tostring(self.BuildConfiguration(client)) -class Core(BuiltinCore): +class MultiprocessingCore(BuiltinCore): """ A multiprocessing core that delegates building the actual client configurations to :class:`Bcfg2.Server.MultiprocessingCore.ChildCore` objects. The @@ -336,14 +335,34 @@ class Core(BuiltinCore): :func:`GetConfig` are delegated to children. All other calls are handled by the parent process. """ + options = BuiltinCore.options + [ + Bcfg2.Options.Option( + '--children', dest="core_children", + cf=('server', 'children'), type=int, + default=multiprocessing.cpu_count(), + help='Spawn this number of children for the multiprocessing core')] + #: How long to wait for a child process to shut down cleanly #: before it is terminated. shutdown_timeout = 10.0 - def __init__(self, setup): - BuiltinCore.__init__(self, setup) - if setup['children'] is None: - setup['children'] = multiprocessing.cpu_count() + def __init__(self): + BuiltinCore.__init__(self) + + #: A dict of child name -> one end of the + #: :class:`multiprocessing.Pipe` object used to communicate + #: with that child. (The child is given the other end of the + #: Pipe.) + self.pipes = dict() + + #: A queue that keeps track of which children are available to + #: render a configuration. A child is popped from the queue + #: when it starts to render a config, then it's pushed back on + #: when it's done. This lets us use a blocking call to + #: :func:`Queue.Queue.get` when waiting for an available + #: child. + self.available_children = \ + Queue(maxsize=Bcfg2.Options.setup.core_children) #: The flag that indicates when to stop child threads and #: processes @@ -363,12 +382,12 @@ class Core(BuiltinCore): self.children = None def _run(self): - for cnum in range(self.setup['children']): + for cnum in range(Bcfg2.Options.setup.core_children): name = "Child-%s" % cnum self.logger.debug("Starting child %s" % name) child_q = self.rpc_q.add_subscriber(name) - childcore = ChildCore(name, self.setup, child_q, self.terminate) + childcore = ChildCore(name, child_q, self.terminate) child = multiprocessing.Process(target=childcore.run, name=name) child.start() self.logger.debug("Child %s started with PID %s" % (name, diff --git a/src/lib/Bcfg2/Server/Plugin/__init__.py b/src/lib/Bcfg2/Server/Plugin/__init__.py index ed1282ba0..a85867134 100644 --- a/src/lib/Bcfg2/Server/Plugin/__init__.py +++ b/src/lib/Bcfg2/Server/Plugin/__init__.py @@ -14,6 +14,7 @@ documentation it's not necessary to use the submodules. E.g., you can import os import sys +import Bcfg2.Options sys.path.append(os.path.dirname(__file__)) # pylint: disable=W0401 @@ -21,3 +22,31 @@ from Bcfg2.Server.Plugin.base import * from Bcfg2.Server.Plugin.interfaces import * from Bcfg2.Server.Plugin.helpers import * from Bcfg2.Server.Plugin.exceptions import * + + +class _OptionContainer(object): + options = [ + Bcfg2.Options.Common.default_paranoid, + Bcfg2.Options.Option( + cf=('mdata', 'owner'), dest="default_owner", default='root', + help='Default Path owner'), + Bcfg2.Options.Option( + cf=('mdata', 'group'), dest="default_group", default='root', + help='Default Path group'), + Bcfg2.Options.Option( + cf=('mdata', 'important'), dest="default_important", + default='false', choices=['true', 'false'], + help='Default Path priority (importance)'), + Bcfg2.Options.Option( + cf=('mdata', 'mode'), dest="default_mode", default='644', + help='Default mode for Path'), + Bcfg2.Options.Option( + cf=('mdata', 'secontext'), dest="default_secontext", + default='__default__', help='Default SELinux context'), + Bcfg2.Options.Option( + cf=('mdata', 'sensitive'), dest="default_sensitive", + default='false', + help='Default Path sensitivity setting')] + + +Bcfg2.Options.get_parser().add_component(_OptionContainer) diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index 03feceb6f..b2d9fa7c8 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -1,66 +1,10 @@ """This module provides the base class for Bcfg2 server plugins.""" import os -import logging +from Bcfg2.Logger import Debuggable from Bcfg2.Utils import ClassName -class Debuggable(object): - """ Mixin to add a debugging interface to an object and expose it - via XML-RPC on :class:`Bcfg2.Server.Plugin.base.Plugin` objects """ - - #: List of names of methods to be exposed as XML-RPC functions - __rmi__ = ['toggle_debug', 'set_debug'] - - #: How exposed XML-RPC functions should be dispatched to child - #: processes. - __child_rmi__ = __rmi__[:] - - def __init__(self, name=None): - """ - :param name: The name of the logger object to get. If none is - supplied, the full name of the class (including - module) will be used. - :type name: string - - .. autoattribute:: __rmi__ - """ - if name is None: - name = "%s.%s" % (self.__class__.__module__, - self.__class__.__name__) - self.debug_flag = False - self.logger = logging.getLogger(name) - - def set_debug(self, debug): - """ Explicitly enable or disable debugging. This method is exposed - via XML-RPC. - - :returns: bool - The new value of the debug flag - """ - self.debug_flag = debug - return debug - - def toggle_debug(self): - """ Turn debugging output on or off. This method is exposed - via XML-RPC. - - :returns: bool - The new value of the debug flag - """ - return self.set_debug(not self.debug_flag) - - def debug_log(self, message, flag=None): - """ Log a message at the debug level. - - :param message: The message to log - :type message: string - :param flag: Override the current debug flag with this value - :type flag: bool - :returns: None - """ - if (flag is None and self.debug_flag) or flag: - self.logger.error(message) - - class Plugin(Debuggable): """ The base class for all Bcfg2 Server plugins. """ diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index a63e9c5f7..0266af909 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -13,8 +13,10 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Options import Bcfg2.Server.FileMonitor +from Bcfg2.Utils import ClassName +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import CmpMixin, wraps -from Bcfg2.Server.Plugin.base import Debuggable, Plugin +from Bcfg2.Server.Plugin.base import Plugin from Bcfg2.Server.Plugin.interfaces import Generator from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ PluginExecutionError @@ -31,63 +33,8 @@ try: except ImportError: HAS_DJANGO = False -#: A dict containing default metadata for Path entries from bcfg2.conf -DEFAULT_FILE_METADATA = Bcfg2.Options.OptionParser( - dict(configfile=Bcfg2.Options.CFILE, - owner=Bcfg2.Options.MDATA_OWNER, - group=Bcfg2.Options.MDATA_GROUP, - mode=Bcfg2.Options.MDATA_MODE, - secontext=Bcfg2.Options.MDATA_SECONTEXT, - important=Bcfg2.Options.MDATA_IMPORTANT, - paranoid=Bcfg2.Options.MDATA_PARANOID, - sensitive=Bcfg2.Options.MDATA_SENSITIVE)) -DEFAULT_FILE_METADATA.parse([Bcfg2.Options.CFILE.cmd, Bcfg2.Options.CFILE]) -del DEFAULT_FILE_METADATA['args'] -del DEFAULT_FILE_METADATA['configfile'] - LOGGER = logging.getLogger(__name__) -#: a compiled regular expression for parsing info and :info files -INFO_REGEX = re.compile(r'owner:\s*(?P<owner>\S+)|' + - r'group:\s*(?P<group>\S+)|' + - r'mode:\s*(?P<mode>\w+)|' + - r'secontext:\s*(?P<secontext>\S+)|' + - r'paranoid:\s*(?P<paranoid>\S+)|' + - r'sensitive:\s*(?P<sensitive>\S+)|' + - r'encoding:\s*(?P<encoding>\S+)|' + - r'important:\s*(?P<important>\S+)|' + - r'mtime:\s*(?P<mtime>\w+)') - - -def bind_info(entry, metadata, infoxml=None, default=DEFAULT_FILE_METADATA): - """ Bind the file metadata in the given - :class:`Bcfg2.Server.Plugin.helpers.InfoXML` object to the given - entry. - - :param entry: The abstract entry to bind the info to - :type entry: lxml.etree._Element - :param metadata: The client metadata to get info for - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param infoxml: The info.xml file to pull file metadata from - :type infoxml: Bcfg2.Server.Plugin.helpers.InfoXML - :param default: Default metadata to supply when the info.xml file - does not include a particular attribute - :type default: dict - :returns: None - :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` - """ - for attr, val in list(default.items()): - entry.set(attr, val) - if infoxml: - mdata = dict() - infoxml.pnode.Match(metadata, mdata, entry=entry) - if 'Info' not in mdata: - msg = "Failed to set metadata for file %s" % entry.get('name') - LOGGER.error(msg) - raise PluginExecutionError(msg) - for attr, val in list(mdata['Info'][None].items()): - entry.set(attr, val) - class track_statistics(object): # pylint: disable=C0103 """ Decorator that tracks execution time for the given @@ -139,53 +86,76 @@ def removecomment(stream): yield kind, data, pos +def bind_info(entry, metadata, infoxml=None, default=None): + """ Bind the file metadata in the given + :class:`Bcfg2.Server.Plugin.helpers.InfoXML` object to the given + entry. + + :param entry: The abstract entry to bind the info to + :type entry: lxml.etree._Element + :param metadata: The client metadata to get info for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param infoxml: The info.xml file to pull file metadata from + :type infoxml: Bcfg2.Server.Plugin.helpers.InfoXML + :param default: Default metadata to supply when the info.xml file + does not include a particular attribute + :type default: dict + :returns: None + :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` + """ + if default is None: + default = default_path_metadata() + for attr, val in list(default.items()): + entry.set(attr, val) + if infoxml: + mdata = dict() + infoxml.pnode.Match(metadata, mdata, entry=entry) + if 'Info' not in mdata: + msg = "Failed to set metadata for file %s" % entry.get('name') + LOGGER.error(msg) + raise PluginExecutionError(msg) + for attr, val in list(mdata['Info'][None].items()): + entry.set(attr, val) + + def default_path_metadata(): """ Get the default Path entry metadata from the config. :returns: dict of metadata attributes and their default values """ - attrs = Bcfg2.Options.PATH_METADATA_OPTIONS.keys() - setup = Bcfg2.Options.get_option_parser() - if not set(attrs).issubset(setup.keys()): - setup.add_options(Bcfg2.Options.PATH_METADATA_OPTIONS) - setup.reparse(argv=[Bcfg2.Options.CFILE.cmd, Bcfg2.Options.CFILE]) - return dict([(k, setup[k]) for k in attrs]) + return dict([(k, getattr(Bcfg2.Options.setup, "default_%s" % k)) + for k in ['owner', 'group', 'mode', 'secontext', 'important', + 'paranoid', 'sensitive']]) class DatabaseBacked(Plugin): """ Provides capabilities for a plugin to read and write to a - database. + database. The plugin must add an option to flag database use with + something like: + + options = Bcfg2.Server.Plugin.Plugins.options + [ + Bcfg2.Options.BooleanOption( + cf=('metadata', 'use_database'), dest="metadata_db", + help="Use database capabilities of the Metadata plugin") + + This must be done manually due to various limitations in Python. .. private-include: _use_db .. private-include: _must_lock """ - #: The option to look up in :attr:`section` to determine whether or - #: not to use the database capabilities of this plugin. The option - #: is retrieved with - #: :py:func:`ConfigParser.SafeConfigParser.getboolean`, and so must - #: conform to the possible values that function can handle. - option = "use_database" - - def _section(self): - """ The section to look in for :attr:`DatabaseBacked.option` - """ - return self.name.lower() - section = property(_section) - @property def _use_db(self): """ Whether or not this plugin is configured to use the database. """ - use_db = self.core.setup.cfp.getboolean(self.section, - self.option, - default=False) + use_db = getattr(Bcfg2.Options.setup, "%s_db" % self.name.lower(), + False) if use_db and HAS_DJANGO and self.core.database_available: return True elif not use_db: return False else: - self.logger.error("%s is true but django not found" % self.option) + self.logger.error("use_database is true but django not found") return False @property @@ -193,11 +163,7 @@ class DatabaseBacked(Plugin): """ Whether or not the backend database must acquire a thread lock before writing, because it does not allow multiple threads to write.""" - engine = \ - self.core.setup.cfp.get(Bcfg2.Options.DB_ENGINE.cf[0], - Bcfg2.Options.DB_ENGINE.cf[1], - default=Bcfg2.Options.DB_ENGINE.default) - return engine == 'sqlite3' + return Bcfg2.Options.setup.db_engine == 'sqlite3' @staticmethod def get_db_lock(func): @@ -679,10 +645,9 @@ class StructFile(XMLFileBacked): dict(Group=lambda el, md, *args: el.get('name') in md.groups, Client=lambda el, md, *args: el.get('name') == md.hostname) - def __init__(self, filename, should_monitor=False): - XMLFileBacked.__init__(self, filename, should_monitor=should_monitor) - self.setup = Bcfg2.Options.get_option_parser() - self.encoding = self.setup['encoding'] + def __init__(self, filename, should_monitor=False, create=None): + XMLFileBacked.__init__(self, filename, should_monitor=should_monitor, + create=create) self.template = None def Index(self): @@ -692,9 +657,10 @@ class StructFile(XMLFileBacked): self.xdata.nsmap['py'] == 'http://genshi.edgewall.org/')): try: loader = genshi.template.TemplateLoader() - self.template = loader.load(self.name, - cls=genshi.template.MarkupTemplate, - encoding=self.encoding) + self.template = \ + loader.load(self.name, + cls=genshi.template.MarkupTemplate, + encoding=Bcfg2.Options.setup.encoding) except LookupError: err = sys.exc_info()[1] self.logger.error('Genshi lookup error in %s: %s' % (self.name, @@ -709,10 +675,9 @@ class StructFile(XMLFileBacked): err)) if HAS_CRYPTO: - strict = self.xdata.get( - "decrypt", - self.setup.cfp.get(Bcfg2.Server.Encryption.CFG_SECTION, - "decrypt", default="strict")) == "strict" + lax_decrypt = self.xdata.get( + "lax_decryption", + str(Bcfg2.Options.setup.lax_decryption)).lower() == "true" for el in self.xdata.xpath("//*[@encrypted]"): try: el.text = self._decrypt(el).encode('ascii', @@ -723,17 +688,17 @@ class StructFile(XMLFileBacked): except Bcfg2.Server.Encryption.EVPError: msg = "Failed to decrypt %s element in %s" % (el.tag, self.name) - if strict: - raise PluginExecutionError(msg) - else: + if lax_decrypt: self.logger.warning(msg) + else: + raise PluginExecutionError(msg) Index.__doc__ = XMLFileBacked.Index.__doc__ def _decrypt(self, element): """ Decrypt a single encrypted properties file element """ if not element.text or not element.text.strip(): return - passes = Bcfg2.Server.Encryption.get_passphrases() + passes = Bcfg2.Options.setup.passphrases try: passphrase = passes[element.get("encrypted")] try: @@ -780,7 +745,7 @@ class StructFile(XMLFileBacked): """ stream = self.template.generate( metadata=metadata, - repo=self.setup['repo']).filter(removecomment) + repo=Bcfg2.Options.setup.repository).filter(removecomment) return lxml.etree.XML(stream.render('xml', strip_whitespace=False), parser=Bcfg2.Server.XMLParser) @@ -897,7 +862,7 @@ class InfoXML(StructFile): metadata (permissions, owner, etc.) of files. """ encryption = False - _include_tests = StructFile._include_tests + _include_tests = copy.copy(StructFile._include_tests) _include_tests['Path'] = lambda el, md, entry, *args: \ entry.get("name") == el.get("name") @@ -1152,7 +1117,7 @@ class SpecificData(Debuggable): """ A file that is specific to certain clients, groups, or all clients. """ - def __init__(self, name, specific, encoding): # pylint: disable=W0613 + def __init__(self, name, specific): # pylint: disable=W0613 """ :param name: The full path to the file :type name: string @@ -1161,8 +1126,6 @@ class SpecificData(Debuggable): object describing what clients this file applies to. :type specific: Bcfg2.Server.Plugin.helpers.Specificity - :param encoding: The encoding to use for data in this file - :type encoding: string """ Debuggable.__init__(self) self.name = name @@ -1210,7 +1173,7 @@ class EntrySet(Debuggable): #: considered a plain string and filenames must match exactly. basename_is_regex = False - def __init__(self, basename, path, entry_type, encoding): + def __init__(self, basename, path, entry_type): """ :param basename: The filename or regular expression that files in this EntrySet must match. See @@ -1225,12 +1188,10 @@ class EntrySet(Debuggable): be an object factory or similar callable. See below for the expected signature. :type entry_type: callable - :param encoding: The encoding of all files in this entry set. - :type encoding: string The ``entry_type`` callable must have the following signature:: - entry_type(filepath, specificity, encoding) + entry_type(filepath, specificity) Where the parameters are: @@ -1241,8 +1202,6 @@ class EntrySet(Debuggable): object describing what clients this file applies to. :type specific: Bcfg2.Server.Plugin.helpers.Specificity - :param encoding: The encoding to use for data in this file - :type encoding: string Additionally, the object returned by ``entry_type`` must have a ``specific`` attribute that is sortable (e.g., a @@ -1257,7 +1216,6 @@ class EntrySet(Debuggable): self.entries = {} self.metadata = default_path_metadata() self.infoxml = None - self.encoding = encoding if self.basename_is_regex: base_pat = basename @@ -1274,6 +1232,12 @@ class EntrySet(Debuggable): #: be overridden on a per-entry basis in :func:`entry_init`. self.specific = re.compile(pattern) + def set_debug(self, debug): + rv = Debuggable.set_debug(self, debug) + for entry in self.entries.values(): + entry.set_debug(debug) + return rv + def get_matching(self, metadata): """ Get a list of all entries that apply to the given client. This gets all matching entries; for example, there could be an @@ -1391,8 +1355,7 @@ class EntrySet(Debuggable): self.logger.error("Could not process filename %s; ignoring" % fpath) return - self.entries[event.filename] = entry_type(fpath, spec, - self.encoding) + self.entries[event.filename] = entry_type(fpath, spec) self.entries[event.filename].handle_event(event) def specificity_from_filename(self, fname, specific=None): @@ -1539,7 +1502,6 @@ class GroupSpool(Plugin, Generator): self.entries = {} self.handles = {} self.AddDirectoryMonitor('') - self.encoding = core.setup['encoding'] __init__.__doc__ = Plugin.__init__.__doc__ def add_entry(self, event): @@ -1563,8 +1525,7 @@ class GroupSpool(Plugin, Generator): dirpath = self.data + ident self.entries[ident] = self.es_cls(self.filename_pattern, dirpath, - self.es_child_cls, - self.encoding) + self.es_child_cls) self.Entries[self.entry_type][ident] = \ self.entries[ident].bind_entry if not os.path.isdir(epath): diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index ed4afb9b2..30275f6ad 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -6,6 +6,7 @@ import copy import threading import lxml.etree import Bcfg2.Server +import Bcfg2.Options from Bcfg2.Compat import Queue, Empty, Full, cPickle from Bcfg2.Server.Plugin.base import Plugin from Bcfg2.Server.Plugin.exceptions import PluginInitError, \ @@ -530,6 +531,11 @@ class Version(Plugin): create = False + options = Plugin.options + [ + Bcfg2.Options.PathOption(cf=('server', 'vcs_root'), + default='<repository>', + help='Server VCS repository root')] + #: The path to the VCS metadata file or directory, relative to the #: base of the Bcfg2 repository. E.g., for Subversion this would #: be ".svn" @@ -540,12 +546,8 @@ class Version(Plugin): def __init__(self, core, datastore): Plugin.__init__(self, core, datastore) - if core.setup['vcs_root']: - self.vcs_root = core.setup['vcs_root'] - else: - self.vcs_root = datastore if self.__vcs_metadata_path__: - self.vcs_path = os.path.join(self.vcs_root, + self.vcs_path = os.path.join(Bcfg2.Options.setup.vcs_root, self.__vcs_metadata_path__) if not os.path.exists(self.vcs_path): diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 2473a3ed2..f91bac634 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -6,7 +6,6 @@ import sys import copy import Bcfg2.Server import Bcfg2.Server.Plugin -import Bcfg2.Server.Lint from genshi.template import TemplateError @@ -45,6 +44,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = BundleFile + patterns = re.compile(r'^.*\.(?:xml|genshi)$') def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -123,57 +123,3 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, return bundleset BuildStructures.__doc__ = \ Bcfg2.Server.Plugin.Structure.BuildStructures.__doc__ - - -class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various :ref:`Bundler - <server-plugins-structures-bundler-index>` checks. """ - - def Run(self): - self.missing_bundles() - for bundle in self.core.plugins['Bundler'].entries.values(): - if self.HandlesFile(bundle.name): - self.bundle_names(bundle) - - @classmethod - def Errors(cls): - return {"bundle-not-found": "error", - "unused-bundle": "warning", - "explicit-bundle-name": "error", - "genshi-extension-bundle": "error"} - - def missing_bundles(self): - """ Find bundles listed in Metadata but not implemented in - Bundler. """ - if self.files is None: - # when given a list of files on stdin, this check is - # useless, so skip it - groupdata = self.metadata.groups_xml.xdata - ref_bundles = set([b.get("name") - for b in groupdata.findall("//Bundle")]) - - allbundles = self.core.plugins['Bundler'].bundles.keys() - for bundle in ref_bundles: - if bundle not in allbundles: - self.LintError("bundle-not-found", - "Bundle %s referenced, but does not exist" % - bundle) - - for bundle in allbundles: - if bundle not in ref_bundles: - self.LintError("unused-bundle", - "Bundle %s defined, but is not referenced " - "in Metadata" % bundle) - - def bundle_names(self, bundle): - """ Verify that deprecated bundle .genshi bundles and explicit - bundle names aren't used """ - if bundle.xdata.get('name'): - self.LintError("explicit-bundle-name", - "Deprecated explicit bundle name in %s" % - bundle.name) - - if bundle.name.endswith(".genshi"): - self.LintError("genshi-extension-bundle", - "Bundle %s uses deprecated .genshi extension" % - bundle.name) diff --git a/src/lib/Bcfg2/Server/Plugins/Bzr.py b/src/lib/Bcfg2/Server/Plugins/Bzr.py index e0cbdf72a..f91cc1943 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bzr.py +++ b/src/lib/Bcfg2/Server/Plugins/Bzr.py @@ -14,13 +14,13 @@ class Bzr(Bcfg2.Server.Plugin.Version): def __init__(self, core, datastore): Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) self.logger.debug("Initialized Bazaar plugin with directory %s at " - "revision = %s" % (self.vcs_root, + "revision = %s" % (Bcfg2.Options.setup.vcs_root, self.get_revision())) def get_revision(self): """Read Bazaar revision information for the Bcfg2 repository.""" try: - working_tree = WorkingTree.open(self.vcs_root) + working_tree = WorkingTree.open(Bcfg2.Options.setup.vcs_root) revision = str(working_tree.branch.revno()) if (working_tree.has_changes(working_tree.basis_tree()) or working_tree.unknowns()): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py index a859da0ba..50de498e6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py @@ -3,6 +3,7 @@ based on an XML specification of which SSH keypairs should granted access. """ import lxml.etree +import Bcfg2.Options from Bcfg2.Server.Plugin import StructFile, PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator, CFG from Bcfg2.Server.Plugins.Metadata import ClientMetadata @@ -27,15 +28,6 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): self.core = CFG.core __init__.__doc__ = CfgGenerator.__init__.__doc__ - @property - def category(self): - """ The name of the metadata category that generated keys are - specific to """ - if (self.setup.cfp.has_section("sshkeys") and - self.setup.cfp.has_option("sshkeys", "category")): - return self.setup.cfp.get("sshkeys", "category") - return None - def handle_event(self, event): CfgGenerator.handle_event(self, event) StructFile.HandleEvent(self, event) @@ -61,12 +53,13 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): key_md = ClientMetadata("dummy", group, [group], [], set(), set(), dict(), None, None, None, None) - elif (self.category and - not metadata.group_in_category(self.category)): + elif (Bcfg2.Options.setup.sshkeys_category and + not metadata.group_in_category( + Bcfg2.Options.setup.sshkeys_category)): self.logger.warning("Cfg: %s ignoring Allow from %s: " "No group in category %s" % (metadata.hostname, pubkey_name, - self.category)) + Bcfg2.Options.setup.sshkeys_category)) continue else: key_md = metadata diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index 4c8adceec..476dc1fc6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -2,6 +2,7 @@ <http://www.cheetahtemplate.org/>`_ templating system to generate :ref:`server-plugins-generators-cfg` files. """ +import Bcfg2.Options from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator @@ -27,19 +28,19 @@ class CfgCheetahGenerator(CfgGenerator): #: :class:`Cheetah.Template.Template` compiler settings settings = dict(useStackFrames=False) - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) if not HAS_CHEETAH: raise PluginExecutionError("Cheetah is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): - template = Template(self.data.decode(self.encoding), + template = Template(self.data.decode(Bcfg2.Options.setup.encoding), compilerSettings=self.settings) template.metadata = metadata template.name = entry.get('realname', entry.get('name')) template.path = entry.get('realname', entry.get('name')) template.source_path = self.name - template.repo = self.setup['repo'] + template.repo = Bcfg2.Options.setup.repository return template.respond() get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index 516eba2f6..e2a2f696a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -21,8 +21,8 @@ class CfgEncryptedGenerator(CfgGenerator): #: .genshi.crypt and .cheetah.crypt files __priority__ = 50 - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 0521485e8..f69ab8e5f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -37,7 +37,7 @@ class CfgEncryptedGenshiGenerator(CfgGenshiGenerator): #: when it's read in __loader_cls__ = EncryptedTemplateLoader - def __init__(self, fname, spec, encoding): - CfgGenshiGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenshiGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index d06b864ac..953473a12 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -15,8 +15,8 @@ class CfgExternalCommandVerifier(CfgVerifier): #: Handle :file:`:test` files __basenames__ = [':test'] - def __init__(self, name, specific, encoding): - CfgVerifier.__init__(self, name, specific, encoding) + def __init__(self, name, specific): + CfgVerifier.__init__(self, name, specific) self.cmd = [] self.exc = Executor(timeout=30) __init__.__doc__ = CfgVerifier.__init__.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index e056c871a..7ba8c4491 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -5,9 +5,9 @@ import re import sys import traceback +import Bcfg2.Options from Bcfg2.Server.Plugin import PluginExecutionError, removecomment from Bcfg2.Server.Plugins.Cfg import CfgGenerator - from genshi.template import TemplateLoader, NewTextTemplate from genshi.template.eval import UndefinedError, Suite @@ -70,8 +70,8 @@ class CfgGenshiGenerator(CfgGenerator): #: occurred. pyerror_re = re.compile(r'<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>') - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) self.template = None self.loader = self.__loader_cls__(max_cache_size=0) __init__.__doc__ = CfgGenerator.__init__.__doc__ @@ -87,13 +87,15 @@ class CfgGenshiGenerator(CfgGenerator): metadata=metadata, path=self.name, source_path=self.name, - repo=self.setup['repo']).filter(removecomment) + repo=Bcfg2.Options.setup.repository).filter(removecomment) try: try: - return stream.render('text', encoding=self.encoding, + return stream.render('text', + encoding=Bcfg2.Options.setup.encoding, strip_whitespace=False) except TypeError: - return stream.render('text', encoding=self.encoding) + return stream.render('text', + encoding=Bcfg2.Options.setup.encoding) except UndefinedError: # a failure in a genshi expression _other_ than %{ python ... %} err = sys.exc_info()[1] @@ -172,8 +174,9 @@ class CfgGenshiGenerator(CfgGenerator): def handle_event(self, event): CfgGenerator.handle_event(self, event) try: - self.template = self.loader.load(self.name, cls=NewTextTemplate, - encoding=self.encoding) + self.template = \ + self.loader.load(self.name, cls=NewTextTemplate, + encoding=Bcfg2.Options.setup.encoding) except: raise PluginExecutionError("Failed to load template: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index 862726788..7bb5d3cf5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -3,8 +3,8 @@ import os import shutil import tempfile +import Bcfg2.Options from Bcfg2.Utils import Executor -from Bcfg2.Options import get_option_parser from Bcfg2.Server.Plugin import StructFile from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError from Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator import CfgPublicKeyCreator @@ -25,6 +25,14 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): #: Handle XML specifications of private keys __basenames__ = ['privkey.xml'] + options = [ + Bcfg2.Options.Option( + cf=("sshkeys", "category"), dest="sshkeys_category", + help="Metadata category that generated SSH keys are specific to"), + Bcfg2.Options.Option( + cf=("sshkeys", "passphrase"), dest="sshkeys_passphrase", + help="Passphrase used to encrypt generated SSH private keys")] + def __init__(self, fname): CfgCreator.__init__(self, fname) StructFile.__init__(self, fname) @@ -32,27 +40,15 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): pubkey_path = os.path.dirname(self.name) + ".pub" pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path)) self.pubkey_creator = CfgPublicKeyCreator(pubkey_name) - self.setup = get_option_parser() self.cmd = Executor() __init__.__doc__ = CfgCreator.__init__.__doc__ @property - def category(self): - """ The name of the metadata category that generated keys are - specific to """ - if (self.setup.cfp.has_section("sshkeys") and - self.setup.cfp.has_option("sshkeys", "category")): - return self.setup.cfp.get("sshkeys", "category") - return None - - @property def passphrase(self): """ The passphrase used to encrypt private keys """ - if (HAS_CRYPTO and - self.setup.cfp.has_section("sshkeys") and - self.setup.cfp.has_option("sshkeys", "passphrase")): - return Bcfg2.Server.Encryption.get_passphrases()[ - self.setup.cfp.get("sshkeys", "passphrase")] + if HAS_CRYPTO and Bcfg2.Options.setup.sshkeys_passphrase: + return Bcfg2.Options.setup.passphrases[ + Bcfg2.Options.setup.sshkeys_passphrase] return None def handle_event(self, event): @@ -141,7 +137,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): """ if spec is None: spec = self.XMLMatch(metadata) - category = spec.get("category", self.category) + category = spec.get("category", Bcfg2.Options.setup.sshkeys_category) if category is None: per_host_default = "true" else: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index fc3de3d68..a7fa92201 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -3,18 +3,14 @@ import re import os import sys -import stat import errno import operator import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin -import Bcfg2.Server.Lint -from fnmatch import fnmatch from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 -from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ - any, oct_mode +from Bcfg2.Compat import u_str, unicode, b64encode, any, oct_mode # pylint: enable=W0622 #: CFG is a reference to the :class:`Bcfg2.Server.Plugins.Cfg.Cfg` @@ -25,27 +21,8 @@ from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ #: facility for passing it otherwise. CFG = None -_HANDLERS = [] - -def handlers(): - """ A list of Cfg handler classes. Loading the handlers must - be done at run-time, not at compile-time, or it causes a - circular import and Bad Things Happen.""" - if not _HANDLERS: - for submodule in walk_packages(path=__path__, prefix=__name__ + "."): - mname = submodule[1].rsplit('.', 1)[-1] - module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, - mname) - hdlr = getattr(module, mname) - if issubclass(hdlr, CfgBaseFileMatcher): - _HANDLERS.append(hdlr) - _HANDLERS.sort(key=operator.attrgetter("__priority__")) - return _HANDLERS - - -class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, - Bcfg2.Server.Plugin.Debuggable): +class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): """ .. currentmodule:: Bcfg2.Server.Plugins.Cfg CfgBaseFileMatcher is the parent class for all Cfg handler @@ -89,12 +66,8 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, #: Flag to indicate an experimental handler. experimental = False - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) - Bcfg2.Server.Plugin.Debuggable.__init__(self) - self.encoding = encoding - self.setup = Bcfg2.Options.get_option_parser() + def __init__(self, name, specific): + Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific) __init__.__doc__ = Bcfg2.Server.Plugin.SpecificData.__init__.__doc__ + \ """ .. ----- @@ -185,7 +158,7 @@ class CfgGenerator(CfgBaseFileMatcher): client. See :class:`Bcfg2.Server.Plugin.helpers.EntrySet` for more details on how the best handler is chosen.""" - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # we define an __init__ that just calls the parent __init__, # so that we can set the docstring on __init__ to something # different from the parent __init__ -- namely, the parent @@ -193,7 +166,7 @@ class CfgGenerator(CfgBaseFileMatcher): # which we use to delineate the actual docs from the # .. autoattribute hacks we have to do to get private # attributes included in sphinx 1.0 """ - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def get_data(self, entry, metadata): # pylint: disable=W0613 @@ -213,9 +186,9 @@ class CfgFilter(CfgBaseFileMatcher): """ CfgFilters modify the initial content of a file after it has been generated by a :class:`Bcfg2.Server.Plugins.Cfg.CfgGenerator`. """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # see comment on CfgGenerator.__init__ above - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def modify_data(self, entry, metadata, data): @@ -253,7 +226,7 @@ class CfgInfo(CfgBaseFileMatcher): .. ----- .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgInfo.__specific__ """ - CfgBaseFileMatcher.__init__(self, fname, None, None) + CfgBaseFileMatcher.__init__(self, fname, None) def bind_info_to_entry(self, entry, metadata): """ Assign the appropriate attributes to the entry, modifying @@ -276,9 +249,9 @@ class CfgVerifier(CfgBaseFileMatcher): etc.), or both. """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # see comment on CfgGenerator.__init__ above - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def verify_entry(self, entry, metadata, data): @@ -317,7 +290,7 @@ class CfgCreator(CfgBaseFileMatcher): .. ----- .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgCreator.__specific__ """ - CfgBaseFileMatcher.__init__(self, fname, None, None) + CfgBaseFileMatcher.__init__(self, fname, None) def create_data(self, entry, metadata): """ Create new data for the given entry and write it to disk @@ -431,26 +404,30 @@ class CfgDefaultInfo(CfgInfo): bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ -class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, - Bcfg2.Server.Plugin.Debuggable): +class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ Handle a collection of host- and group-specific Cfg files with multiple different Cfg handlers in a single directory. """ - def __init__(self, basename, path, entry_type, encoding): - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, - entry_type, encoding) - Bcfg2.Server.Plugin.Debuggable.__init__(self) + def __init__(self, basename, path, entry_type): + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type) self.specific = None self._handlers = None - self.setup = Bcfg2.Options.get_option_parser() __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ def set_debug(self, debug): - rv = Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) + rv = Bcfg2.Server.Plugin.EntrySet.set_debug(self, debug) for entry in self.entries.values(): entry.set_debug(debug) return rv + @property + def handlers(self): + """ A list of Cfg handler classes. """ + if self._handlers is None: + self._handlers = Bcfg2.Options.setup.cfg_handlers + self._handlers.sort(key=operator.attrgetter("__priority__")) + return self._handlers + def handle_event(self, event): """ Dispatch a FAM event to :func:`entry_init` or the appropriate child handler object. @@ -467,7 +444,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # process a bogus changed event like a created return - for hdlr in handlers(): + for hdlr in self.handlers: if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -560,7 +537,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # most specific to least specific. data = fltr.modify_data(entry, metadata, data) - if self.setup['validate']: + if Bcfg2.Options.setup.cfg_validation: try: self._validate_data(entry, metadata, data) except CfgVerificationError: @@ -576,7 +553,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, if not isinstance(data, unicode): if not isinstance(data, str): data = data.decode('utf-8') - data = u_str(data, self.encoding) + data = u_str(data, Bcfg2.Options.setup.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) @@ -757,10 +734,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, self.logger.error(msg) raise PluginExecutionError(msg) try: - etext = new_entry['text'].encode(self.encoding) + etext = new_entry['text'].encode(Bcfg2.Options.setup.encoding) except: msg = "Cfg: Cannot encode content of %s as %s" % \ - (name, self.encoding) + (name, Bcfg2.Options.setup.encoding) self.logger.error(msg) raise PluginExecutionError(msg) open(name, 'w').write(etext) @@ -785,6 +762,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, flag=log) +class CfgHandlerAction(Bcfg2.Options.ComponentAction): + bases = ['Bcfg2.Server.Plugins.Cfg'] + + class Cfg(Bcfg2.Server.Plugin.GroupSpool, Bcfg2.Server.Plugin.PullTarget): """ The Cfg plugin provides a repository to describe configuration @@ -796,17 +777,27 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, es_cls = CfgEntrySet es_child_cls = Bcfg2.Server.Plugin.SpecificData + options = Bcfg2.Server.Plugin.GroupSpool.options + [ + Bcfg2.Options.BooleanOption( + '--cfg-validation', cf=('cfg', 'validation'), default=True, + help='Run validation on Cfg files'), + Bcfg2.Options.Option( + cf=("cfg", "handlers"), dest="cfg_handlers", + help="Cfg handlers to load", + type=Bcfg2.Options.Types.comma_list, action=CfgHandlerAction, + default=['CfgAuthorizedKeysGenerator', 'CfgEncryptedGenerator', + 'CfgCheetahGenerator', 'CfgEncryptedCheetahGenerator', + 'CfgGenshiGenerator', 'CfgEncryptedGenshiGenerator', + 'CfgExternalCommandVerifier', 'CfgInfoXML', + 'CfgPlaintextGenerator', + 'CfgPrivateKeyCreator', 'CfgPublicKeyCreator'])] + def __init__(self, core, datastore): global CFG # pylint: disable=W0603 Bcfg2.Server.Plugin.GroupSpool.__init__(self, core, datastore) Bcfg2.Server.Plugin.PullTarget.__init__(self) CFG = self - - setup = Bcfg2.Options.get_option_parser() - if 'validate' not in setup: - setup.add_option('validate', Bcfg2.Options.CFG_VALIDATION) - setup.reparse() __init__.__doc__ = Bcfg2.Server.Plugin.GroupSpool.__init__.__doc__ def has_generator(self, entry, metadata): @@ -840,96 +831,3 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, log) AcceptPullData.__doc__ = \ Bcfg2.Server.Plugin.PullTarget.AcceptPullData.__doc__ - - -class CfgLint(Bcfg2.Server.Lint.ServerPlugin): - """ warn about usage of .cat and .diff files """ - - def Run(self): - for basename, entry in list(self.core.plugins['Cfg'].entries.items()): - self.check_pubkey(basename, entry) - self.check_missing_files() - - @classmethod - def Errors(cls): - return {"no-pubkey-xml": "warning", - "unknown-cfg-files": "error", - "extra-cfg-files": "error"} - - def check_pubkey(self, basename, entry): - """ check that privkey.xml files have corresponding pubkey.xml - files """ - if "privkey.xml" not in entry.entries: - return - privkey = entry.entries["privkey.xml"] - if not self.HandlesFile(privkey.name): - return - - pubkey = basename + ".pub" - if pubkey not in self.core.plugins['Cfg'].entries: - self.LintError("no-pubkey-xml", - "%s has no corresponding pubkey.xml at %s" % - (basename, pubkey)) - else: - pubset = self.core.plugins['Cfg'].entries[pubkey] - if "pubkey.xml" not in pubset.entries: - self.LintError("no-pubkey-xml", - "%s has no corresponding pubkey.xml at %s" % - (basename, pubkey)) - - def _list_path_components(self, path): - """ Get a list of all components of a path. E.g., - ``self._list_path_components("/foo/bar/foobaz")`` would return - ``["foo", "bar", "foo", "baz"]``. The list is not guaranteed - to be in order.""" - rv = [] - remaining, component = os.path.split(path) - while component != '': - rv.append(component) - remaining, component = os.path.split(remaining) - return rv - - def check_missing_files(self): - """ check that all files on the filesystem are known to Cfg """ - cfg = self.core.plugins['Cfg'] - - # first, collect ignore patterns from handlers - ignore = set() - for hdlr in handlers(): - ignore.update(hdlr.__ignore__) - - # next, get a list of all non-ignored files on the filesystem - all_files = set() - for root, _, files in os.walk(cfg.data): - for fname in files: - fpath = os.path.join(root, fname) - # check against the handler ignore patterns and the - # global FAM ignore list - if (not any(fname.endswith("." + i) for i in ignore) and - not any(fnmatch(fpath, p) - for p in self.config['ignore']) and - not any(fnmatch(c, p) - for p in self.config['ignore'] - for c in self._list_path_components(fpath))): - all_files.add(fpath) - - # next, get a list of all files known to Cfg - cfg_files = set() - for root, eset in cfg.entries.items(): - cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) - for fname in eset.entries.keys()) - - # finally, compare the two - unknown_files = all_files - cfg_files - extra_files = cfg_files - all_files - if unknown_files: - self.LintError( - "unknown-cfg-files", - "Files on the filesystem could not be understood by Cfg: %s" % - "; ".join(unknown_files)) - if extra_files: - self.LintError( - "extra-cfg-files", - "Cfg has entries for files that do not exist on the " - "filesystem: %s\nThis is probably a bug." % - "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py index 0054a8a37..09fbfaea7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cvs.py +++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py @@ -20,7 +20,7 @@ class Cvs(Bcfg2.Server.Plugin.Version): def get_revision(self): """Read cvs revision information for the Bcfg2 repository.""" result = self.cmd.run(["env LC_ALL=C", "cvs", "log"], - shell=True, cwd=self.vcs_root) + shell=True, cwd=Bcfg2.Options.setup.vcs_root) try: return result.stdout.splitlines()[0].strip() except (IndexError, AttributeError): diff --git a/src/lib/Bcfg2/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py index 2c6dde393..b48809cac 100644 --- a/src/lib/Bcfg2/Server/Plugins/Darcs.py +++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py @@ -20,7 +20,7 @@ class Darcs(Bcfg2.Server.Plugin.Version): def get_revision(self): """Read Darcs changeset information for the Bcfg2 repository.""" result = self.cmd.run(["env LC_ALL=C", "darcs", "changes"], - shell=True, cwd=self.vcs_root) + shell=True, cwd=Bcfg2.Options.setup.vcs_root) if result.success: return result.stdout.splitlines()[0].strip() else: diff --git a/src/lib/Bcfg2/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py index 04c14aa96..79e2ca0e2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Defaults.py +++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py @@ -9,6 +9,8 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, """Set default attributes on bound entries""" __author__ = 'bcfg-dev@mcs.anl.gov' + options = Bcfg2.Server.Plugin.PrioDir.options + # Rules is a Generator that happens to implement all of the # functionality we want, so we overload it, but Defaults should # _not_ handle any entries; it does its stuff in the structure diff --git a/src/lib/Bcfg2/Server/Plugins/Deps.py b/src/lib/Bcfg2/Server/Plugins/Deps.py index 312b03bae..fa821aad3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Deps.py +++ b/src/lib/Bcfg2/Server/Plugins/Deps.py @@ -48,7 +48,8 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, prereqs = self.calculate_prereqs(metadata, entries) self.cache[(entries, groups)] = prereqs - newstruct = lxml.etree.Element("Independent") + newstruct = lxml.etree.Element("Independent", + name=self.__class__.__name__) for tag, name in prereqs: lxml.etree.SubElement(newstruct, tag, name=name) structures.append(newstruct) diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index a3bba14f3..45511eb52 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -8,7 +8,6 @@ import os import sys import errno import lxml.etree -import Bcfg2.Options import Bcfg2.Server import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor @@ -220,12 +219,12 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, return self.logger.info("Writing %s for %s" % (infoxml, data.get("name"))) + default_mdata = Bcfg2.Server.Plugin.default_path_metadata() info = lxml.etree.Element( "Info", - owner=data.get("owner", Bcfg2.Options.MDATA_OWNER.value), - group=data.get("group", Bcfg2.Options.MDATA_GROUP.value), - mode=data.get("mode", Bcfg2.Options.MDATA_MODE.value), - encoding=entry.get("encoding", Bcfg2.Options.ENCODING.value)) + owner=data.get("owner", default_mdata['owner']), + group=data.get("group", default_mdata['group']), + mode=data.get("mode", default_mdata['mode'])) root = lxml.etree.Element("FileInfo") root.append(info) diff --git a/src/lib/Bcfg2/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index 05cf4e5d4..f6aa3221a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -20,7 +20,7 @@ class Fossil(Bcfg2.Server.Plugin.Version): def get_revision(self): """Read fossil revision information for the Bcfg2 repository.""" result = self.cmd.run(["env LC_ALL=C", "fossil", "info"], - shell=True, cwd=self.vcs_root) + shell=True, cwd=Bcfg2.Options.setup.vcs_root) try: revision = None for line in result.stdout.splitlines(): diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 58a5c58f0..d0502ed6a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -23,7 +23,7 @@ class Git(Version): def __init__(self, core, datastore): Version.__init__(self, core, datastore) if HAS_GITPYTHON: - self.repo = git.Repo(self.vcs_root) + self.repo = git.Repo(Bcfg2.Options.setup.vcs_root) self.cmd = None else: self.logger.debug("Git: GitPython not found, using CLI interface " @@ -45,7 +45,8 @@ class Git(Version): return self.repo.head.commit.hexsha else: cmd = ["git", "--git-dir", self.vcs_path, - "--work-tree", self.vcs_root, "rev-parse", "HEAD"] + "--work-tree", Bcfg2.Options.setup.vcs_root, + "rev-parse", "HEAD"] self.debug_log("Git: Running %s" % cmd) result = self.cmd.run(cmd) if not result.success: @@ -53,7 +54,7 @@ class Git(Version): return result.stdout except: raise PluginExecutionError("Git: Error getting revision from %s: " - "%s" % (self.vcs_root, + "%s" % (Bcfg2.Options.setup.vcs_root, sys.exc_info()[1])) def Update(self, ref=None): @@ -62,14 +63,15 @@ class Git(Version): """ self.logger.info("Git: Git.Update(ref='%s')" % ref) self.debug_log("Git: Performing garbage collection on repo at %s" % - self.vcs_root) + Bcfg2.Options.setup.vcs_root) try: self._log_git_cmd(self.repo.git.gc('--auto')) except git.GitCommandError: self.logger.warning("Git: Failed to perform garbage collection: %s" % sys.exc_info()[1]) - self.debug_log("Git: Fetching all refs for repo at %s" % self.vcs_root) + self.debug_log("Git: Fetching all refs for repo at %s" % + Bcfg2.Options.setup.vcs_root) try: self._log_git_cmd(self.repo.git.fetch('--all')) except git.GitCommandError: @@ -102,5 +104,5 @@ class Git(Version): "upstream: %s" % sys.exc_info()[1]) self.logger.info("Git: Repo at %s updated to %s" % - (self.vcs_root, self.get_revision())) + (Bcfg2.Options.setup.vcs_root, self.get_revision())) return True diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 3e5508160..90cbd083d 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -122,40 +122,3 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, def get_additional_groups(self, metadata): return self.config.process_patterns(metadata.hostname) - - -class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns - <server-plugins-grouping-grouppatterns>` patterns for validity. - This is simply done by trying to create a - :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for - each pattern, and catching exceptions and presenting them as - ``bcfg2-lint`` errors.""" - - def Run(self): - cfg = self.core.plugins['GroupPatterns'].config - for entry in cfg.xdata.xpath('//GroupPattern'): - groups = [g.text for g in entry.findall('Group')] - self.check(entry, groups, ptype='NamePattern') - self.check(entry, groups, ptype='NameRange') - - @classmethod - def Errors(cls): - return {"pattern-fails-to-initialize": "error"} - - def check(self, entry, groups, ptype="NamePattern"): - """ Check a single pattern for validity """ - if ptype == "NamePattern": - pmap = lambda p: PatternMap(p, None, groups) - else: - pmap = lambda p: PatternMap(None, p, groups) - - for el in entry.findall(ptype): - pat = el.text - try: - pmap(pat) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - self.LintError("pattern-fails-to-initialize", - "Failed to initialize %s %s for %s: %s" % - (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Bcfg2/Server/Plugins/Hg.py b/src/lib/Bcfg2/Server/Plugins/Hg.py index 3fd3918bd..f9a9f858c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Hg.py +++ b/src/lib/Bcfg2/Server/Plugins/Hg.py @@ -20,7 +20,7 @@ class Hg(Bcfg2.Server.Plugin.Version): def get_revision(self): """Read hg revision information for the Bcfg2 repository.""" try: - repo_path = self.vcs_root + "/" + repo_path = Bcfg2.Options.setup.vcs_root + "/" repo = hg.repository(ui.ui(), repo_path) tip = repo.changelog.tip() return repo.changelog.rev(tip) diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index 8e8b078d9..6fc89b4f3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -3,7 +3,6 @@ import logging import sys import time import traceback -import Bcfg2.Options import Bcfg2.Server.Plugin logger = logging.getLogger('Bcfg2.Plugins.Ldap') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index f355fd7de..24adee4f4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -12,23 +12,28 @@ import socket import logging import lxml.etree import Bcfg2.Server -import Bcfg2.Server.Lint +import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 from Bcfg2.version import Bcfg2VersionInfo -try: - from django.db import models - HAS_DJANGO = True -except ImportError: - HAS_DJANGO = False -LOGGER = logging.getLogger(__name__) +MetadataClientModel = None +HAS_DJANGO = False -if HAS_DJANGO: +def load_django_models(): + global MetadataClientModel, ClientVersions, HAS_DJANGO + + try: + from django.db import models + HAS_DJANGO = True + except ImportError: + HAS_DJANGO = False + return + class MetadataClientModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): """ django model for storing clients in the database """ @@ -39,12 +44,12 @@ if HAS_DJANGO: Bcfg2.Server.Plugin.DatabaseBacked): """ dict-like object to make it easier to access client bcfg2 versions from the database """ - create = False def __getitem__(self, key): try: - return MetadataClientModel.objects.get(hostname=key).version + return MetadataClientModel.objects.get( + hostname=key).version except MetadataClientModel.DoesNotExist: raise KeyError(key) @@ -350,6 +355,8 @@ class MetadataQuery(object): def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): + self.logger = logging.getLogger(self.__class__.__name__) + #: Get :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` #: object for the given hostname. #: @@ -402,8 +409,9 @@ class MetadataQuery(object): @wraps(func) def inner(arg): if isinstance(arg, str): - LOGGER.warning("%s: %s takes a list as argument, not a string" - % (self.__class__.__name__, func.__name__)) + self.logger.warning("%s: %s takes a list as argument, not a " + "string" % (self.__class__.__name__, + func.__name__)) return func(arg) # pylint: enable=C0111 @@ -493,6 +501,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, __author__ = 'bcfg-dev@mcs.anl.gov' sort_order = 500 + options = Bcfg2.Server.Plugin.DatabaseBacked.options + [ + Bcfg2.Options.Common.password, + Bcfg2.Options.BooleanOption( + cf=('metadata', 'use_database'), dest="metadata_db", + help="Use database capabilities of the Metadata plugin"), + Bcfg2.Options.Option( + cf=('communication', 'authentication'), default='cert+password', + choices=['cert', 'bootstrap', 'cert+password'], + help='Default client authentication method')] + def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Metadata.__init__(self) Bcfg2.Server.Plugin.Caching.__init__(self) @@ -541,7 +559,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.session_cache = {} self.default = None self.pdirty = False - self.password = core.setup['password'] + self.password = Bcfg2.Options.setup.password self.query = MetadataQuery(core.build_metadata, lambda: list(self.clients), self.get_client_names_by_groups, @@ -1296,6 +1314,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return False resolved = self.resolve_client(addresspair) if resolved.lower() == client.lower(): + self.logger.debug("Client %s address validates" % client) return True else: self.logger.error("Got request for %s from incorrect address %s" % @@ -1315,7 +1334,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, client = certinfo['commonName'] self.debug_log("Got cN %s; using as client name" % client) auth_type = self.auth.get(client, - self.core.setup['authentication']) + Bcfg2.Options.setup.authentication) elif user == 'root': id_method = 'address' try: @@ -1346,6 +1365,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # remember the cert-derived client name for this connection if client in self.floating: self.session_cache[address] = (time.time(), client) + self.logger.debug("Client %s certificate validates" % client) # we are done if cert+password not required return True @@ -1372,13 +1392,14 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # populate the session cache if user != 'root': self.session_cache[address] = (time.time(), client) + self.logger.debug("Client %s authenticated successfully" % client) return True # pylint: enable=R0911,R0912 def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, - self.core.setup['authentication']) == 'bootstrap': + Bcfg2.Options.setup.authentication) == 'bootstrap': self.update_client(metadata.hostname, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): @@ -1494,149 +1515,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, (group.get('name'), parent.get('name'))) return rv - -class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin for :ref:`Metadata - <server-plugins-grouping-metadata>`. This checks for several things: - - * ``<Client>`` tags nested inside other ``<Client>`` tags; - * Deprecated options (like ``location="floating"``); - * Profiles that don't exist, or that aren't profile groups; - * Groups or clients that are defined multiple times; - * Multiple default groups or a default group that isn't a profile - group. - """ - - def Run(self): - self.nested_clients() - self.deprecated_options() - self.bogus_profiles() - self.duplicate_groups() - self.duplicate_default_groups() - self.duplicate_clients() - self.default_is_profile() - - @classmethod - def Errors(cls): - return {"nested-client-tags": "warning", - "deprecated-clients-options": "warning", - "nonexistent-profile-group": "error", - "non-profile-set-as-profile": "error", - "duplicate-group": "error", - "duplicate-client": "error", - "multiple-default-groups": "error", - "default-is-not-profile": "error"} - - def deprecated_options(self): - """ Check for the ``location='floating'`` option, which has - been deprecated in favor of ``floating='true'``. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - clientdata = self.metadata.clients_xml.xdata - for el in clientdata.xpath("//Client"): - loc = el.get("location") - if loc: - if loc == "floating": - floating = True - else: - floating = False - self.LintError("deprecated-clients-options", - "The location='%s' option is deprecated. " - "Please use floating='%s' instead:\n%s" % - (loc, floating, self.RenderXML(el))) - - def nested_clients(self): - """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, - which is either redundant or will never match. """ - groupdata = self.metadata.groups_xml.xdata - for el in groupdata.xpath("//Client//Client"): - self.LintError("nested-client-tags", - "Client %s nested within Client tag: %s" % - (el.get("name"), self.RenderXML(el))) - - def bogus_profiles(self): - """ Check for clients that have profiles that are either not - flagged as profile groups in ``groups.xml``, or don't exist. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - for client in self.metadata.clients_xml.xdata.findall('.//Client'): - profile = client.get("profile") - if profile not in self.metadata.groups: - self.LintError("nonexistent-profile-group", - "%s has nonexistent profile group %s:\n%s" % - (client.get("name"), profile, - self.RenderXML(client))) - elif not self.metadata.groups[profile].is_profile: - self.LintError("non-profile-set-as-profile", - "%s is set as profile for %s, but %s is not a " - "profile group:\n%s" % - (profile, client.get("name"), profile, - self.RenderXML(client))) - - def duplicate_default_groups(self): - """ Check for multiple default groups. """ - defaults = [] - for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): - if grp.get("default", "false").lower() == "true": - defaults.append(self.RenderXML(grp)) - if len(defaults) > 1: - self.LintError("multiple-default-groups", - "Multiple default groups defined:\n%s" % - "\n".join(defaults)) - - def duplicate_clients(self): - """ Check for clients that are defined more than once. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - self.duplicate_entries( - self.metadata.clients_xml.xdata.xpath("//Client"), - "client") - - def duplicate_groups(self): - """ Check for groups that are defined more than once. We - count a group tag as a definition if it a) has profile or - public set; or b) has any children.""" - allgroups = [ - g - for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") - if g.get("profile") or g.get("public") or g.getchildren()] - self.duplicate_entries(allgroups, "group") - - def duplicate_entries(self, allentries, etype): - """ Generic duplicate entry finder. - - :param allentries: A list of all entries to check for - duplicates. - :type allentries: list of lxml.etree._Element - :param etype: The entry type. This will be used to determine - the error name (``duplicate-<etype>``) and for - display to the end user. - :type etype: string - """ - entries = dict() - for el in allentries: - if el.get("name") in entries: - entries[el.get("name")].append(self.RenderXML(el)) - else: - entries[el.get("name")] = [self.RenderXML(el)] - for ename, els in entries.items(): - if len(els) > 1: - self.LintError("duplicate-%s" % etype, - "%s %s is defined multiple times:\n%s" % - (etype.title(), ename, "\n".join(els))) - - def default_is_profile(self): - """ Ensure that the default group is a profile group. """ - if (self.metadata.default and - not self.metadata.groups[self.metadata.default].is_profile): - xdata = \ - self.metadata.groups_xml.xdata.xpath("//Group[@name='%s']" % - self.metadata.default)[0] - self.LintError("default-is-not-profile", - "Default group is not a profile group:\n%s" % - self.RenderXML(xdata)) + @staticmethod + def options_parsed_hook(): + load_django_models() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 4d05f9d97..0df8624f6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -73,20 +73,17 @@ The Collection Module --------------------- """ -import sys import copy -import logging import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Server.FileMonitor import get_fam -from Bcfg2.Options import get_option_parser +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import any, md5 # pylint: disable=W0622 +from Bcfg2.Server.FileMonitor import get_fam from Bcfg2.Server.Statistics import track_statistics -LOGGER = logging.getLogger(__name__) - -class Collection(list, Bcfg2.Server.Plugin.Debuggable): +class Collection(list, Debuggable): """ ``Collection`` objects represent the set of :class:`Bcfg2.Server.Plugins.Packages.Source` objects that apply to a given client, and can be used to query all software @@ -119,15 +116,14 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): .. ----- .. autoattribute:: __package_groups__ """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) + Debuggable.__init__(self) list.__init__(self, sources) - self.debug_flag = debug + self.debug_flag = self.debug_flag or debug self.metadata = metadata self.basepath = basepath self.cachepath = cachepath self.virt_pkgs = dict() self.fam = get_fam() - self.setup = get_option_parser() try: self.ptype = sources[0].ptype @@ -447,9 +443,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ for pkg in pkglist: lxml.etree.SubElement(entry, 'BoundPackage', name=pkg, - version=self.setup.cfp.get("packages", - "version", - default="auto"), + version=Bcfg2.Options.setup.packages_version, type=self.ptype, origin='Packages') def get_new_packages(self, initial, complete): @@ -601,22 +595,9 @@ def get_collection_class(source_type): :type source_type: string :returns: type - the Collection subclass that should be used to instantiate an object to contain sources of the given type. """ - modname = "Bcfg2.Server.Plugins.Packages.%s" % source_type.title() - try: - module = sys.modules[modname] - except KeyError: - try: - module = __import__(modname).Server.Plugins.Packages - except ImportError: - msg = "Packages: Unknown source type %s" % source_type - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - try: - cclass = getattr(module, source_type.title() + "Collection") - except AttributeError: - msg = "Packages: No collection class found for %s sources" % \ - source_type - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - return cclass + cls = None + for mod in Bcfg2.Options.setup.packages_backends: + if mod.__name__.endswith(".%s" % source_type.title()): + return getattr(mod, "%sCollection" % source_type.title()) + raise Bcfg2.Server.Plugin.PluginExecutionError( + "Packages: No collection class found for %s sources" % source_type) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 9ff2d53a0..1af046ec0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -4,14 +4,12 @@ import os import sys import Bcfg2.Server.Plugin -from Bcfg2.Options import get_option_parser from Bcfg2.Server.Statistics import track_statistics from Bcfg2.Server.Plugins.Packages.Source import SourceInitError # pylint: disable=E0012,R0924 -class PackagesSources(Bcfg2.Server.Plugin.StructFile, - Bcfg2.Server.Plugin.Debuggable): +class PackagesSources(Bcfg2.Server.Plugin.StructFile): """ PackagesSources handles parsing of the :mod:`Bcfg2.Server.Plugins.Packages` ``sources.xml`` file, and the creation of the appropriate @@ -37,7 +35,6 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` - If ``sources.xml`` cannot be read """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) Bcfg2.Server.Plugin.StructFile.__init__(self, filename, should_monitor=True) @@ -54,8 +51,6 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, err = sys.exc_info()[1] self.logger.error("Could not create Packages cache at %s: %s" % (self.cachepath, err)) - #: The Bcfg2 options dict - self.setup = get_option_parser() #: The :class:`Bcfg2.Server.Plugins.Packages.Packages` that #: instantiated this ``PackagesSources`` object @@ -69,10 +64,9 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, self.parsed = set() def set_debug(self, debug): - Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) + Bcfg2.Server.Plugin.StructFile.set_debug(self, debug) for source in self.entries: source.set_debug(debug) - set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ def HandleEvent(self, event=None): """ HandleEvent is called whenever the FAM registers an event. @@ -138,15 +132,13 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, xsource.get("url")))) return None - try: - module = getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cls = getattr(module, "%sSource" % stype.title()) - except (ImportError, AttributeError): - err = sys.exc_info()[1] - self.logger.error("Packages: Unknown source type %s (%s)" % (stype, - err)) + cls = None + for mod in Bcfg2.Options.setup.packages_backends: + if mod.__name__.endswith(".%s" % stype.title()): + cls = getattr(mod, "%sSource" % stype.title()) + break + else: + self.logger.error("Packages: Unknown source type %s" % stype) return None try: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 767ac13ac..e1659dbb3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -50,7 +50,7 @@ import os import re import sys import Bcfg2.Server.Plugin -from Bcfg2.Options import get_option_parser +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ cPickle, md5 @@ -93,7 +93,7 @@ class SourceInitError(Exception): REPO_RE = re.compile(r'(?:pulp/repos/|/RPMS\.|/)([^/]+)/?$') -class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 +class Source(Debuggable): # pylint: disable=R0902 """ ``Source`` objects represent a single <Source> tag in ``sources.xml``. Note that a single Source tag can itself describe multiple repositories (if it uses the "url" attribute @@ -121,7 +121,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :type source: lxml.etree._Element :raises: :class:`Bcfg2.Server.Plugins.Packages.Source.SourceInitError` """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) + Debuggable.__init__(self) #: The base filesystem path under which cache data for this #: source should be stored @@ -130,9 +130,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: The XML tag that describes this source self.xsource = xsource - #: A Bcfg2 options dict - self.setup = get_option_parser() - #: A set of package names that are deemed "essential" by this #: source self.essentialpkgs = set() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 75dab3f76..0d49473c6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -59,12 +59,10 @@ import errno import socket import logging import lxml.etree -from lockfile import FileLock - -import Bcfg2.Server.FileMonitor import Bcfg2.Server.Plugin +import Bcfg2.Server.FileMonitor +from lockfile import FileLock from Bcfg2.Utils import Executor -from Bcfg2.Options import get_option_parser # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -109,6 +107,30 @@ PULPSERVER = None PULPCONFIG = None +options = [ + Bcfg2.Options.PathOption( + cf=("packages:yum", "helper"), dest="yum_helper", + help="Path to the bcfg2-yum-helper executable"), + Bcfg2.Options.BooleanOption( + cf=("packages:yum", "use_yum_libraries"), + help="Use Python yum libraries"), + Bcfg2.Options.PathOption( + cf=("packages:yum", "gpg_keypath"), default="/etc/pki/rpm-gpg", + help="GPG key path on the client"), + Bcfg2.Options.Option( + cf=("packages:yum", "*"), dest="yum_options", + help="Other yum options to include in generated yum configs")] +if HAS_PULP: + options.append( + Bcfg2.Options.Option( + cf=("packages:pulp", "username"), dest="pulp_username", + help="Username for Pulp authentication")) + options.append( + Bcfg2.Options.Option( + cf=("packages:pulp", "password"), dest="pulp_password", + help="Password for Pulp authentication")) + + def _setup_pulp(): """ Connect to a Pulp server and pass authentication credentials. This only needs to be called once, but multiple calls won't hurt @@ -124,20 +146,6 @@ def _setup_pulp(): raise Bcfg2.Server.Plugin.PluginInitError(msg) if PULPSERVER is None: - setup = get_option_parser() - try: - username = setup.cfp.get("packages:pulp", "username") - password = setup.cfp.get("packages:pulp", "password") - except ConfigParser.NoSectionError: - msg = "Packages: No [pulp] section found in bcfg2.conf" - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - except ConfigParser.NoOptionError: - msg = "Packages: Required option not found in bcfg2.conf: %s" % \ - sys.exc_info()[1] - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - PULPCONFIG = ConsumerConfig() serveropts = PULPCONFIG.server @@ -145,7 +153,9 @@ def _setup_pulp(): int(serveropts['port']), serveropts['scheme'], serveropts['path']) - PULPSERVER.set_basic_auth_credentials(username, password) + PULPSERVER.set_basic_auth_credentials( + Bcfg2.Options.setup.pulp_username, + Bcfg2.Options.setup.pulp_password) server.set_active_server(PULPSERVER) return PULPSERVER @@ -357,10 +367,8 @@ class YumCollection(Collection): the default location. """ if not self._helper: # pylint: disable=W0212 - try: - self.__class__._helper = self.setup.cfp.get("packages:yum", - "helper") - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.__class__._helper = Bcfg2.Options.setup.yum_helper + if not self.__class__._helper: # first see if bcfg2-yum-helper is in PATH try: self.debug_log("Checking for bcfg2-yum-helper in $PATH") @@ -375,9 +383,7 @@ class YumCollection(Collection): def use_yum(self): """ True if we should use the yum Python libraries, False otherwise """ - return HAS_YUM and self.setup.cfp.getboolean("packages:yum", - "use_yum_libraries", - default=False) + return HAS_YUM and Bcfg2.Options.setup.use_yum_libraries @property def has_pulp_sources(self): @@ -413,15 +419,15 @@ class YumCollection(Collection): debuglevel="0", sslverify="0", reposdir="/dev/null") - if self.setup['debug']: + if Bcfg2.Options.setup.debug: mainopts['debuglevel'] = "5" - elif self.setup['verbose']: + elif Bcfg2.Options.setup.verbose: mainopts['debuglevel'] = "2" try: - for opt in self.setup.cfp.options("packages:yum"): + for opt, val in Bcfg2.Options.setup.yum_options.items(): if opt not in self.option_blacklist: - mainopts[opt] = self.setup.cfp.get("packages:yum", opt) + mainopts[opt] = val except ConfigParser.NoSectionError: pass @@ -543,8 +549,7 @@ class YumCollection(Collection): for key in needkeys: # figure out the path of the key on the client - keydir = self.setup.cfp.get("global", "gpg_keypath", - default="/etc/pki/rpm-gpg") + keydir = Bcfg2.Options.setup.gpg_keypath remotekey = os.path.join(keydir, os.path.basename(key)) localkey = os.path.join(self.keypath, os.path.basename(key)) kdata = open(localkey).read() @@ -763,8 +768,7 @@ class YumCollection(Collection): """ Given a package tuple, return a dict of attributes suitable for applying to either a Package or an Instance tag """ - attrs = dict(version=self.setup.cfp.get("packages", "version", - default="auto")) + attrs = dict(version=Bcfg2.Options.setup.packages_version) if attrs['version'] == 'any' or not isinstance(pkgtup, tuple): return attrs @@ -923,14 +927,10 @@ class YumCollection(Collection): ``bcfg2-yum-helper`` command. """ cmd = [self.helper, "-c", self.cfgfile] - if self.setup['verbose']: + if Bcfg2.Options.setup.verbose: cmd.append("-v") if self.debug_flag: - if not self.setup['verbose']: - # ensure that running in debug gets -vv, even if - # verbose is not enabled - cmd.append("-v") - cmd.append("-v") + cmd.append("-d") cmd.append(command) self.debug_log("Packages: running %s" % " ".join(cmd)) @@ -1053,9 +1053,7 @@ class YumSource(Source): def use_yum(self): """ True if we should use the yum Python libraries, False otherwise """ - return HAS_YUM and self.setup.cfp.getboolean("packages:yum", - "use_yum_libraries", - default=False) + return HAS_YUM and Bcfg2.Options.setup.use_yum_libraries def save_state(self): """ If using the builtin yum parser, save state to diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py new file mode 100644 index 000000000..32db0b32d --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py @@ -0,0 +1,384 @@ +""" Libraries for bcfg2-yum-helper plugin, used if yum library support +is enabled. The yum libs have horrific memory leaks, so apparently +the right way to get around that in long-running processes it to have +a short-lived helper. No, seriously -- check out the yum-updatesd +code. It's pure madness. """ + +import os +import sys +import yum +import logging +import Bcfg2.Options +import Bcfg2.Logger +from Bcfg2.Compat import wraps +from lockfile import FileLock, LockTimeout +try: + import json +except ImportError: + import simplejson as json + + +def pkg_to_tuple(package): + """ json doesn't distinguish between tuples and lists, but yum + does, so we convert a package in list format to one in tuple + format """ + if isinstance(package, list): + return tuple(package) + else: + return package + + +def pkgtup_to_string(package): + """ given a package tuple, return a human-readable string + describing the package """ + if package[3] in ['auto', 'any']: + return package[0] + + rv = [package[0], "-"] + if package[2]: + rv.extend([package[2], ':']) + rv.extend([package[3], '-', package[4]]) + if package[1]: + rv.extend(['.', package[1]]) + return ''.join(str(e) for e in rv) + + +class YumHelper(object): + """ Yum helper base object """ + + def __init__(self, cfgfile, verbose=1): + self.cfgfile = cfgfile + self.yumbase = yum.YumBase() + # pylint: disable=E1121,W0212 + try: + self.yumbase.preconf.debuglevel = verbose + self.yumbase.preconf.fn = cfgfile + self.yumbase._getConfig() + except AttributeError: + self.yumbase._getConfig(cfgfile, debuglevel=verbose) + # pylint: enable=E1121,W0212 + self.logger = logging.getLogger(self.__class__.__name__) + + +class DepSolver(YumHelper): + """ Yum dependency solver. This is used for operations that only + read from the yum cache, and thus operates in cacheonly mode. """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + # internally, yum uses an integer, not a boolean, for conf.cache + self.yumbase.conf.cache = 1 + self._groups = None + + def get_groups(self): + """ getter for the groups property """ + if self._groups is not None: + return self._groups + else: + return ["noarch"] + + def set_groups(self, groups): + """ setter for the groups property """ + self._groups = set(groups).union(["noarch"]) + + groups = property(get_groups, set_groups) + + def get_package_object(self, pkgtup, silent=False): + """ given a package tuple, get a yum package object """ + try: + matches = yum.packageSack.packagesNewestByName( + self.yumbase.pkgSack.searchPkgTuple(pkgtup)) + except yum.Errors.PackageSackError: + if not silent: + self.logger.warning("Package '%s' not found" % + self.get_package_name(pkgtup)) + matches = [] + except yum.Errors.RepoError: + err = sys.exc_info()[1] + self.logger.error("Temporary failure loading metadata for %s: %s" % + (self.get_package_name(pkgtup), err)) + matches = [] + + pkgs = self._filter_arch(matches) + if pkgs: + return pkgs[0] + else: + return None + + def get_group(self, group, ptype="default"): + """ Resolve a package group name into a list of packages """ + if group.startswith("@"): + group = group[1:] + + try: + if self.yumbase.comps.has_group(group): + group = self.yumbase.comps.return_group(group) + else: + self.logger.error("%s is not a valid group" % group) + return [] + except yum.Errors.GroupsError: + err = sys.exc_info()[1] + self.logger.warning(err) + return [] + + if ptype == "default": + return [p + for p, d in list(group.default_packages.items()) + if d] + elif ptype == "mandatory": + return [p + for p, m in list(group.mandatory_packages.items()) + if m] + elif ptype == "optional" or ptype == "all": + return group.packages + else: + self.logger.warning("Unknown group package type '%s'" % ptype) + return [] + + def _filter_arch(self, packages): + """ filter packages in the given list that do not have an + architecture in the list of groups for this client """ + matching = [] + for pkg in packages: + if pkg.arch in self.groups: + matching.append(pkg) + else: + self.logger.debug("%s has non-matching architecture (%s)" % + (pkg, pkg.arch)) + if matching: + return matching + else: + # no packages match architecture; we'll assume that the + # user knows what s/he is doing and this is a multiarch + # box. + return packages + + def get_package_name(self, package): + """ get the name of a package or virtual package from the + internal representation used by this Collection class """ + if isinstance(package, tuple): + if len(package) == 3: + return yum.misc.prco_tuple_to_string(package) + else: + return pkgtup_to_string(package) + else: + return str(package) + + def complete(self, packagelist): + """ resolve dependencies and generate a complete package list + from the given list of initial packages """ + packages = set() + unknown = set() + for pkg in packagelist: + if isinstance(pkg, tuple): + pkgtup = pkg + else: + pkgtup = (pkg, None, None, None, None) + pkgobj = self.get_package_object(pkgtup) + if not pkgobj: + self.logger.debug("Unknown package %s" % + self.get_package_name(pkg)) + unknown.add(pkg) + else: + if self.yumbase.tsInfo.exists(pkgtup=pkgobj.pkgtup): + self.logger.debug("%s added to transaction multiple times" + % pkgobj) + else: + self.logger.debug("Adding %s to transaction" % pkgobj) + self.yumbase.tsInfo.addInstall(pkgobj) + self.yumbase.resolveDeps() + + for txmbr in self.yumbase.tsInfo: + packages.add(txmbr.pkgtup) + return list(packages), list(unknown) + + +def acquire_lock(func): + """ decorator for CacheManager methods that gets and release a + lock while the method runs """ + @wraps(func) + def inner(self, *args, **kwargs): + """ Get and release a lock while running the function this + wraps. """ + self.logger.debug("Acquiring lock at %s" % self.lockfile) + while not self.lock.i_am_locking(): + try: + self.lock.acquire(timeout=60) # wait up to 60 seconds + except LockTimeout: + self.lock.break_lock() + self.lock.acquire() + try: + func(self, *args, **kwargs) + finally: + self.lock.release() + self.logger.debug("Released lock at %s" % self.lockfile) + + return inner + + +class CacheManager(YumHelper): + """ Yum cache manager. Unlike :class:`DepSolver`, this can write + to the yum cache, and so is used for operations that muck with the + cache. (Technically, :func:`CacheManager.clean_cache` could be in + either DepSolver or CacheManager, but for consistency I've put it + here.) """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + self.lockfile = \ + os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), + "lock") + self.lock = FileLock(self.lockfile) + + @acquire_lock + def clean_cache(self): + """ clean the yum cache """ + for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", + "ExpireCache"]: + # for reasons that are entirely obvious, all of the yum + # API clean* methods return a tuple of 0 (zero, always + # zero) and a list containing a single message about how + # many files were deleted. so useful. thanks, yum. + msg = getattr(self.yumbase, "clean%s" % mdtype)()[1][0] + if not msg.startswith("0 "): + self.logger.info(msg) + + @acquire_lock + def populate_cache(self): + """ populate the yum cache """ + for repo in self.yumbase.repos.findRepos('*'): + repo.metadata_expire = 0 + repo.mdpolicy = "group:all" + self.yumbase.doRepoSetup() + self.yumbase.repos.doSetup() + for repo in self.yumbase.repos.listEnabled(): + # this populates the cache as a side effect + repo.repoXML # pylint: disable=W0104 + try: + repo.getGroups() + except yum.Errors.RepoMDError: + pass # this repo has no groups + self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) + # this does something with the groups cache as a side effect + self.yumbase.comps # pylint: disable=W0104 + + +class HelperSubcommand(Bcfg2.Options.Subcommand): + # the value to JSON encode and print out if the command fails + fallback = None + + # whether or not this command accepts input on stdin + accept_input = True + + def __init__(self): + Bcfg2.Options.Subcommand.__init__(self) + self.verbosity = 0 + if Bcfg2.Options.setup.debug: + self.verbosity = 5 + elif Bcfg2.Options.setup.verbose: + self.verbosity = 1 + + def run(self, setup): + try: + data = json.loads(sys.stdin.read()) + except: # pylint: disable=W0702 + self.logger.error("Unexpected error decoding JSON input: %s" % + sys.exc_info()[1]) + print(json.dumps(self.fallback)) + return 2 + + try: + print(json.dumps(self._run(setup, data))) + except: # pylint: disable=W0702 + self.logger.error("Unexpected error running %s: %s" % + self.__class__.__name__.lower(), + sys.exc_info()[1], exc_info=1) + print(json.dumps(self.fallback)) + return 2 + return 0 + + def _run(self, setup, data): + raise NotImplementedError + + +class DepSolverSubcommand(HelperSubcommand): + def __init__(self): + HelperSubcommand.__init__(self) + self.depsolver = DepSolver(Bcfg2.Options.setup.yum_config, + self.verbosity) + + +class CacheManagerSubcommand(HelperSubcommand): + fallback = False + accept_input = False + + def __init__(self): + HelperSubcommand.__init__(self) + self.cachemgr = CacheManager(Bcfg2.Options.setup.yum_config, + self.verbosity) + + +class Clean(CacheManagerSubcommand): + def _run(self, setup, data): # pylint: disable=W0613 + self.cachemgr.clean_cache() + return True + + +class MakeCache(CacheManagerSubcommand): + def _run(self, setup, data): # pylint: disable=W0613 + self.cachemgr.populate_cache() + return True + + +class Complete(DepSolverSubcommand): + fallback = dict(packages=[], unknown=[]) + + def _run(self, _, data): + self.depsolver.groups = data['groups'] + self.fallback['unknown'] = data['packages'] + (packages, unknown) = self.depsolver.complete( + [pkg_to_tuple(p) for p in data['packages']]) + return dict(packages=list(packages), unknown=list(unknown)) + + +class GetGroups(DepSolverSubcommand): + def _run(self, _, data): + rv = dict() + for gdata in data: + if "type" in gdata: + packages = self.depsolver.get_group(gdata['group'], + ptype=gdata['type']) + else: + packages = self.depsolver.get_group(gdata['group']) + rv[gdata['group']] = list(packages) + return rv + + +Get_Groups = GetGroups + + +class CLI(Bcfg2.Options.CommandRegistry): + options = [ + Bcfg2.Options.PathOption( + "-c", "--yum-config", help="Yum config file"), + Bcfg2.Options.PositionalArgument( + "command", help="Yum helper command", + choices=['clean', 'complete', 'get_groups'])] + + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, globals().values(), + parent=HelperSubcommand) + parser = Bcfg2.Options.get_parser("Bcfg2 yum helper", + components=[self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + def run(self): + if not os.path.exists(Bcfg2.Options.setup.yum_config): + self.logger.error("Config file %s not found" % + Bcfg2.Options.setup.yum_config) + return 1 + return self.runcommand() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 20a75c678..e6240f39a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -7,21 +7,30 @@ import sys import glob import shutil import lxml.etree -import Bcfg2.Logger +import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ - MutableMapping +from Bcfg2.Compat import urlopen, HTTPError, URLError, MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources from Bcfg2.Server.Statistics import track_statistics -#: The default path for generated yum configs -YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" -#: The default path for generated apt configs -APT_CONFIG_DEFAULT = \ - "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +def packages_boolean(value): + """ For historical reasons, the Packages booleans 'resolver' and + 'metadata' both accept "enabled" in addition to the normal boolean + values. """ + if value == 'disabled': + return False + elif value == 'enabled': + return True + else: + return value + + +class PackagesBackendAction(Bcfg2.Options.ComponentAction): + bases = ['Bcfg2.Server.Plugins.Packages'] + module = True class OnDemandDict(MutableMapping): @@ -86,6 +95,37 @@ class Packages(Bcfg2.Server.Plugin.Plugin, .. private-include: _build_packages""" + options = [ + Bcfg2.Options.Option( + cf=("packages", "backends"), dest="packages_backends", + help="Packages backends to load", + type=Bcfg2.Options.Types.comma_list, + action=PackagesBackendAction, default=['Yum', 'Apt', 'Pac']), + Bcfg2.Options.PathOption( + cf=("packages", "cache"), dest="packages_cache", + help="Path to the Packages cache", + default='<repository>/Packages/cache'), + Bcfg2.Options.Option( + cf=("packages", "resolver"), dest="packages_resolver", + help="Disable the Packages resolver", + type=packages_boolean, default=True), + Bcfg2.Options.Option( + cf=("packages", "metadata"), dest="packages_metadata", + help="Disable all Packages metadata processing", + type=packages_boolean, default=True), + Bcfg2.Options.Option( + cf=("packages", "version"), dest="packages_version", + help="Set default Package entry version", default="auto", + choices=["auto", "any"]), + Bcfg2.Options.PathOption( + cf=("packages", "yum_config"), + help="The default path for generated yum configs", + default="/etc/yum.repos.d/bcfg2.repo"), + Bcfg2.Options.PathOption( + cf=("packages", "apt_config"), + help="The default path for generated apt configs", + default="/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list")] + #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. conflicts = ['Pkgmgr'] @@ -108,9 +148,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: Packages does a potentially tremendous amount of on-disk #: caching. ``cachepath`` holds the base directory to where #: data should be cached. - self.cachepath = \ - self.core.setup.cfp.get("packages", "cache", - default=os.path.join(self.data, 'cache')) + self.cachepath = Bcfg2.Options.setup.packages_cache #: Where Packages should store downloaded GPG key files self.keypath = os.path.join(self.cachepath, 'keys') @@ -186,40 +224,17 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :attr:`disableMetaData`) implies disabling the resolver. This property cannot be set. """ - if self.disableMetaData: - # disabling metadata without disabling the resolver Breaks - # Things - return True - try: - return not self.core.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled", which are not handled according to the - # Python docs but appear to be handled properly by - # ConfigParser in at least some versions - return self.core.setup.cfp.get( - "packages", - "resolver", - default="enabled").lower() == "disabled" + # disabling metadata without disabling the resolver Breaks + # Things + return not Bcfg2.Options.setup.packages_metadata or \ + not Bcfg2.Options.setup.packages_resolver @property def disableMetaData(self): """ Report whether or not metadata processing is enabled. This property cannot be set. """ - try: - return not self.core.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled" - return self.core.setup.cfp.get( - "packages", - "metadata", - default="enabled").lower() == "disabled" + return not Bcfg2.Options.setup.packages_metadata def create_config(self, entry, metadata): """ Create yum/apt config for the specified client. @@ -268,9 +283,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ if entry.tag == 'Package': collection = self.get_collection(metadata) - entry.set('version', self.core.setup.cfp.get("packages", - "version", - default="auto")) + entry.set('version', Bcfg2.Options.setup.packages_version) entry.set('type', collection.ptype) elif entry.tag == 'Path': self.create_config(entry, metadata) @@ -299,14 +312,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return True elif entry.tag == 'Path': # managed entries for yum/apt configs - if (entry.get("name") == - self.core.setup.cfp.get("packages", - "yum_config", - default=YUM_CONFIG_DEFAULT) or - entry.get("name") == - self.core.setup.cfp.get("packages", - "apt_config", - default=APT_CONFIG_DEFAULT)): + if entry.get("name") in [Bcfg2.Options.setup.apt_config, + Bcfg2.Options.setup.yum_config]: return True return False @@ -339,7 +346,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :returns: None """ collection = self.get_collection(metadata) - indep = lxml.etree.Element('Independent') + indep = lxml.etree.Element('Independent', name=self.__class__.__name__) self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index 293ec8e1a..c85bc7d41 100644 --- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -1,12 +1,9 @@ '''This module implements a package management scheme for all images''' -import os import re import sys -import glob import logging import lxml.etree -import Bcfg2.Server.Lint import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError @@ -295,44 +292,3 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): def HandleEntry(self, entry, metadata): self.BindEntry(entry, metadata) - - -class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): - """ Find duplicate :ref:`Pkgmgr - <server-plugins-generators-pkgmgr>` entries with the same - priority. """ - - def Run(self): - pset = set() - for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr', - '*.xml')): - if self.HandlesFile(pfile): - xdata = lxml.etree.parse(pfile).getroot() - # get priority, type, group - priority = xdata.get('priority') - ptype = xdata.get('type') - for pkg in xdata.xpath("//Package"): - if pkg.getparent().tag == 'Group': - grp = pkg.getparent().get('name') - if (type(grp) is not str and - grp.getparent().tag == 'Group'): - pgrp = grp.getparent().get('name') - else: - pgrp = 'none' - else: - grp = 'none' - pgrp = 'none' - ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) - # check if package is already listed with same - # priority, type, grp - if ptuple in pset: - self.LintError( - "duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) - else: - pset.add(ptuple) - - @classmethod - def Errors(cls): - return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 7b85f180d..0d264a5a6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -12,10 +12,19 @@ import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Server.Statistics import track_statistics -try: - from django.db import models - from django.core.exceptions import MultipleObjectsReturned - HAS_DJANGO = True +HAS_DJANGO = False +ProbesDataModel = None +ProbesGroupModel = None + + +def load_django_models(): + global ProbesDataModel, ProbesGroupModel, HAS_DJANGO + try: + from django.db import models + HAS_DJANGO = True + except ImportError: + HAS_DJANGO = False + return class ProbesDataModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): @@ -30,8 +39,7 @@ try: """ The database model for storing probe groups """ hostname = models.CharField(max_length=255) group = models.CharField(max_length=255) -except ImportError: - HAS_DJANGO = False + try: import json @@ -120,11 +128,15 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): bangline = re.compile(r'^#!\s*(?P<interpreter>.*)$') basename_is_regex = True - def __init__(self, path, encoding, plugin_name): + options = [ + Bcfg2.Options.BooleanOption( + cf=('probes', 'use_database'), dest="probes_db", + help="Use database capabilities of the Probes plugin")] + + def __init__(self, path, plugin_name): self.plugin_name = plugin_name Bcfg2.Server.Plugin.EntrySet.__init__(self, r'[0-9A-Za-z_\-]+', path, - Bcfg2.Server.Plugin.SpecificData, - encoding) + Bcfg2.Server.Plugin.SpecificData) Bcfg2.Server.FileMonitor.get_fam().AddMonitor(path, self) def HandleEvent(self, event): @@ -196,8 +208,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) try: - self.probes = ProbeSet(self.data, core.setup['encoding'], - self.name) + self.probes = ProbeSet(self.data, self.name) except: err = sys.exc_info()[1] raise Bcfg2.Server.Plugin.PluginInitError(err) @@ -260,7 +271,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, ProbesGroupsModel.objects.get_or_create( hostname=client.hostname, group=group) - except MultipleObjectsReturned: + except ProbesGroupsModel.MultipleObjectsReturned: ProbesGroupsModel.objects.filter(hostname=client.hostname, group=group).delete() ProbesGroupsModel.objects.get_or_create( diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 8e54da19b..bbc00556b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -7,7 +7,7 @@ import sys import copy import logging import lxml.etree -from Bcfg2.Options import get_option_parser +import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError @@ -40,18 +40,14 @@ class PropertyFile(object): .. automethod:: _write """ self.name = name - self.setup = get_option_parser() def write(self): """ Write the data in this data structure back to the property file. This public method performs checking to ensure that writing is possible and then calls :func:`_write`. """ - if not self.setup.cfp.getboolean("properties", "writes_enabled", - default=True): - msg = "Properties files write-back is disabled in the " + \ - "configuration" - LOGGER.error(msg) - raise PluginExecutionError(msg) + if not Bcfg2.Options.setup.writes_enabled: + raise PluginExecutionError("Properties files write-back is " + "disabled in the configuration") try: self.validate_data() except PluginExecutionError: @@ -199,7 +195,7 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): validate_data.__doc__ = PropertyFile.validate_data.__doc__ def get_additional_data(self, metadata): - if self.setup.cfp.getboolean("properties", "automatch", default=False): + if Bcfg2.Options.setup.automatch: default_automatch = "true" else: default_automatch = "false" @@ -221,6 +217,13 @@ class Properties(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): """ The properties plugin maps property files into client metadata instances. """ + options = [ + Bcfg2.Options.BooleanOption( + cf=("properties", "writes_enabled"), default=True, + help="Enable or disable Properties write-back"), + Bcfg2.Options.BooleanOption( + cf=("properties", "automatch"), + help="Enable Properties automatch")] #: Extensions that are understood by Properties. extensions = ["xml"] diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 3354763d4..9d1019441 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -5,11 +5,10 @@ import time import platform import traceback import lxml.etree -from Bcfg2.Reporting.Transport import load_transport_from_config, \ - TransportError -from Bcfg2.Options import REPORTING_COMMON_OPTIONS +import Bcfg2.Options +from Bcfg2.Reporting.Transport.base import TransportError from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \ - Debuggable, PluginInitError, PluginExecutionError + PluginInitError, PluginExecutionError # required for reporting try: @@ -33,9 +32,11 @@ def _rpc_call(method): # pylint: disable=W0223 -class Reporting(Statistics, Threaded, PullSource, Debuggable): +class Reporting(Statistics, Threaded, PullSource): """ Unified statistics and reporting plugin """ - __rmi__ = Debuggable.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry'] + __rmi__ = Statistics.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry'] + + options = [Bcfg2.Options.Common.reporting_transport] CLIENT_METADATA_FIELDS = ('profile', 'bundles', 'aliases', 'addresses', 'groups', 'categories', 'uuid', 'version') @@ -44,14 +45,10 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): Statistics.__init__(self, core, datastore) PullSource.__init__(self) Threaded.__init__(self) - Debuggable.__init__(self) self.whoami = platform.node() self.transport = None - core.setup.update(REPORTING_COMMON_OPTIONS) - core.setup.reparse() - if not HAS_SOUTH: msg = "Django south is required for Reporting" self.logger.error(msg) @@ -59,17 +56,15 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): def start_threads(self): try: - self.transport = load_transport_from_config(self.core.setup) + self.transport = Bcfg2.Options.setup.reporting_transport() except TransportError: - msg = "%s: Failed to load transport: %s" % \ - (self.name, traceback.format_exc().splitlines()[-1]) - self.logger.error(msg) - raise PluginInitError(msg) + raise PluginInitError("%s: Failed to instantiate transport: %s" % + (self.name, sys.exc_info()[1])) if self.debug_flag: self.transport.set_debug(self.debug_flag) def set_debug(self, debug): - rv = Debuggable.set_debug(self, debug) + rv = Statistics.set_debug(self, debug) if self.transport is not None: self.transport.set_debug(debug) return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index 3d4e8671d..541116db3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -1,6 +1,7 @@ """This generator provides rule-based entry mappings.""" import re +import Bcfg2.Options import Bcfg2.Server.Plugin @@ -8,6 +9,11 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles service assignments.""" __author__ = 'bcfg-dev@mcs.anl.gov' + options = Bcfg2.Server.Plugin.PrioDir.options + [ + Bcfg2.Options.BooleanOption( + cf=("rules", "regex"), dest="rules_regex", + help="Allow regular expressions in Rules")] + def __init__(self, core, datastore): Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) self._regex_cache = dict() @@ -42,4 +48,4 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): @property def _regex_enabled(self): """ Return True if rules regexes are enabled, False otherwise """ - return self.core.setup.cfp.getboolean("rules", "regex", default=False) + return Bcfg2.Options.setup.rules_regex diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index f350a7761..c858b881b 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -7,8 +7,9 @@ import socket import shutil import logging import tempfile -from itertools import chain +import Bcfg2.Options import Bcfg2.Server.Plugin +from itertools import chain from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Compat import any, u_str, b64encode # pylint: disable=W0622 @@ -20,8 +21,7 @@ class KeyData(Bcfg2.Server.Plugin.SpecificData): """ class to handle key data for HostKeyEntrySet """ def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) + Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific) self.encoding = encoding def __lt__(self, other): @@ -62,27 +62,30 @@ class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet): """ EntrySet to handle all kinds of host keys """ def __init__(self, basename, path): if basename.startswith("ssh_host_key"): - encoding = "base64" + self.encoding = "base64" else: - encoding = None - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData, - encoding) + self.encoding = None + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData) self.metadata = {'owner': 'root', 'group': 'root', 'type': 'file'} - if encoding is not None: - self.metadata['encoding'] = encoding + if self.encoding is not None: + self.metadata['encoding'] = self.encoding if basename.endswith('.pub'): self.metadata['mode'] = '0644' else: self.metadata['mode'] = '0600' + def get_keydata_object(self, filepath, specificity): + return KeyData(filepath, specificity, + self.encoding or Bcfg2.Options.setup.encoding) + class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): """ EntrySet to handle the ssh_known_hosts file """ def __init__(self, path): Bcfg2.Server.Plugin.EntrySet.__init__(self, "ssh_known_hosts", path, - KeyData, None) + KeyData) self.metadata = {'owner': 'root', 'group': 'root', 'type': 'file', diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index b21732666..74d8833f4 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -3,17 +3,13 @@ certificates and their keys. """ import os import sys -import logging import tempfile import lxml.etree -import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Utils import Executor from Bcfg2.Compat import ConfigParser from Bcfg2.Server.Plugin import PluginExecutionError -LOGGER = logging.getLogger(__name__) - class SSLCAXMLSpec(Bcfg2.Server.Plugin.StructFile): """ Base class to handle key.xml and cert.xml """ @@ -31,10 +27,9 @@ class SSLCAXMLSpec(Bcfg2.Server.Plugin.StructFile): metadata.hostname, self.name)) elif len(entries) > 1: - LOGGER.warning("More than one matching %s entry found for %s in " - "%s; using first match" % (self.tag, - metadata.hostname, - self.name)) + self.logger.warning( + "More than one matching %s entry found for %s in %s; " + "using first match" % (self.tag, metadata.hostname, self.name)) rv = dict() for attr, default in self.attrs.items(): val = entries[0].get(attr.lower(), default) @@ -84,9 +79,9 @@ class SSLCADataFile(Bcfg2.Server.Plugin.SpecificData): class SSLCAEntrySet(Bcfg2.Server.Plugin.EntrySet): """ Entry set to handle SSLCA entries and XML files """ - def __init__(self, _, path, entry_type, encoding, parent=None): + def __init__(self, _, path, entry_type, parent=None): Bcfg2.Server.Plugin.EntrySet.__init__(self, os.path.basename(path), - path, entry_type, encoding) + path, entry_type) self.parent = parent self.key = None self.cert = None @@ -361,10 +356,32 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): """ The SSLCA generator handles the creation and management of ssl certificates and their keys. """ __author__ = 'g.hagger@gmail.com' + + options = Bcfg2.Server.Plugin.GroupSpool.options + [ + Bcfg2.Options.WildcardSectionGroup( + Bcfg2.Options.PathOption( + cf=("sslca_*", "config"), + help="Path to the openssl config for the CA"), + Bcfg2.Options.Option( + cf=("sslca_*", "passphrase"), + help="Passphrase for the CA private key"), + Bcfg2.Options.PathOption( + cf=("sslca_*", "chaincert"), + help="Path to the SSL chaining certificate for verification"), + Bcfg2.Options.BooleanOption( + cf=("sslca_*", "root_ca"), + help="Whether or not <chaincert> is a root CA (as opposed to " + "an intermediate cert"))] + # python 2.5 doesn't support mixing *magic and keyword arguments es_cls = lambda self, *args: SSLCAEntrySet(*args, **dict(parent=self)) es_child_cls = SSLCADataFile def get_ca(self, name): """ get a dict describing a CA from the config file """ - return dict(self.core.setup.cfp.items("sslca_%s" % name)) + rv = dict() + prefix = "sslca_%s_" % name + for attr in dir(Bcfg2.Options.setup): + if attr.startswith(prefix): + rv[attr[len(prefix):]] = getattr(Bcfg2.Options.setup, attr) + return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index dfe864d48..679e38ff9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -4,8 +4,8 @@ additional XML-RPC methods for committing data to the repository and updating the repository. """ import sys +import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser try: import pysvn HAS_SVN = True @@ -16,6 +16,21 @@ except ImportError: class Svn(Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" + options = Bcfg2.Server.Plugin.Version.options + [ + Bcfg2.Options.Option( + cf=("svn", "conflict_resolution"), dest="svn_conflict_resolution", + type=lambda v: v.replace("-", "_"), + choices=dir(pysvn.wc_conflict_choice), + default=pysvn.wc_conflict_choice.postpone, + help="SVN conflict resolution method"), + Bcfg2.Options.Option( + cf=("svn", "user"), dest="svn_user", help="SVN username"), + Bcfg2.Options.Option( + cf=("svn", "password"), dest="svn_password", help="SVN password"), + Bcfg2.Options.BooleanOption( + cf=("svn", "always_trust"), dest="svn_trust_ssl", + help="Always trust SSL certs from SVN server")] + __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".svn" if HAS_SVN: @@ -36,62 +51,29 @@ class Svn(Bcfg2.Server.Plugin.Version): self.cmd = Executor() else: self.client = pysvn.Client() - # pylint: disable=E1101 - choice = pysvn.wc_conflict_choice.postpone - try: - resolution = self.core.setup.cfp.get( - "svn", - "conflict_resolution").replace('-', '_') - if resolution in ["edit", "launch", "working"]: - self.logger.warning("Svn: Conflict resolver %s requires " - "manual intervention, using %s" % - choice) - else: - choice = getattr(pysvn.wc_conflict_choice, resolution) - except AttributeError: - self.logger.warning("Svn: Conflict resolver %s does not " - "exist, using %s" % choice) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.info("Svn: No conflict resolution method " - "selected, using %s" % choice) - # pylint: enable=E1101 self.debug_log("Svn: Conflicts will be resolved with %s" % - choice) - self.client.callback_conflict_resolver = \ - self.get_conflict_resolver(choice) + Bcfg2.Options.setup.svn_conflict_resolution) + self.client.callback_conflict_resolver = self.conflict_resolver - try: - if self.core.setup.cfp.get( - "svn", - "always_trust").lower() == "true": - self.client.callback_ssl_server_trust_prompt = \ - self.ssl_server_trust_prompt - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.debug("Svn: Using subversion cache for SSL " - "certificate trust") + if Bcfg2.Options.setup.svn_trust_ssl: + self.client.callback_ssl_server_trust_prompt = \ + self.ssl_server_trust_prompt - try: - if (self.core.setup.cfp.get("svn", "user") and - self.core.setup.cfp.get("svn", "password")): - self.client.callback_get_login = \ - self.get_login - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.info("Svn: Using subversion cache for " - "password-based authetication") + if (Bcfg2.Options.setup.svn_user and + Bcfg2.Options.setup.svn_password): + self.client.callback_get_login = self.get_login self.logger.debug("Svn: Initialized svn plugin with SVN directory %s" % self.vcs_path) - # pylint: disable=W0613 - def get_login(self, realm, username, may_save): + def get_login(self, realm, username, may_save): # pylint: disable=W0613 """ PySvn callback to get credentials for HTTP basic authentication """ self.logger.debug("Svn: Logging in with username: %s" % - self.core.setup.cfp.get("svn", "user")) - return True, \ - self.core.setup.cfp.get("svn", "user"), \ - self.core.setup.cfp.get("svn", "password"), \ - False - # pylint: enable=W0613 + Bcfg2.Options.setup.svn_user) + return (True, + Bcfg2.Options.setup.svn_user, + Bcfg2.Options.setup.svn_password, + False) def ssl_server_trust_prompt(self, trust_dict): """ PySvn callback to always trust SSL certificates from SVN server """ @@ -102,22 +84,19 @@ class Svn(Bcfg2.Server.Plugin.Version): trust_dict['realm'])) return True, trust_dict['failures'], False - def get_conflict_resolver(self, choice): - """ Get a PySvn conflict resolution callback """ - def callback(conflict_description): - """ PySvn callback function to resolve conflicts """ - self.logger.info("Svn: Resolving conflict for %s with %s" % - (conflict_description['path'], choice)) - return choice, None, False - - return callback + def conflict_resolver(self, conflict_description): + """ PySvn callback function to resolve conflicts """ + self.logger.info("Svn: Resolving conflict for %s with %s" % + (conflict_description['path'], + Bcfg2.Options.setup.svn_conflict_resolution)) + return Bcfg2.Options.setup.svn_conflict_resolution, None, False def get_revision(self): """Read svn revision information for the Bcfg2 repository.""" msg = None if HAS_SVN: try: - info = self.client.info(self.vcs_root) + info = self.client.info(Bcfg2.Options.setup.vcs_root) self.revision = info.revision self.svn_root = info.url return str(self.revision.number) @@ -125,7 +104,7 @@ class Svn(Bcfg2.Server.Plugin.Version): msg = "Svn: Failed to get revision: %s" % sys.exc_info()[1] else: result = self.cmd.run(["env LC_ALL=C", "svn", "info", - self.vcs_root], + Bcfg2.Options.setup.vcs_root], shell=True) if result.success: self.revision = [line.split(': ')[1] @@ -141,7 +120,8 @@ class Svn(Bcfg2.Server.Plugin.Version): '''Svn.Update() => True|False\nUpdate svn working copy\n''' try: old_revision = self.revision.number - self.revision = self.client.update(self.vcs_root, recurse=True)[0] + self.revision = self.client.update(Bcfg2.Options.setup.vcs_root, + recurse=True)[0] except pysvn.ClientError: # pylint: disable=E1101 err = sys.exc_info()[1] # try to be smart about the error we got back @@ -163,7 +143,7 @@ class Svn(Bcfg2.Server.Plugin.Version): self.logger.debug("repository is current") else: self.logger.info("Updated %s from revision %s to %s" % - (self.vcs_root, old_revision, + (Bcfg2.Options.setup.vcs_root, old_revision, self.revision.number)) return True @@ -176,10 +156,11 @@ class Svn(Bcfg2.Server.Plugin.Version): return False try: - self.revision = self.client.checkin([self.vcs_root], + self.revision = self.client.checkin([Bcfg2.Options.setup.vcs_root], 'Svn: autocommit', recurse=True) - self.revision = self.client.update(self.vcs_root, recurse=True)[0] + self.revision = self.client.update(Bcfg2.Options.setup.vcs_root, + recurse=True)[0] self.logger.info("Svn: Commited changes. At %s" % self.revision.number) return True diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 77bdd6576..a32b7dea2 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -4,7 +4,6 @@ import re import imp import sys import logging -import Bcfg2.Server.Lint import Bcfg2.Server.Plugin LOGGER = logging.getLogger(__name__) @@ -93,75 +92,3 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, def get_additional_data(self, _): return dict([(h._module_name, h) # pylint: disable=W0212 for h in self.entries.values()]) - - -class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper - <server-plugins-connectors-templatehelper>` modules are valid. - This can check for: - - * A TemplateHelper module that cannot be imported due to syntax or - other compile-time errors; - * A TemplateHelper module that does not have an ``__export__`` - attribute, or whose ``__export__`` is not a list; - * Bogus symbols listed in ``__export__``, including symbols that - don't exist, that are reserved, or that start with underscores. - """ - - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.reserved_keywords = dir(HelperModule("foo.py")) - - def Run(self): - for helper in self.core.plugins['TemplateHelper'].entries.values(): - if self.HandlesFile(helper.name): - self.check_helper(helper.name) - - def check_helper(self, helper): - """ Check a single helper module. - - :param helper: The filename of the helper module - :type helper: string - """ - module_name = MODULE_RE.search(helper).group(1) - - try: - module = imp.load_source(safe_module_name(module_name), helper) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - self.LintError("templatehelper-import-error", - "Failed to import %s: %s" % - (helper, err)) - return - - if not hasattr(module, "__export__"): - self.LintError("templatehelper-no-export", - "%s has no __export__ list" % helper) - return - elif not isinstance(module.__export__, list): - self.LintError("templatehelper-nonlist-export", - "__export__ is not a list in %s" % helper) - return - - for sym in module.__export__: - if not hasattr(module, sym): - self.LintError("templatehelper-nonexistent-export", - "%s: exported symbol %s does not exist" % - (helper, sym)) - elif sym in self.reserved_keywords: - self.LintError("templatehelper-reserved-export", - "%s: exported symbol %s is reserved" % - (helper, sym)) - elif sym.startswith("_"): - self.LintError("templatehelper-underscore-export", - "%s: exported symbol %s starts with underscore" - % (helper, sym)) - - @classmethod - def Errors(cls): - return {"templatehelper-import-error": "error", - "templatehelper-no-export": "error", - "templatehelper-nonlist-export": "error", - "templatehelper-nonexistent-export": "error", - "templatehelper-reserved-export": "error", - "templatehelper-underscore-export": "warning"} diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 862fd98c1..646124fcc 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -15,6 +15,10 @@ from Bcfg2.Compat import xmlrpclib, SimpleXMLRPCServer, SocketServer, \ b64decode +class XMLRPCACLCheckException(Exception): + """ Raised when ACL checks fail on an RPC request """ + + class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): """ An XML-RPC dispatcher. """ @@ -33,6 +37,8 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): def _marshaled_dispatch(self, address, data): params, method = xmlrpclib.loads(data) + if not self.instance.check_acls(address, method): + raise XMLRPCACLCheckException try: if '.' not in method: params = (address, ) + params @@ -42,12 +48,12 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): response = (response.decode('utf-8'), ) else: response = (response, ) - raw_response = xmlrpclib.dumps(response, methodresponse=1, + raw_response = xmlrpclib.dumps(response, methodresponse=True, allow_none=self.allow_none, encoding=self.encoding) except xmlrpclib.Fault: fault = sys.exc_info()[1] - raw_response = xmlrpclib.dumps(fault, + raw_response = xmlrpclib.dumps(fault, methodresponse=True, allow_none=self.allow_none, encoding=self.encoding) except: @@ -56,7 +62,8 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): # report exception back to server raw_response = xmlrpclib.dumps( xmlrpclib.Fault(1, "%s:%s" % (err[0].__name__, err[1])), - allow_none=self.allow_none, encoding=self.encoding) + methodresponse=True, allow_none=self.allow_none, + encoding=self.encoding) return raw_response @@ -209,9 +216,8 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): password = "" cert = self.request.getpeercert() client_address = self.request.getpeername() - return (self.server.instance.authenticate(cert, username, - password, client_address) and - self.server.instance.check_acls(client_address[0], None)) + return self.server.instance.authenticate(cert, username, + password, client_address) def parse_request(self): """Extends parse_request. @@ -241,7 +247,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): try: select.select([self.rfile.fileno()], [], [], 3) except select.error: - print("got select timeout") + self.logger.error("Got select timeout") raise chunk_size = min(size_remaining, max_chunk_size) L.append(self.rfile.read(chunk_size).decode('utf-8')) @@ -251,7 +257,12 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): data) if sys.hexversion >= 0x03000000: response = response.encode('utf-8') + except XMLRPCACLCheckException: + self.send_error(401, self.responses[401][0]) + self.end_headers() except: # pylint: disable=W0702 + self.logger.error("Unexpected dispatch error for %s: %s" % + (self.client_address, sys.exc_info()[1])) try: self.send_response(500) self.end_headers() @@ -262,12 +273,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): raise else: # got a valid XML RPC response - # first, check ACLs client_address = self.request.getpeername() - method = xmlrpclib.loads(data)[1] - if not self.server.instance.check_acls(client_address, method): - self.send_error(401, self.responses[401][0]) - self.end_headers() try: self.send_response(200) self.send_header("Content-type", "text/xml") diff --git a/src/lib/Bcfg2/Server/Test.py b/src/lib/Bcfg2/Server/Test.py new file mode 100644 index 000000000..912a8f19c --- /dev/null +++ b/src/lib/Bcfg2/Server/Test.py @@ -0,0 +1,281 @@ +""" bcfg2-test libraries and CLI """ + +import os +import sys +import shlex +import signal +import fnmatch +import logging +import Bcfg2.Logger +import Bcfg2.Server.Core +from math import ceil +from nose.core import TestProgram +from nose.suite import LazySuite +from unittest import TestCase + +try: + from multiprocessing import Process, Queue, active_children + HAS_MULTIPROC = True +except ImportError: + HAS_MULTIPROC = False + active_children = lambda: [] # pylint: disable=C0103 + + +def get_sigint_handler(core): + """ Get a function that handles SIGINT/Ctrl-C by shutting down the + core and exiting properly.""" + + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + core.shutdown() + os._exit(1) # pylint: disable=W0212 + + return hdlr + + +class CapturingLogger(object): + """ Fake logger that captures logging output so that errors are + only displayed for clients that fail tests """ + def __init__(self, *args, **kwargs): # pylint: disable=W0613 + self.output = [] + + def error(self, msg): + """ discard error messages """ + self.output.append(msg) + + def warning(self, msg): + """ discard error messages """ + self.output.append(msg) + + def info(self, msg): + """ discard error messages """ + self.output.append(msg) + + def debug(self, msg): + """ discard error messages """ + self.output.append(msg) + + def reset_output(self): + """ Reset the captured output """ + self.output = [] + + +class ClientTestFromQueue(TestCase): + """ A test case that tests a value that has been enqueued by a + child test process. ``client`` is the name of the client that has + been tested; ``result`` is the result from the :class:`ClientTest` + test. ``None`` indicates a successful test; a string value + indicates a failed test; and an exception indicates an error while + running the test. """ + __test__ = False # Do not collect + + def __init__(self, client, result): + TestCase.__init__(self) + self.client = client + self.result = result + + def shortDescription(self): + return "Building configuration for %s" % self.client + + def runTest(self): + """ parse the result from this test """ + if isinstance(self.result, Exception): + raise self.result + assert self.result is None, self.result + + +class ClientTest(TestCase): + """ A test case representing the build of all of the configuration for + a single host. Checks that none of the build config entities has + had a failure when it is building. Optionally ignores some config + files that we know will cause errors (because they are private + files we don't have access to, for instance) """ + __test__ = False # Do not collect + divider = "-" * 70 + + def __init__(self, core, client, ignore=None): + TestCase.__init__(self) + self.core = core + self.core.logger = CapturingLogger() + self.client = client + if ignore is None: + self.ignore = dict() + else: + self.ignore = ignore + + def ignore_entry(self, tag, name): + """ return True if an error on a given entry should be ignored + """ + if tag in self.ignore: + if name in self.ignore[tag]: + return True + else: + # try wildcard matching + for pattern in self.ignore[tag]: + if fnmatch.fnmatch(name, pattern): + return True + return False + + def shortDescription(self): + return "Building configuration for %s" % self.client + + def runTest(self): + """ run this individual test """ + config = self.core.BuildConfiguration(self.client) + output = self.core.logger.output[:] + if output: + output.append(self.divider) + self.core.logger.reset_output() + + # check for empty client configuration + assert len(config.findall("Bundle")) > 0, \ + "\n".join(output + ["%s has no content" % self.client]) + + # check for missing bundles + metadata = self.core.build_metadata(self.client) + sbundles = [el.get('name') for el in config.findall("Bundle")] + missing = [b for b in metadata.bundles if b not in sbundles] + assert len(missing) == 0, \ + "\n".join(output + ["Configuration is missing bundle(s): %s" % + ':'.join(missing)]) + + # check for unknown packages + unknown_pkgs = [el.get("name") + for el in config.xpath('//Package[@type="unknown"]') + if not self.ignore_entry(el.tag, el.get("name"))] + assert len(unknown_pkgs) == 0, \ + "Configuration contains unknown packages: %s" % \ + ", ".join(unknown_pkgs) + + failures = [] + msg = output + ["Failures:"] + for failure in config.xpath('//*[@failure]'): + if not self.ignore_entry(failure.tag, failure.get('name')): + failures.append(failure) + msg.append("%s:%s: %s" % (failure.tag, failure.get("name"), + failure.get("failure"))) + + assert len(failures) == 0, "\n".join(msg) + + def __str__(self): + return "ClientTest(%s)" % self.client + + id = __str__ + + +class CLI(object): + options = [ + Bcfg2.Options.PositionalArgument( + "clients", help="Specific clients to build", nargs="*"), + Bcfg2.Options.Option( + "--nose-options", cf=("bcfg2_test", "nose_options"), + type=shlex.split, default=[], + help='Options to pass to nosetests. Only honored with ' + '--children 0'), + Bcfg2.Options.Option( + "--ignore", cf=('bcfg2_test', 'ignore_entries'), default=[], + dest="test_ignore", type=Bcfg2.Options.Types.comma_list, + help='Ignore these entries if they fail to build'), + Bcfg2.Options.Option( + "--children", cf=('bcfg2_test', 'children'), default=0, type=int, + help='Spawn this number of children for bcfg2-test (python 2.6+)')] + + def __init__(self): + parser = Bcfg2.Options.get_parser( + description="Verify that all clients build without failures", + components=[Bcfg2.Server.Core.Core, self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + if Bcfg2.Options.setup.children and not HAS_MULTIPROC: + self.logger.warning("Python multiprocessing library not found, " + "running with no children") + Bcfg2.Options.setup.children = 0 + + def get_core(self): + """ Get a server core, with events handled """ + core = Bcfg2.Server.Core.Core() + core.load_plugins() + core.block_for_fam_events(handle_events=True) + signal.signal(signal.SIGINT, get_sigint_handler(core)) + return core + + def get_ignore(self): + """ Get a dict of entry tags and names to + ignore errors from """ + ignore = dict() + for entry in Bcfg2.Options.setup.test_ignore: + tag, name = entry.split(":") + try: + ignore[tag].append(name) + except KeyError: + ignore[tag] = [name] + return ignore + + def run_child(self, clients, queue): + """ Run tests for the given clients in a child process, returning + results via the given Queue """ + core = self.get_core() + ignore = self.get_ignore() + for client in clients: + try: + ClientTest(core, client, ignore).runTest() + queue.put((client, None)) + except AssertionError: + queue.put((client, str(sys.exc_info()[1]))) + except: + queue.put((client, sys.exc_info()[1])) + + core.shutdown() + + def run(self): + core = self.get_core() + clients = Bcfg2.Options.setup.clients or core.metadata.clients + ignore = self.get_ignore() + + if Bcfg2.Options.setup.children: + if Bcfg2.Options.setup.children > len(clients): + self.logger.info("Refusing to spawn more children than " + "clients to test, setting children=%s" % + len(clients)) + Bcfg2.Options.setup.children = len(clients) + perchild = int(ceil(len(clients) / + float(Bcfg2.Options.setup.children + 1))) + queue = Queue() + for child in range(Bcfg2.Options.setup.children): + start = child * perchild + end = (child + 1) * perchild + child = Process(target=self.run_child, + args=(clients[start:end], queue)) + child.start() + + def generate_tests(): + """ Read test results for the clients """ + start = Bcfg2.Options.setup.children * perchild + for client in clients[start:]: + yield ClientTest(core, client, ignore) + + for i in range(start): # pylint: disable=W0612 + yield ClientTestFromQueue(*queue.get()) + else: + def generate_tests(): + """ Run tests for the clients """ + for client in clients: + yield ClientTest(core, client, ignore) + + result = TestProgram( + argv=sys.argv[:1] + Bcfg2.Options.setup.nose_options, + suite=LazySuite(generate_tests), exit=False) + + # block until all children have completed -- should be + # immediate since we've already gotten all the results we + # expect + for child in active_children(): + child.join() + + core.shutdown() + if result.success: + os._exit(0) # pylint: disable=W0212 + else: + os._exit(1) # pylint: disable=W0212 diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 370854881..51cc835dc 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -1,44 +1,64 @@ """ Django database models for all plugins """ import sys -import copy import logging import Bcfg2.Options import Bcfg2.Server.Plugins from Bcfg2.Compat import walk_packages -from django.db import models LOGGER = logging.getLogger('Bcfg2.Server.models') MODELS = [] -def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): +def _get_all_plugins(): + rv = [] + for submodule in walk_packages(path=Bcfg2.Server.Plugins.__path__, + prefix="Bcfg2.Server.Plugins."): + module = submodule[1].rsplit('.', 1)[-1] + if submodule[1] == "Bcfg2.Server.Plugins.%s" % module: + # we only include direct children of + # Bcfg2.Server.Plugins -- e.g., all_plugins should + # include Bcfg2.Server.Plugins.Cfg, but not + # Bcfg2.Server.Plugins.Cfg.CfgInfoXML + rv.append(module) + return rv + + +_ALL_PLUGINS = _get_all_plugins() + + +class _OptionContainer(object): + # we want to provide a different default plugin list -- + # namely, _all_ plugins, so that the database is guaranteed to + # work, even if /etc/bcfg2.conf isn't set up properly + options = [ + Bcfg2.Options.Option( + cf=('server', 'plugins'), type=Bcfg2.Options.Types.comma_list, + default=_ALL_PLUGINS, dest="models_plugins", + action=Bcfg2.Options.PluginsAction)] + + @staticmethod + def options_parsed_hook(): + # basic invocation to ensure that a default set of models is + # loaded, and thus that this module will always work. + load_models() + +Bcfg2.Options.get_parser().add_component(_OptionContainer) + + +def load_models(plugins=None): """ load models from plugins specified in the config """ + # this has to be imported after options are parsed, because Django + # finalizes its settings as soon as it's loaded, which means that + # if we import this before Bcfg2.settings has been populated, + # Django gets a null configuration, and subsequent updates to + # Bcfg2.settings won't help. + from django.db import models global MODELS - if plugins is None: - # we want to provide a different default plugin list -- - # namely, _all_ plugins, so that the database is guaranteed to - # work, even if /etc/bcfg2.conf isn't set up properly - plugin_opt = copy.deepcopy(Bcfg2.Options.SERVER_PLUGINS) - all_plugins = [] - for submodule in walk_packages(path=Bcfg2.Server.Plugins.__path__, - prefix="Bcfg2.Server.Plugins."): - module = submodule[1].rsplit('.', 1)[-1] - if submodule[1] == "Bcfg2.Server.Plugins.%s" % module: - # we only include direct children of - # Bcfg2.Server.Plugins -- e.g., all_plugins should - # include Bcfg2.Server.Plugins.Cfg, but not - # Bcfg2.Server.Plugins.Cfg.CfgInfoXML - all_plugins.append(module) - plugin_opt.default = all_plugins - - setup = Bcfg2.Options.get_option_parser() - setup.add_option("plugins", plugin_opt) - setup.add_option("configfile", Bcfg2.Options.CFILE) - setup.reparse(argv=[Bcfg2.Options.CFILE.cmd, cfile]) - plugins = setup['plugins'] + if not plugins: + plugins = Bcfg2.Options.setup.models_plugins if MODELS: # load_models() has been called once, so first unload all of @@ -49,45 +69,22 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): delattr(sys.modules[__name__], model) MODELS = [] - for plugin in plugins: - try: - mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % - plugin).Server.Plugins, plugin) - except ImportError: - try: - err = sys.exc_info()[1] - mod = __import__(plugin) - except: # pylint: disable=W0702 - if plugins != plugin_opt.default: - # only produce errors if the default plugin list - # was not used -- i.e., if the config file was set - # up. don't produce errors when trying to load - # all plugins, IOW. the error from the first - # attempt to import is probably more accurate than - # the second attempt. - LOGGER.error("Failed to load plugin %s: %s" % (plugin, - err)) - continue + for mod in plugins: for sym in dir(mod): obj = getattr(mod, sym) - if hasattr(obj, "__bases__") and models.Model in obj.__bases__: + if isinstance(obj, type) and issubclass(obj, models.Model): setattr(sys.modules[__name__], sym, obj) MODELS.append(sym) -# basic invocation to ensure that a default set of models is loaded, -# and thus that this module will always work. -load_models(quiet=True) - - -class InternalDatabaseVersion(models.Model): - """ Object that tell us to which version the database is """ - version = models.IntegerField() - updated = models.DateTimeField(auto_now_add=True) + class InternalDatabaseVersion(models.Model): + """ Object that tell us to which version the database is """ + version = models.IntegerField() + updated = models.DateTimeField(auto_now_add=True) - def __str__(self): - return "version %d updated the %s" % (self.version, + def __str__(self): + return "version %d updated %s" % (self.version, self.updated.isoformat()) - class Meta: # pylint: disable=C0111,W0232 - app_label = "reports" - get_latest_by = "version" + class Meta: # pylint: disable=C0111,W0232 + app_label = "reports" + get_latest_by = "version" diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 993fd9e0f..ccb79249e 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -2,11 +2,15 @@ used by both client and server. Stuff that doesn't fit anywhere else. """ +import os +import re +import sys import fcntl +import select import logging import subprocess import threading -from Bcfg2.Compat import any # pylint: disable=W0622 +from Bcfg2.Compat import input, any # pylint: disable=W0622 class ClassName(object): @@ -248,3 +252,64 @@ class Executor(object): finally: if timeout is not None: timer.cancel() + + +def list2range(lst): + ''' convert a list of integers to a set of human-readable ranges. e.g.: + + [1, 2, 3, 6, 9, 10, 11] -> "[1-3,6,9-11]" ''' + ilst = sorted(int(i) for i in lst) + ranges = [] + start = None + last = None + for i in ilst: + if not last or i != last + 1: + if start: + if start == last: + ranges.append(str(start)) + else: + ranges.append("%d-%d" % (start, last)) + start = i + last = i + if start: + if start == last: + ranges.append(str(start)) + else: + ranges.append("%d-%d" % (start, last)) + if not ranges: + return "" + elif len(ranges) > 1 or "-" in ranges[0]: + return "[%s]" % ",".join(ranges) + else: + # only one range consisting of only a single number + return ranges[0] + + +def hostnames2ranges(hostnames): + ''' convert a list of hostnames to a set of human-readable ranges. e.g.: + + ["foo1.example.com", "foo2.example.com", "foo3.example.com", + "foo6.example.com"] -> ["foo[1-3,6].example.com"]''' + hosts = {} + hostre = re.compile(r'(\w+?)(\d+)(\..*)$') + for host in hostnames: + match = hostre.match(host) + if match: + key = (match.group(1), match.group(3)) + try: + hosts[key].append(match.group(2)) + except KeyError: + hosts[key] = [match.group(2)] + + ranges = [] + for name, nums in hosts.items(): + ranges.append(name[0] + list2range(nums) + name[1]) + return ranges + + +def safe_input(msg): + """ input() that flushes the input buffer before accepting input """ + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) + return input(msg) diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 13512ff58..42d415232 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -1,7 +1,6 @@ """ Django settings for the Bcfg2 server """ import os -import sys import Bcfg2.Options try: @@ -17,91 +16,17 @@ try: except ImportError: HAS_SOUTH = False -DATABASES = dict() +DATABASES = dict(default=dict()) TIME_ZONE = None -DEBUG = False -TEMPLATE_DEBUG = DEBUG +TEMPLATE_DEBUG = DEBUG = False ALLOWED_HOSTS = ['*'] MEDIA_URL = '/site_media/' - -def _default_config(): - """ get the default config file. returns /etc/bcfg2-web.conf, - UNLESS /etc/bcfg2.conf exists AND /etc/bcfg2-web.conf does not - exist. """ - setup = Bcfg2.Options.get_option_parser() - setup.add_option("configfile", Bcfg2.Options.CFILE) - setup.add_option("web_configfile", Bcfg2.Options.WEB_CFILE) - setup.reparse(argv=sys.argv[1:], do_getopt=False) - if (not os.path.exists(setup['web_configfile']) and - os.path.exists(setup['configfile'])): - return setup['configfile'] - else: - return setup['web_configfile'] - -DEFAULT_CONFIG = _default_config() - - -def read_config(cfile=DEFAULT_CONFIG, repo=None): - """ read the config file and set django settings based on it """ - # pylint: disable=W0602,W0603 - global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ - DATABASE_HOST, DATABASE_PORT, DATABASE_OPTIONS, DATABASE_SCHEMA, \ - DEBUG, TEMPLATE_DEBUG, TIME_ZONE, MEDIA_URL - # pylint: enable=W0602,W0603 - - if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): - print("%s does not exist, using %s for database configuration" % - (cfile, DEFAULT_CONFIG)) - cfile = DEFAULT_CONFIG - - # when setting a different config file, it has to be set in either - # sys.argv or in the OptionSet() constructor AS WELL AS the argv - # that's passed to setup.parse() - argv = [Bcfg2.Options.CFILE.cmd, cfile, - Bcfg2.Options.WEB_CFILE.cmd, cfile] - setup = Bcfg2.Options.get_option_parser() - setup.add_options(Bcfg2.Options.DATABASE_COMMON_OPTIONS) - setup.add_option("repo", Bcfg2.Options.SERVER_REPOSITORY) - setup.reparse(argv=argv) - - if repo is None: - repo = setup['repo'] - - DATABASES['default'] = \ - dict(ENGINE="django.db.backends.%s" % setup['db_engine'], - NAME=setup['db_name'], - USER=setup['db_user'], - PASSWORD=setup['db_password'], - HOST=setup['db_host'], - PORT=setup['db_port'], - OPTIONS=setup['db_options'], - SCHEMA=setup['db_schema']) - - # dropping the version check. This was added in 1.1.2 - TIME_ZONE = setup['time_zone'] - - DEBUG = setup['django_debug'] - TEMPLATE_DEBUG = DEBUG - if DEBUG: - print("Warning: Setting web_debug to True causes extraordinary memory " - "leaks. Only use this setting if you know what you're doing.") - - if setup['web_prefix']: - MEDIA_URL = setup['web_prefix'].rstrip('/') + MEDIA_URL - else: - MEDIA_URL = '/site_media/' - -# initialize settings from /etc/bcfg2-web.conf or /etc/bcfg2.conf, or -# set up basic defaults. this lets manage.py work in all cases -read_config() - -ADMINS = (('Root', 'root')) -MANAGERS = ADMINS +MANAGERS = ADMINS = (('Root', 'root')) # Language code for this installation. All choices can be found here: # http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes @@ -183,3 +108,77 @@ TEMPLATE_CONTEXT_PROCESSORS = ( 'django.core.context_processors.media', 'django.core.context_processors.request' ) + + +def read_config(): + """ read the config file and set django settings based on it """ + global DEBUG, TEMPLATE_DEBUG, TIME_ZONE, MEDIA_URL # pylint: disable=W0603 + + DATABASES['default'] = \ + dict(ENGINE="django.db.backends.%s" % Bcfg2.Options.setup.db_engine, + NAME=Bcfg2.Options.setup.db_name, + USER=Bcfg2.Options.setup.db_user, + PASSWORD=Bcfg2.Options.setup.db_password, + HOST=Bcfg2.Options.setup.db_host, + PORT=Bcfg2.Options.setup.db_port, + OPTIONS=Bcfg2.Options.setup.db_opts, + SCHEMA=Bcfg2.Options.setup.db_schema) + + TIME_ZONE = Bcfg2.Options.setup.timezone + + TEMPLATE_DEBUG = DEBUG = Bcfg2.Options.setup.web_debug + if DEBUG: + print("Warning: Setting web_debug to True causes extraordinary memory " + "leaks. Only use this setting if you know what you're doing.") + + if Bcfg2.Options.setup.web_prefix: + MEDIA_URL = Bcfg2.Options.setup.web_prefix.rstrip('/') + MEDIA_URL + + +class _OptionContainer(object): + """ Container for options loaded at import-time to configure + databases """ + options = [ + Bcfg2.Options.Common.repository, + Bcfg2.Options.PathOption( + '-W', '--web-config', cf=('reporting', 'config'), + default="/etc/bcfg2-web.conf", + action=Bcfg2.Options.ConfigFileAction, + help='Web interface configuration file'), + Bcfg2.Options.Option( + cf=('database', 'engine'), default='sqlite3', + help='Database engine', dest='db_engine'), + Bcfg2.Options.Option( + cf=('database', 'name'), default='<repository>/etc/bcfg2.sqlite', + help="Database name", dest="db_name"), + Bcfg2.Options.Option( + cf=('database', 'user'), help='Database username', dest='db_user'), + Bcfg2.Options.Option( + cf=('database', 'password'), help='Database password', + dest='db_password'), + Bcfg2.Options.Option( + cf=('database', 'host'), help='Database host', dest='db_host'), + Bcfg2.Options.Option( + cf=('database', 'port'), help='Database port', dest='db_port'), + Bcfg2.Options.Option( + cf=('database', 'schema'), help='Database schema', + dest='db_schema'), + Bcfg2.Options.Option( + cf=('database', 'options'), help='Database options', + dest='db_opts', type=Bcfg2.Options.Types.comma_dict), + Bcfg2.Options.Option( + cf=('reporting', 'timezone'), help='Django timezone'), + Bcfg2.Options.BooleanOption( + cf=('reporting', 'web_debug'), help='Django debug'), + Bcfg2.Options.Option( + cf=('reporting', 'web_prefix'), help='Web prefix')] + + @staticmethod + def options_parsed_hook(): + """ initialize settings from /etc/bcfg2-web.conf or + /etc/bcfg2.conf, or set up basic defaults. this lets + manage.py work in all cases """ + read_config() + + +Bcfg2.Options.get_parser().add_component(_OptionContainer) diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 62f749b80..eca7c3395 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -2,27 +2,9 @@ """Bcfg2 Client""" import sys -import signal -from Bcfg2.Client.Client import Client -from Bcfg2.Options import load_option_parser, CLIENT_COMMON_OPTIONS - - -def cb_sigint_handler(signum, frame): - """ Exit upon CTRL-C. """ - raise SystemExit(1) - - -def main(): - setup = load_option_parser(CLIENT_COMMON_OPTIONS) - setup.parse(sys.argv[1:]) - - if setup['args']: - print("Bcfg2 takes no arguments, only options") - print(setup.buildHelpMessage()) - raise SystemExit(1) - - signal.signal(signal.SIGINT, cb_sigint_handler) - return Client().run() +from Bcfg2.Options import get_parser +from Bcfg2.Client import Client if __name__ == '__main__': - sys.exit(main()) + get_parser("Bcfg2 client", components=[Client]).parse() + sys.exit(Client().run()) diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 0e1e34c60..d57cd8b35 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -2,97 +2,11 @@ """ bcfg2-admin is a script that helps to administer a Bcfg2 deployment. """ -import re import sys -import logging -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Admin -from Bcfg2.Compat import StringIO - - -def mode_import(modename): - """Load Bcfg2.Server.Admin.<mode>.""" - modname = modename.capitalize() - mod = getattr(__import__("Bcfg2.Server.Admin.%s" % - (modname)).Server.Admin, modname) - return getattr(mod, modname) - - -def get_modes(): - """Get all available modes, except for the base mode.""" - return [x.lower() for x in Bcfg2.Server.Admin.__all__ if x != 'mode'] - - -def create_description(): - """Create the description string from the list of modes.""" - modes = get_modes() - description = StringIO() - description.write("Available modes are:\n\n") - for mode in modes: - try: - doc = re.sub(r'\s{2,}', ' ', mode_import(mode).__doc__.strip()) - except (ImportError, SystemExit): - continue - description.write((" %-15s %s\n" % (mode, doc))) - return description.getvalue() - - -def main(): - optinfo = dict() - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.load_option_parser(optinfo) - # override default help message to include description of all modes - setup.hm = "Usage:\n\n%s\n%s" % (setup.buildHelpMessage(), - create_description()) - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-admin', to_syslog=setup['syslog'], - level=level) - - log = logging.getLogger('bcfg2-admin') - - # Provide help if requested or no args were specified - if (not setup['args'] or len(setup['args']) < 1 or - setup['args'][0] == 'help' or setup['help']): - if len(setup['args']) > 1: - # Get help for a specific mode by passing it the help argument - setup['args'] = [setup['args'][1], setup['args'][0]] - else: - # Print short help for all modes - print(setup.hm) - raise SystemExit(0) - - if setup['args'][0] in get_modes(): - modname = setup['args'][0].capitalize() - if len(setup['args']) > 1 and setup['args'][1] == 'help': - mode_cls = mode_import(modname) - mode_cls.usage(rv=0) - try: - mode_cls = mode_import(modname) - except ImportError: - err = sys.exc_info()[1] - log.error("Failed to load admin mode %s: %s" % (modname, err)) - raise SystemExit(1) - mode = mode_cls() - try: - return mode(setup['args'][1:]) - finally: - mode.shutdown() - else: - log.error("Error: Unknown mode '%s'\n" % setup['args'][0]) - print(create_description()) - raise SystemExit(1) +from Bcfg2.Server.Admin import CLI if __name__ == '__main__': try: - sys.exit(main()) + sys.exit(CLI().run()) except KeyboardInterrupt: raise SystemExit(1) diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index 9190f1390..26d5eedf1 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -1,445 +1,8 @@ #!/usr/bin/env python """ helper for encrypting/decrypting Cfg and Properties files """ -import os import sys -import copy -import select -import logging -import lxml.etree -import Bcfg2.Logger -import Bcfg2.Options -from Bcfg2.Server import XMLParser -from Bcfg2.Compat import input # pylint: disable=W0622 -try: - import Bcfg2.Server.Encryption -except ImportError: - print("Could not import %s. Is M2Crypto installed?" % sys.exc_info()[1]) - raise SystemExit(1) - - -class PassphraseError(Exception): - """ Exception raised when there's a problem determining the - passphrase to encrypt or decrypt with """ - - -class CryptoTool(object): - """ Generic decryption/encryption interface base object """ - def __init__(self, filename, setup): - self.setup = setup - self.logger = logging.getLogger(self.__class__.__name__) - self.passphrases = Bcfg2.Server.Encryption.get_passphrases(self.setup) - - self.filename = filename - try: - self.data = open(self.filename).read() - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error reading %s, skipping: %s" % (filename, - err)) - return False - - self.pname, self.passphrase = self._get_passphrase() - - def _get_passphrase(self): - """ get the passphrase for the current file """ - if (not self.setup.cfp.has_section( - Bcfg2.Server.Encryption.CFG_SECTION) or - len(Bcfg2.Server.Encryption.get_passphrases(self.setup)) == 0): - raise PassphraseError("No passphrases available in %s" % - self.setup['configfile']) - - pname = None - if self.setup['passphrase']: - pname = self.setup['passphrase'] - - if pname: - if self.setup.cfp.has_option(Bcfg2.Server.Encryption.CFG_SECTION, - pname): - passphrase = self.setup.cfp.get( - Bcfg2.Server.Encryption.CFG_SECTION, pname) - self.logger.debug("Using passphrase %s specified on command " - "line" % pname) - return (pname, passphrase) - else: - raise PassphraseError("Could not find passphrase %s in %s" % - (pname, self.setup['configfile'])) - else: - pnames = Bcfg2.Server.Encryption.get_passphrases() - if len(pnames) == 1: - pname = pnames.keys()[0] - passphrase = pnames[pname] - self.logger.info("Using passphrase %s" % pname) - return (pname, passphrase) - elif len(pnames) > 1: - return (None, None) - raise PassphraseError("No passphrase could be determined") - - def get_destination_filename(self, original_filename): - """ Get the filename where data should be written """ - return original_filename - - def write(self, data): - """ write data to disk """ - new_fname = self.get_destination_filename(self.filename) - try: - self._write(new_fname, data) - self.logger.info("Wrote data to %s" % new_fname) - return True - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error writing data from %s to %s: %s" % - (self.filename, new_fname, err)) - return False - - def _write(self, filename, data): - """ Perform the actual write of data. This is separate from - :func:`CryptoTool.write` so it can be easily - overridden. """ - open(filename, "wb").write(data) - - -class Decryptor(CryptoTool): - """ Decryptor interface """ - def decrypt(self): - """ decrypt the file, returning the encrypted data """ - raise NotImplementedError - - -class Encryptor(CryptoTool): - """ encryptor interface """ - def encrypt(self): - """ encrypt the file, returning the encrypted data """ - raise NotImplementedError - - -class CfgEncryptor(Encryptor): - """ encryptor class for Cfg files """ - - def __init__(self, filename, setup): - Encryptor.__init__(self, filename, setup) - if self.passphrase is None: - raise PassphraseError("Multiple passphrases found in %s, " - "specify one on the command line with -p" % - self.setup['configfile']) - - def encrypt(self): - return Bcfg2.Server.Encryption.ssl_encrypt( - self.data, self.passphrase, - Bcfg2.Server.Encryption.get_algorithm(self.setup)) - - def get_destination_filename(self, original_filename): - return original_filename + ".crypt" - - -class CfgDecryptor(Decryptor): - """ Decrypt Cfg files """ - - def decrypt(self): - """ decrypt the given file, returning the plaintext data """ - if self.passphrase: - try: - return Bcfg2.Server.Encryption.ssl_decrypt( - self.data, self.passphrase, - Bcfg2.Server.Encryption.get_algorithm(self.setup)) - except Bcfg2.Server.Encryption.EVPError: - self.logger.info("Could not decrypt %s with the " - "specified passphrase" % self.filename) - return False - except: - err = sys.exc_info()[1] - self.logger.error("Error decrypting %s: %s" % - (self.filename, err)) - return False - else: # no passphrase given, brute force - try: - return Bcfg2.Server.Encryption.bruteforce_decrypt( - self.data, passphrases=self.passphrases.values(), - algorithm=Bcfg2.Server.Encryption.get_algorithm( - self.setup)) - except Bcfg2.Server.Encryption.EVPError: - self.logger.info("Could not decrypt %s with any passphrase" % - self.filename) - return False - - def get_destination_filename(self, original_filename): - if original_filename.endswith(".crypt"): - return original_filename[:-6] - else: - return Decryptor.get_plaintext_filename(self, original_filename) - - -class PropertiesCryptoMixin(object): - """ Mixin to provide some common methods for Properties crypto """ - default_xpath = '//*' - - def _get_elements(self, xdata): - """ Get the list of elements to encrypt or decrypt """ - if self.setup['xpath']: - elements = xdata.xpath(self.setup['xpath']) - if not elements: - self.logger.warning("XPath expression %s matched no " - "elements" % self.setup['xpath']) - else: - elements = xdata.xpath(self.default_xpath) - if not elements: - elements = list(xdata.getiterator(tag=lxml.etree.Element)) - - # filter out elements without text data - for el in elements[:]: - if not el.text: - elements.remove(el) - - if self.setup['interactive']: - for element in elements[:]: - if len(element): - elt = copy.copy(element) - for child in elt.iterchildren(): - elt.remove(child) - else: - elt = element - print(lxml.etree.tostring( - elt, - xml_declaration=False).decode("UTF-8").strip()) - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], - 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - ans = input("Encrypt this element? [y/N] ") - if not ans.lower().startswith("y"): - elements.remove(element) - return elements - - def _get_element_passphrase(self, element): - """ Get the passphrase to use to encrypt or decrypt a given - element """ - pname = element.get("encrypted") - if pname in self.passphrases: - passphrase = self.passphrases[pname] - elif self.passphrase: - if pname: - self.logger.warning("Passphrase %s not found in %s, " - "using passphrase given on command line" - % (pname, self.setup['configfile'])) - passphrase = self.passphrase - pname = self.pname - else: - raise PassphraseError("Multiple passphrases found in %s, " - "specify one on the command line with -p" % - self.setup['configfile']) - return (pname, passphrase) - - def _write(self, filename, data): - """ Write the data """ - data.getroottree().write(filename, - xml_declaration=False, - pretty_print=True) - - -class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): - """ encryptor class for Properties files """ - - def encrypt(self): - xdata = lxml.etree.XML(self.data, parser=XMLParser) - for elt in self._get_elements(xdata): - try: - pname, passphrase = self._get_element_passphrase(elt) - except PassphraseError: - self.logger.error(str(sys.exc_info()[1])) - return False - elt.text = Bcfg2.Server.Encryption.ssl_encrypt( - elt.text, passphrase, - Bcfg2.Server.Encryption.get_algorithm(self.setup)).strip() - elt.set("encrypted", pname) - return xdata - - def _write(self, filename, data): - PropertiesCryptoMixin._write(self, filename, data) - - -class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): - """ decryptor class for Properties files """ - default_xpath = '//*[@encrypted]' - - def decrypt(self): - xdata = lxml.etree.XML(self.data, parser=XMLParser) - for elt in self._get_elements(xdata): - try: - pname, passphrase = self._get_element_passphrase(elt) - except PassphraseError: - self.logger.error(str(sys.exc_info()[1])) - return False - decrypted = Bcfg2.Server.Encryption.ssl_decrypt( - elt.text, passphrase, - Bcfg2.Server.Encryption.get_algorithm(self.setup)).strip() - try: - elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') - elt.set("encrypted", pname) - except UnicodeDecodeError: - # we managed to decrypt the value, but it contains - # content that can't even be encoded into xml - # entities. what probably happened here is that we - # coincidentally could decrypt a value encrypted with - # a different key, and wound up with gibberish. - self.logger.warning("Decrypted %s to gibberish, skipping" % - elt.tag) - return xdata - - def _write(self, filename, data): - PropertiesCryptoMixin._write(self, filename, data) - - -def main(): # pylint: disable=R0912,R0915 - optinfo = dict(interactive=Bcfg2.Options.INTERACTIVE) - optinfo.update(Bcfg2.Options.CRYPT_OPTIONS) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - setup = Bcfg2.Options.load_option_parser(optinfo) - setup.hm = " bcfg2-crypt [options] <filename>\nOptions:\n%s" % \ - setup.buildHelpMessage() - setup.parse() - - if not setup['args']: - print(setup.hm) - raise SystemExit(1) - - log_args = dict(to_syslog=setup['syslog'], to_console=logging.WARNING) - if setup['verbose']: - log_args['to_console'] = logging.DEBUG - Bcfg2.Logger.setup_logging('bcfg2-crypt', **log_args) - logger = logging.getLogger('bcfg2-crypt') - - if setup['decrypt']: - if setup['encrypt']: - logger.error("You cannot specify both --encrypt and --decrypt") - raise SystemExit(1) - elif setup['remove']: - logger.error("--remove cannot be used with --decrypt, ignoring") - setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default - elif setup['interactive']: - logger.error("Cannot decrypt interactively") - setup['interactive'] = False - - if setup['cfg']: - if setup['properties']: - logger.error("You cannot specify both --cfg and --properties") - raise SystemExit(1) - if setup['xpath']: - logger.error("Specifying --xpath with --cfg is nonsensical, " - "ignoring --xpath") - setup['xpath'] = Bcfg2.Options.CRYPT_XPATH.default - if setup['interactive']: - logger.error("You cannot use interactive mode with --cfg, " - "ignoring -I") - setup['interactive'] = False - elif setup['properties']: - if setup['remove']: - logger.error("--remove cannot be used with --properties, ignoring") - setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default - - for fname in setup['args']: - if not os.path.exists(fname): - logger.error("%s does not exist, skipping" % fname) - continue - - # figure out if we need to encrypt this as a Properties file - # or as a Cfg file - props = False - if setup['properties']: - props = True - elif setup['cfg']: - props = False - elif fname.endswith(".xml"): - try: - xroot = lxml.etree.parse(fname).getroot() - if xroot.tag == "Properties": - props = True - else: - props = False - except IOError: - err = sys.exc_info()[1] - logger.error("Error reading %s, skipping: %s" % (fname, err)) - continue - except lxml.etree.XMLSyntaxError: - props = False - else: - props = False - - if props: - if setup['remove']: - logger.info("Cannot use --remove with Properties file %s, " - "ignoring for this file" % fname) - tools = (PropertiesEncryptor, PropertiesDecryptor) - else: - if setup['xpath']: - logger.info("Cannot use xpath with Cfg file %s, ignoring " - "xpath for this file" % fname) - if setup['interactive']: - logger.info("Cannot use interactive mode with Cfg file %s, " - "ignoring -I for this file" % fname) - tools = (CfgEncryptor, CfgDecryptor) - - data = None - mode = None - if setup['encrypt']: - try: - tool = tools[0](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - mode = "encrypt" - elif setup['decrypt']: - try: - tool = tools[1](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - mode = "decrypt" - else: - logger.info("Neither --encrypt nor --decrypt specified, " - "determining mode") - try: - tool = tools[1](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - - try: - data = tool.decrypt() - mode = "decrypt" - except: # pylint: disable=W0702 - pass - if data is False: - data = None - logger.info("Failed to decrypt %s, trying encryption" % fname) - try: - tool = tools[0](fname, setup) - except PassphraseError: - logger.error(str(sys.exc_info()[1])) - return 2 - mode = "encrypt" - - if data is None: - data = getattr(tool, mode)() - if not data: - logger.error("Failed to %s %s, skipping" % (mode, fname)) - continue - if setup['crypt_stdout']: - if len(setup['args']) > 1: - print("----- %s -----" % fname) - print(data) - if len(setup['args']) > 1: - print("") - else: - tool.write(data) - - if (setup['remove'] and - tool.get_destination_filename(fname) != fname): - try: - os.unlink(fname) - except IOError: - err = sys.exc_info()[1] - logger.error("Error removing %s: %s" % (fname, err)) - continue +from Bcfg2.Server.Encryption import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 9e3a671da..adfa96852 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -1,801 +1,8 @@ #!/usr/bin/env python """This tool loads the Bcfg2 core into an interactive debugger.""" -import os -import re import sys -import cmd -import getopt -import fnmatch -import logging -import lxml.etree -import traceback -from code import InteractiveConsole -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Core -import Bcfg2.Server.Plugin -import Bcfg2.Client.Tools.POSIX -from Bcfg2.Compat import unicode # pylint: disable=W0622 - -try: - try: - import cProfile as profile - except ImportError: - import profile - import pstats - HAS_PROFILE = True -except ImportError: - HAS_PROFILE = False - - -class MockLog(object): - """ Fake logger that just discards all messages in order to mask - errors from builddir being unable to chown files it creates """ - def error(self, *args, **kwargs): - """ discard error messages """ - pass - - def warning(self, *args, **kwargs): - """ discard warning messages """ - pass - - def info(self, *args, **kwargs): - """ discard info messages """ - pass - - def debug(self, *args, **kwargs): - """ discard debug messages """ - pass - - -class FileNotBuilt(Exception): - """Thrown when File entry contains no content.""" - def __init__(self, value): - Exception.__init__(self) - self.value = value - - def __str__(self): - return repr(self.value) - - -def print_tabular(rows): - """Print data in tabular format.""" - cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 - for index in range(len(rows[0]))]) - fstring = (" %%-%ss |" * len(cmax)) % cmax - fstring = ('|'.join([" %%-%ss "] * len(cmax))) % cmax - print(fstring % rows[0]) - print((sum(cmax) + (len(cmax) * 2) + (len(cmax) - 1)) * '=') - for row in rows[1:]: - print(fstring % row) - - -def display_trace(trace): - """ display statistics from a profile trace """ - stats = pstats.Stats(trace) - stats.sort_stats('cumulative', 'calls', 'time') - stats.print_stats(200) - - -def load_interpreters(): - """ Load a dict of available Python interpreters """ - interpreters = dict(python=lambda v: InteractiveConsole(v).interact()) - best = "python" - try: - import bpython.cli - interpreters["bpython"] = lambda v: bpython.cli.main(args=[], - locals_=v) - best = "bpython" - except ImportError: - pass - - try: - # whether ipython is actually better than bpython is - # up for debate, but this is the behavior that existed - # before --interpreter was added, so we call IPython - # better - import IPython - # pylint: disable=E1101 - if hasattr(IPython, "Shell"): - interpreters["ipython"] = lambda v: \ - IPython.Shell.IPShell(argv=[], user_ns=v).mainloop() - best = "ipython" - elif hasattr(IPython, "embed"): - interpreters["ipython"] = lambda v: IPython.embed(user_ns=v) - best = "ipython" - else: - print("Unknown IPython API version") - # pylint: enable=E1101 - except ImportError: - pass - - interpreters['best'] = interpreters[best] - return interpreters - - -class InfoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): - """Main class for bcfg2-info.""" - doc_header = "bcfg2-info commands (type help <command>):" - prompt = 'bcfg2-info> ' - - def __init__(self): - cmd.Cmd.__init__(self) - self.setup = Bcfg2.Options.get_option_parser() - self.setup.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - self.setup.update(dict(interpreter=Bcfg2.Options.INTERPRETER)) - Bcfg2.Server.Core.BaseCore.__init__(self) - - def _get_client_list(self, hostglobs): - """ given a host glob, get a list of clients that match it """ - # special cases to speed things up: - if '*' in hostglobs: - return self.metadata.clients - has_wildcards = False - for glob in hostglobs: - # check if any wildcard characters are in the string - if set('*?[]') & set(glob): - has_wildcards = True - break - if not has_wildcards: - return hostglobs - - rv = set() - clist = set(self.metadata.clients) - for glob in hostglobs: - for client in clist: - if fnmatch.fnmatch(client, glob): - rv.update(client) - clist.difference_update(rv) - return list(rv) - - def do_debug(self, args): - """debug [-n] [-f <command list>] - Shell out to native python interpreter""" - try: - opts, _ = getopt.getopt(args.split(), 'nf:') - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self.do_debug.__doc__) - return - scriptmode = False - interactive = True - for opt in opts: - if opt[0] == '-f': - scriptmode = True - spath = opt[1] - elif opt[0] == '-n': - interactive = False - if scriptmode: - console = InteractiveConsole(locals()) - for command in [c.strip() for c in open(spath).readlines()]: - if command: - console.push(command) - if interactive: - interpreters = load_interpreters() - if self.setup['interpreter'] in interpreters: - print("Dropping to %s interpreter; press ^D to resume" % - self.setup['interpreter']) - interpreters[self.setup['interpreter']](locals()) - else: - self.logger.error("Invalid interpreter %s" % - self.setup['interpreter']) - self.logger.error("Valid interpreters are: %s" % - ", ".join(interpreters.keys())) - - def do_quit(self, _): - """quit|exit - Exit program""" - print("") # put user's prompt on a new line - self.shutdown() - os._exit(0) # pylint: disable=W0212 - - do_EOF = do_quit - do_exit = do_quit - - def do_update(self, _): - """update - Process pending filesystem events""" - self.fam.handle_events_in_interval(0.1) - - def do_build(self, args): - """build [-f] <hostname> <filename> - Build config for hostname, writing to filename""" - alist = args.split() - path_force = False - for arg in alist: - if arg == '-f': - alist.remove('-f') - path_force = True - if len(alist) == 2: - client, ofile = alist - if not ofile.startswith('/tmp') and not path_force: - print("Refusing to write files outside of /tmp without -f " - "option") - return - try: - lxml.etree.ElementTree(self.BuildConfiguration(client)).write( - ofile, - encoding='UTF-8', xml_declaration=True, - pretty_print=True) - except IOError: - err = sys.exc_info()[1] - print("Failed to write File %s: %s" % (ofile, err)) - else: - print(self.do_build.__doc__) - - def help_builddir(self): - """Display help for builddir command.""" - print("""Usage: builddir [-f] <hostname> <output dir> - -Generates a config for client <hostname> and writes the -individual configuration files out separately in a tree -under <output dir>. The <output dir> directory must be -rooted under /tmp unless the -f argument is provided, in -which case it can be located anywhere. - -NOTE: Currently only handles file entries and writes -all content with the default owner and permissions. These -could be much more permissive than would be created by the -Bcfg2 client itself.""") - - def do_builddir(self, args): - """ builddir [-f] <hostname> <dirname> - Build config for hostname, writing separate files to dirname""" - alist = args.split() - path_force = False - if '-f' in args: - alist.remove('-f') - path_force = True - if len(alist) == 2: - client, odir = alist - if not odir.startswith('/tmp') and not path_force: - print("Refusing to write files outside of /tmp without -f " - "option") - return - client_config = self.BuildConfiguration(client) - if client_config.tag == 'error': - print("Building client configuration failed.") - return - - for struct in client_config: - for entry in struct: - if entry.tag == 'Path': - entry.set('name', odir + '/' + entry.get('name')) - - posix = Bcfg2.Client.Tools.POSIX.POSIX(MockLog(), - self.setup, - client_config) - states = posix.Inventory() - posix.Install(list(states.keys())) - else: - print('Error: Incorrect number of parameters.') - print(self.do_builddir.__doc__) - - def do_buildall(self, args): - """buildall <directory> [<hostnames*>] - Build configs for all clients in directory""" - alist = args.split() - if len(alist) < 1: - print(self.do_buildall.__doc__) - return - - destdir = alist[0] - try: - os.mkdir(destdir) - except OSError: - err = sys.exc_info()[1] - if err.errno != 17: - print("Could not create %s: %s" % (destdir, err)) - if len(alist) > 1: - clients = self._get_client_list(alist[1:]) - else: - clients = self.metadata.clients - for client in clients: - self.do_build("%s %s" % (client, os.path.join(destdir, - client + ".xml"))) - - def do_buildallfile(self, args): - """ buildallfile <directory> <filename> [<hostnames*>] - Build config file for all clients in directory""" - try: - opts, args = getopt.gnu_getopt(args.split(), '', ['altsrc=']) - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self.do_buildallfile.__doc__) - return - altsrc = None - for opt in opts: - if opt[0] == '--altsrc': - altsrc = opt[1] - if len(args) < 2: - print(self.do_buildallfile.__doc__) - return - - destdir = args[0] - filename = args[1] - try: - os.mkdir(destdir) - except OSError: - err = sys.exc_info()[1] - if err.errno != 17: - print("Could not create %s: %s" % (destdir, err)) - if len(args) > 2: - clients = self._get_client_list(args[1:]) - else: - clients = self.metadata.clients - if altsrc: - args = "--altsrc %s -f %%s %%s %%s" % altsrc - else: - args = "-f %s %s %s" - for client in clients: - self.do_buildfile(args % (os.path.join(destdir, client), - filename, client)) - - def do_buildfile(self, args): - """buildfile [-f <outfile>] [--altsrc=<altsrc>] <filename> <hostname> - Build config file for hostname (not written to disk)""" - try: - opts, alist = getopt.gnu_getopt(args.split(), 'f:', ['altsrc=']) - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self.do_buildfile.__doc__) - return - altsrc = None - outfile = None - for opt in opts: - if opt[0] == '--altsrc': - altsrc = opt[1] - elif opt[0] == '-f': - outfile = opt[1] - if len(alist) != 2: - print(self.do_buildfile.__doc__) - return - - fname, client = alist - entry = lxml.etree.Element('Path', type='file', name=fname) - if altsrc: - entry.set("altsrc", altsrc) - try: - metadata = self.build_metadata(client) - self.Bind(entry, metadata) - data = lxml.etree.tostring(entry, - xml_declaration=False).decode('UTF-8') - except Exception: - print("Failed to build entry %s for host %s: %s" % - (fname, client, traceback.format_exc().splitlines()[-1])) - raise - try: - if outfile: - open(outfile, 'w').write(data) - else: - print(data) - except IOError: - err = sys.exc_info()[1] - print("Could not write to %s: %s" % (outfile, err)) - print(data) - - def do_buildbundle(self, args): - """buildbundle <bundle> <hostname> - Render a templated bundle for hostname (not written to disk)""" - if len(args.split()) != 2: - print(self.do_buildbundle.__doc__) - return - - bname, client = args.split() - try: - metadata = self.build_metadata(client) - bundle = self.plugins['Bundler'].entries[bname] - print(lxml.etree.tostring(bundle.get_xml_value(metadata), - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - except KeyError: - print("No such bundle %s" % bname) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - print("Failed to render bundle %s for host %s: %s" % (bname, - client, - err)) - - def do_automatch(self, args): - """automatch [-f] <propertyfile> <hostname> - Perform automatch on a Properties file""" - alist = args.split() - force = False - for arg in alist: - if arg == '-f': - alist.remove('-f') - force = True - if len(alist) != 2: - print(self.do_automatch.__doc__) - return - - if 'Properties' not in self.plugins: - print("Properties plugin not enabled") - return - - pname, client = alist - automatch = self.setup.cfp.getboolean("properties", "automatch", - default=False) - pfile = self.plugins['Properties'].entries[pname] - if (not force and - not automatch and - pfile.xdata.get("automatch", "false").lower() != "true"): - print("Automatch not enabled on %s" % pname) - else: - metadata = self.build_metadata(client) - print(lxml.etree.tostring(pfile.XMLMatch(metadata), - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - - def do_bundles(self, _): - """bundles - Print out group/bundle info""" - data = [('Group', 'Bundles')] - groups = list(self.metadata.groups.keys()) - groups.sort() - for group in groups: - data.append((group, - ','.join(self.metadata.groups[group][0]))) - print_tabular(data) - - def do_clients(self, _): - """clients - Print out client/profile info""" - data = [('Client', 'Profile')] - for client in sorted(self.metadata.list_clients()): - imd = self.metadata.get_initial_metadata(client) - data.append((client, imd.profile)) - print_tabular(data) - - def do_config(self, _): - """config - Print out the current configuration of Bcfg2""" - output = [ - ('Description', 'Value'), - ('Path Bcfg2 repository', self.setup['repo']), - ('Plugins', self.setup['plugins']), - ('Password', self.setup['password']), - ('Filemonitor', self.setup['filemonitor']), - ('Server address', self.setup['location']), - ('Path to key', self.setup['key']), - ('Path to SSL certificate', self.setup['cert']), - ('Path to SSL CA certificate', self.setup['ca']), - ('Protocol', self.setup['protocol']), - ('Logging', self.setup['logging'])] - print_tabular(output) - - def do_expirecache(self, args): - """ expirecache [<hostname> [<hostname> ...]]- Expire the - metadata cache """ - alist = args.split() - if len(alist): - for client in self._get_client_list(alist): - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, - key=client) - else: - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) - - def do_probes(self, args): - """probes [-p] <hostname> - Get probe list for the given host, in XML (the default) \ -or human-readable pretty (with -p) format""" - alist = args.split() - pretty = False - if '-p' in alist: - pretty = True - alist.remove('-p') - if len(alist) != 1: - print(self.do_probes.__doc__) - return - hostname = alist[0] - if pretty: - probes = [] - else: - probes = lxml.etree.Element('probes') - metadata = self.build_metadata(hostname) - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): - for probe in plugin.GetProbes(metadata): - probes.append(probe) - if pretty: - for probe in probes: - pname = probe.get("name") - print("=" * (len(pname) + 2)) - print(" %s" % pname) - print("=" * (len(pname) + 2)) - print("") - print(probe.text) - print("") - else: - print(lxml.etree.tostring(probes, - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - - def do_showentries(self, args): - """showentries <hostname> <type> - Show abstract configuration entries for a given host""" - arglen = len(args.split()) - if arglen not in [1, 2]: - print(self.do_showentries.__doc__) - return - client = args.split()[0] - try: - meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Unable to find metadata for host %s" % client) - return - structures = self.GetStructures(meta) - output = [('entrytype', 'name')] - if arglen == 1: - for item in structures: - for child in item.getchildren(): - output.append((child.tag, child.get('name'))) - if arglen == 2: - etype = args.split()[1] - for item in structures: - for child in item.getchildren(): - if child.tag in [etype, "Bound%s" % etype]: - output.append((child.tag, child.get('name'))) - print_tabular(output) - - def do_groups(self, _): - """groups - Print out group info""" - data = [("Groups", "Profile", "Category")] - grouplist = list(self.metadata.groups.keys()) - grouplist.sort() - for group in grouplist: - if self.metadata.groups[group].is_profile: - prof = 'yes' - else: - prof = 'no' - cat = self.metadata.groups[group].category - data.append((group, prof, cat)) - print_tabular(data) - - def do_showclient(self, args): - """showclient <client> [<client> ...] - Show metadata for the given hosts""" - if not len(args): - print(self.do_showclient.__doc__) - return - for client in args.split(): - try: - client_meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Client %s not defined" % client) - continue - fmt = "%-10s %s" - print(fmt % ("Hostname:", client_meta.hostname)) - print(fmt % ("Profile:", client_meta.profile)) - - group_fmt = "%-10s %-30s %s" - header = False - for group in list(client_meta.groups): - category = "" - for cat, grp in client_meta.categories.items(): - if grp == group: - category = "Category: %s" % cat - break - if not header: - print(group_fmt % ("Groups:", group, category)) - header = True - else: - print(group_fmt % ("", group, category)) - - if client_meta.bundles: - print(fmt % ("Bundles:", list(client_meta.bundles)[0])) - for bnd in list(client_meta.bundles)[1:]: - print(fmt % ("", bnd)) - if client_meta.connectors: - print("Connector data") - print("=" * 80) - for conn in client_meta.connectors: - if getattr(client_meta, conn): - print(fmt % (conn + ":", getattr(client_meta, conn))) - print("=" * 80) - - def do_mappings(self, args): - """mappings <type*> <name*> - Print generator mappings for optional type and name""" - # Dump all mappings unless type specified - data = [('Plugin', 'Type', 'Name')] - arglen = len(args.split()) - for generator in self.plugins_by_type(Bcfg2.Server.Plugin.Generator): - if arglen == 0: - etypes = list(generator.Entries.keys()) - else: - etypes = [args.split()[0]] - if arglen == 2: - interested = [(etype, [args.split()[1]]) - for etype in etypes] - else: - interested = [(etype, generator.Entries[etype]) - for etype in etypes - if etype in generator.Entries] - for etype, names in interested: - for name in [name for name in names if name in - generator.Entries.get(etype, {})]: - data.append((generator.name, etype, name)) - print_tabular(data) - - def do_event_debug(self, _): - """event_debug - Display filesystem events as they are processed""" - self.fam.debug = True - - def do_packageresolve(self, args): - """packageresolve <hostname> [<package> [<package>...]] - Resolve packages for the given host, optionally specifying a \ -set of packages""" - arglist = args.split(" ") - if len(arglist) < 1: - print(self.do_packageresolve.__doc__) - return - - try: - pkgs = self.plugins['Packages'] - except KeyError: - print("Packages plugin not enabled") - return - pkgs.toggle_debug() - - hostname = arglist[0] - metadata = self.build_metadata(hostname) - - indep = lxml.etree.Element("Independent") - if len(arglist) > 1: - structures = [lxml.etree.Element("Bundle", name="packages")] - for arg in arglist[1:]: - lxml.etree.SubElement(structures[0], "Package", name=arg) - else: - structures = self.GetStructures(metadata) - - pkgs._build_packages(metadata, indep, # pylint: disable=W0212 - structures) - print("%d new packages added" % len(indep.getchildren())) - if len(indep.getchildren()): - print(" %s" % "\n ".join( - lxml.etree.tostring(p, encoding=unicode) - for p in indep.getchildren())) - - def do_packagesources(self, args): - """packagesources <hostname> - Show package sources""" - if not args: - print(self.do_packagesources.__doc__) - return - if 'Packages' not in self.plugins: - print("Packages plugin not enabled") - return - try: - metadata = self.build_metadata(args) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Unable to build metadata for host %s" % args) - return - collection = self.plugins['Packages'].get_collection(metadata) - print(collection.sourcelist()) - - def do_query(self, args): - """query <-g group|-p profile|-b bundle> - Query clients""" - if len(args) == 0: - print("\n".join(self.metadata.clients)) - return - arglist = args.split(" ") - if len(arglist) != 2: - print(self.do_query.__doc__) - return - - qtype, qparam = arglist - if qtype == '-p': - res = self.metadata.get_client_names_by_profiles(qparam.split(',')) - elif qtype == '-g': - res = self.metadata.get_client_names_by_groups(qparam.split(',')) - elif qtype == '-b': - res = self.metadata.get_client_names_by_bundles(qparam.split(',')) - else: - print(self.do_query.__doc__) - return - print("\n".join(res)) - - def do_profile(self, arg): - """profile <command> <args> - Profile a single bcfg2-info command""" - if not HAS_PROFILE: - print("Profiling functionality not available.") - return - if len(arg) == 0: - print(self.do_profile.__doc__) - return - prof = profile.Profile() - prof.runcall(self.onecmd, arg) - display_trace(prof) - - def run(self, args): # pylint: disable=W0221 - try: - self.load_plugins() - self.block_for_fam_events(handle_events=True) - if args: - self.onecmd(" ".join(args)) - else: - try: - self.cmdloop('Welcome to bcfg2-info\n' - 'Type "help" for more information') - except KeyboardInterrupt: - print("\nCtrl-C pressed exiting...") - self.do_exit([]) - except Bcfg2.Server.Plugin.PluginExecutionError: - pass - finally: - self.shutdown() - - def _daemonize(self): - pass - - def _run(self): - pass - - def _block(self): - pass - - -def build_usage(): - """build usage message""" - cmd_blacklist = ["do_loop", "do_EOF"] - usage = dict() - for attrname in dir(InfoCore): - attr = getattr(InfoCore, attrname) - - # shim for python 2.4, __func__ is im_func - funcattr = getattr(attr, "__func__", getattr(attr, "im_func", None)) - if (funcattr is not None and - funcattr.func_name not in cmd_blacklist and - funcattr.func_name.startswith("do_") and - funcattr.func_doc): - usage[attr.__name__] = re.sub(r'\s+', ' ', attr.__doc__) - return "Commands:\n" + "\n".join(usage[k] for k in sorted(usage.keys())) - - -USAGE = build_usage() - - -def main(): - optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, - interactive=Bcfg2.Options.INTERACTIVE, - interpreter=Bcfg2.Options.INTERPRETER, - command_timeout=Bcfg2.Options.CLIENT_COMMAND_TIMEOUT) - optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.hm = "\n".join(["bcfg2-info [options] [command <command args>]", - "Options:", - setup.buildHelpMessage(), - USAGE]) - - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-info', to_syslog=False, level=level) - - if setup['args'] and setup['args'][0] == 'help': - print(setup.hm) - sys.exit(0) - elif setup['profile'] and HAS_PROFILE: - prof = profile.Profile() - loop = prof.runcall(InfoCore) - display_trace(prof) - else: - if setup['profile']: - print("Profiling functionality not available.") - loop = InfoCore() - - loop.run(setup['args']) - +from Bcfg2.Server.Info import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 27d8d4291..e818dc3be 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -1,213 +1,8 @@ #!/usr/bin/env python - """This tool examines your Bcfg2 specifications for errors.""" import sys -import time -import inspect -import logging -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Core -import Bcfg2.Server.Lint - -LOGGER = logging.getLogger('bcfg2-lint') - - -def run_serverless_plugins(plugins, errorhandler=None, files=None): - """ Run serverless plugins """ - LOGGER.debug("Running serverless plugins") - for plugin_name, plugin in list(plugins.items()): - run_plugin(plugin, plugin_name, errorhandler=errorhandler, files=files) - - -def run_server_plugins(plugins, errorhandler=None, files=None): - """ run plugins that require a running server to run """ - core = load_server() - try: - LOGGER.debug("Running server plugins") - for plugin_name, plugin in list(plugins.items()): - run_plugin(plugin, plugin_name, args=[core], - errorhandler=errorhandler, files=files) - finally: - core.shutdown() - - -def run_plugin(plugin, plugin_name, errorhandler=None, args=None, files=None): - """ run a single plugin, server-ful or serverless. """ - LOGGER.debug(" Running %s" % plugin_name) - if args is None: - args = [] - - if errorhandler is None: - errorhandler = get_errorhandler() - - setup = Bcfg2.Options.get_option_parser() - if setup.cfp.has_section(plugin_name): - arg = setup - for key, val in setup.cfp.items(plugin_name): - arg[key] = val - args.append(arg) - else: - args.append(setup) - - # python 2.5 doesn't support mixing *magic and keyword arguments - start = time.time() - rv = plugin(*args, **dict(files=files, errorhandler=errorhandler)).Run() - LOGGER.debug(" Ran %s in %0.2f seconds" % (plugin_name, - time.time() - start)) - return rv - - -def get_errorhandler(): - """ get a Bcfg2.Server.Lint.ErrorHandler object """ - setup = Bcfg2.Options.get_option_parser() - if setup.cfp.has_section("errors"): - errors = dict(setup.cfp.items("errors")) - else: - errors = None - return Bcfg2.Server.Lint.ErrorHandler(errors=errors) - - -def load_server(): - """ load server """ - core = Bcfg2.Server.Core.BaseCore() - core.load_plugins() - core.block_for_fam_events(handle_events=True) - return core - - -def load_plugin(module, obj_name=None): - """ load a single plugin """ - parts = module.split(".") - if obj_name is None: - obj_name = parts[-1] - - mod = __import__(module) - for part in parts[1:]: - mod = getattr(mod, part) - return getattr(mod, obj_name) - - -def load_plugins(): - """ get list of plugins to run """ - setup = Bcfg2.Options.get_option_parser() - if setup['args']: - plugin_list = setup['args'] - elif "bcfg2-repo-validate" in sys.argv[0]: - plugin_list = 'RequiredAttrs,Validate'.split(',') - elif setup['lint_plugins']: - plugin_list = setup['lint_plugins'] - else: - plugin_list = Bcfg2.Server.Lint.plugins - - allplugins = dict() - for plugin in plugin_list: - try: - allplugins[plugin] = load_plugin("Bcfg2.Server.Lint." + plugin) - except ImportError: - try: - allplugins[plugin] = \ - load_plugin("Bcfg2.Server.Plugins." + plugin, - obj_name=plugin + "Lint") - except (ImportError, AttributeError): - err = sys.exc_info()[1] - LOGGER.error("Failed to load plugin %s: %s" % - (plugin + "Lint", err)) - except AttributeError: - err = sys.exc_info()[1] - LOGGER.error("Failed to load plugin %s: %s" % (plugin, err)) - - for plugin in setup['plugins']: - if plugin in allplugins: - # already loaded - continue - - try: - allplugins[plugin] = \ - load_plugin("Bcfg2.Server.Plugins." + plugin, - obj_name=plugin + "Lint") - except AttributeError: - pass - except ImportError: - err = sys.exc_info()[1] - LOGGER.error("Failed to load plugin %s: %s" % (plugin + "Lint", - err)) - - serverplugins = dict() - serverlessplugins = dict() - for plugin_name, plugin in allplugins.items(): - if [c for c in inspect.getmro(plugin) - if c == Bcfg2.Server.Lint.ServerPlugin]: - serverplugins[plugin_name] = plugin - else: - serverlessplugins[plugin_name] = plugin - return (serverlessplugins, serverplugins) - - -def main(): - optinfo = dict(lint_config=Bcfg2.Options.LINT_CONFIG, - showerrors=Bcfg2.Options.LINT_SHOW_ERRORS, - stdin=Bcfg2.Options.LINT_FILES_ON_STDIN, - schema=Bcfg2.Options.SCHEMA_PATH, - lint_plugins=Bcfg2.Options.LINT_PLUGINS) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.load_option_parser(optinfo) - setup.parse(sys.argv[1:]) - - log_args = dict(to_syslog=setup['syslog'], to_console=logging.WARNING) - if setup['verbose']: - log_args['to_console'] = logging.DEBUG - Bcfg2.Logger.setup_logging('bcfg2-info', **log_args) - - setup.cfp.read(setup['lint_config']) - setup.reparse() - - if setup['stdin']: - files = [s.strip() for s in sys.stdin.readlines()] - else: - files = None - - (serverlessplugins, serverplugins) = load_plugins() - errorhandler = get_errorhandler() - - if setup['showerrors']: - for plugin in serverplugins.values() + serverlessplugins.values(): - errorhandler.RegisterErrors(getattr(plugin, 'Errors')()) - - print("%-35s %-35s" % ("Error name", "Handler")) - for err, handler in errorhandler.errortypes.items(): - print("%-35s %-35s" % (err, handler.__name__)) - raise SystemExit(0) - - run_serverless_plugins(serverlessplugins, errorhandler=errorhandler, - files=files) - - if serverplugins: - if errorhandler.errors: - # it would be swell if we could try to start the server - # even if there were errors with the serverless plugins, - # but since XML parsing errors occur in the FAM thread - # (not in the core server thread), there's no way we can - # start the server and try to catch exceptions -- - # bcfg2-lint isn't in the same stack as the exceptions. - # so we're forced to assume that a serverless plugin error - # will prevent the server from starting - print("Serverless plugins encountered errors, skipping server " - "plugins") - else: - run_server_plugins(serverplugins, errorhandler=errorhandler, - files=files) - - if errorhandler.errors or errorhandler.warnings or setup['verbose']: - print("%d errors" % errorhandler.errors) - print("%d warnings" % errorhandler.warnings) - - if errorhandler.errors: - raise SystemExit(2) - elif errorhandler.warnings: - raise SystemExit(3) +from Bcfg2.Server.Lint import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-report-collector b/src/sbin/bcfg2-report-collector index ae6d3b167..00e015100 100755 --- a/src/sbin/bcfg2-report-collector +++ b/src/sbin/bcfg2-report-collector @@ -11,20 +11,14 @@ from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError def main(): + parser = Bcfg2.Options.get_parser(description="Collect Bcfg2 report data", + components=[ReportingCollector]) + parser.parse() logger = logging.getLogger('bcfg2-report-collector') - optinfo = dict(daemon=Bcfg2.Options.DAEMON, - repo=Bcfg2.Options.SERVER_REPOSITORY, - filemonitor=Bcfg2.Options.SERVER_FILEMONITOR, - web_configfile=Bcfg2.Options.WEB_CFILE) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.REPORTING_COMMON_OPTIONS) - setup = Bcfg2.Options.load_option_parser(optinfo) - setup.parse() # run collector try: - collector = ReportingCollector(setup) - collector.run() + ReportingCollector().run() except ReportingError: msg = sys.exc_info()[1] logger.error(msg) diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index beb19cef6..d6ce7d44f 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -2,63 +2,47 @@ """The XML-RPC Bcfg2 server.""" -import os import sys import logging -import Bcfg2.Logger import Bcfg2.Options from Bcfg2.Server.Core import CoreInitError -LOGGER = logging.getLogger('bcfg2-server') - -def main(): - optinfo = dict() - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.DAEMON_COMMON_OPTIONS) - setup = Bcfg2.Options.load_option_parser(optinfo) - setup.parse(sys.argv[1:]) - # check whether the specified bcfg2.conf exists - if not os.path.exists(setup['configfile']): - print("Could not read %s" % setup['configfile']) - sys.exit(1) - - # TODO: normalize case of various core modules so we can add a new - # core without modifying this script - backends = dict(cherrypy='CherryPyCore', - builtin='BuiltinCore', - best='BuiltinCore', - multiprocessing='MultiprocessingCore') - - if setup['backend'] not in backends: - print("Unknown server backend %s, using 'best'" % setup['backend']) - setup['backend'] = 'best' - - coremodule = backends[setup['backend']] - try: - corecls = getattr(__import__("Bcfg2.Server.%s" % coremodule).Server, - coremodule).Core - except ImportError: - err = sys.exc_info()[1] - print("Unable to import %s server core: %s" % (setup['backend'], err)) - raise - except AttributeError: - err = sys.exc_info()[1] - print("Unable to load %s server core: %s" % (setup['backend'], err)) - raise - - try: - core = corecls(setup) - core.run() - except CoreInitError: - msg = sys.exc_info()[1] - LOGGER.error(msg) - sys.exit(1) - except KeyboardInterrupt: - sys.exit(1) - sys.exit(0) +class BackendAction(Bcfg2.Options.ComponentAction): + """ Action to load Bcfg2 backends """ + islist = False + bases = ['Bcfg2.Server'] + + +class CLI(object): + """ bcfg2-server CLI class """ + options = [ + Bcfg2.Options.Option( + cf=('server', 'backend'), help='Server Backend', + default='Builtin', type=lambda b: b.title() + "Core", + action=BackendAction)] + + def __init__(self): + parser = Bcfg2.Options.get_parser("Bcfg2 server", components=[self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + def run(self): + """ Run the bcfg2 server """ + try: + core = Bcfg2.Options.setup.backend() + core.run() + except CoreInitError: + self.logger.error(sys.exc_info()[1]) + return 1 + except TypeError: + self.logger.error("Failed to load %s server backend: %s" % + (Bcfg2.Options.setup.backend.__name__, + sys.exc_info()[1])) + raise + except KeyboardInterrupt: + return 1 if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 564ddec49..73d9f13a7 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -1,319 +1,9 @@ #!/usr/bin/env python +""" This tool verifies that all clients known to the server build +without failures """ -"""This tool verifies that all clients known to the server build -without failures""" - -import os import sys -import signal -import fnmatch -import logging -import Bcfg2.Logger -import Bcfg2.Server.Core -from math import ceil -from nose.core import TestProgram -from nose.suite import LazySuite -from unittest import TestCase - -try: - from multiprocessing import Process, Queue, active_children - HAS_MULTIPROC = True -except ImportError: - HAS_MULTIPROC = False - active_children = lambda: [] # pylint: disable=C0103 - - -class CapturingLogger(object): - """ Fake logger that captures logging output so that errors are - only displayed for clients that fail tests """ - def __init__(self, *args, **kwargs): # pylint: disable=W0613 - self.output = [] - - def error(self, msg): - """ discard error messages """ - self.output.append(msg) - - def warning(self, msg): - """ discard error messages """ - self.output.append(msg) - - def info(self, msg): - """ discard error messages """ - self.output.append(msg) - - def debug(self, msg): - """ discard error messages """ - self.output.append(msg) - - def reset_output(self): - """ Reset the captured output """ - self.output = [] - - -class ClientTestFromQueue(TestCase): - """ A test case that tests a value that has been enqueued by a - child test process. ``client`` is the name of the client that has - been tested; ``result`` is the result from the :class:`ClientTest` - test. ``None`` indicates a successful test; a string value - indicates a failed test; and an exception indicates an error while - running the test. """ - __test__ = False # Do not collect - - def __init__(self, client, result): - TestCase.__init__(self) - self.client = client - self.result = result - - def shortDescription(self): - return "Building configuration for %s" % self.client - - def runTest(self): - """ parse the result from this test """ - if isinstance(self.result, Exception): - raise self.result - assert self.result is None, self.result - - -class ClientTest(TestCase): - """ A test case representing the build of all of the configuration for - a single host. Checks that none of the build config entities has - had a failure when it is building. Optionally ignores some config - files that we know will cause errors (because they are private - files we don't have access to, for instance) """ - __test__ = False # Do not collect - divider = "-" * 70 - - def __init__(self, core, client, ignore=None): - TestCase.__init__(self) - self.core = core - self.core.logger = CapturingLogger() - self.client = client - if ignore is None: - self.ignore = dict() - else: - self.ignore = ignore - - def ignore_entry(self, tag, name): - """ return True if an error on a given entry should be ignored - """ - if tag in self.ignore: - if name in self.ignore[tag]: - return True - else: - # try wildcard matching - for pattern in self.ignore[tag]: - if fnmatch.fnmatch(name, pattern): - return True - return False - - def shortDescription(self): - return "Building configuration for %s" % self.client - - def runTest(self): - """ run this individual test """ - config = self.core.BuildConfiguration(self.client) - output = self.core.logger.output[:] - if output: - output.append(self.divider) - self.core.logger.reset_output() - - # check for empty client configuration - assert len(config.findall("Bundle")) > 0, \ - "\n".join(output + ["%s has no content" % self.client]) - - # check for missing bundles - metadata = self.core.build_metadata(self.client) - sbundles = [el.get('name') for el in config.findall("Bundle")] - missing = [b for b in metadata.bundles if b not in sbundles] - assert len(missing) == 0, \ - "\n".join(output + ["Configuration is missing bundle(s): %s" % - ':'.join(missing)]) - - # check for unknown packages - unknown_pkgs = [el.get("name") - for el in config.xpath('//Package[@type="unknown"]') - if not self.ignore_entry(el.tag, el.get("name"))] - assert len(unknown_pkgs) == 0, \ - "Configuration contains unknown packages: %s" % \ - ", ".join(unknown_pkgs) - - # check for render failures - failures = [] - msg = output + ["Failures:"] - for failure in config.xpath('//*[@failure]'): - if not self.ignore_entry(failure.tag, failure.get('name')): - failures.append(failure) - msg.append("%s:%s: %s" % (failure.tag, failure.get("name"), - failure.get("failure"))) - - assert len(failures) == 0, "\n".join(msg) - - def __str__(self): - return "ClientTest(%s)" % self.client - - id = __str__ - - -def get_core(setup): - """ Get a server core, with events handled """ - core = Bcfg2.Server.Core.BaseCore(setup) - core.load_plugins() - core.block_for_fam_events(handle_events=True) - return core - - -def get_ignore(setup): - """ Given an options dict, get a dict of entry tags and names to - ignore errors from """ - ignore = dict() - for entry in setup['test_ignore']: - tag, name = entry.split(":") - try: - ignore[tag].append(name) - except KeyError: - ignore[tag] = [name] - return ignore - - -def run_child(setup, clients, queue): - """ Run tests for the given clients in a child process, returning - results via the given Queue """ - core = get_core(setup) - ignore = get_ignore(setup) - for client in clients: - try: - ClientTest(core, client, ignore).runTest() - queue.put((client, None)) - except AssertionError: - queue.put((client, str(sys.exc_info()[1]))) - except: - queue.put((client, sys.exc_info()[1])) - - core.shutdown() - - -def get_sigint_handler(core): - """ Get a function that handles SIGINT/Ctrl-C by shutting down the - core and exiting properly.""" - - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - core.shutdown() - os._exit(1) # pylint: disable=W0212 - - return hdlr - - -def parse_args(): - """ Parse command line arguments. """ - optinfo = dict(Bcfg2.Options.TEST_COMMON_OPTIONS) - - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.load_option_parser(optinfo) - setup.hm = \ - "bcfg2-test [options] [client] [client] [...]\nOptions:\n %s" % \ - setup.buildHelpMessage() - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging("bcfg2-test", - to_console=setup['verbose'] or setup['debug'], - to_syslog=False, - to_file=setup['logging'], - level=level) - logger = logging.getLogger(sys.argv[0]) - if (setup['debug'] or setup['verbose']) and "-v" not in setup['noseopts']: - setup['noseopts'].append("-v") - - if setup['children'] and not HAS_MULTIPROC: - logger.warning("Python multiprocessing library not found, running " - "with no children") - setup['children'] = 0 - - if (setup['children'] and ('--with-xunit' in setup['noseopts'] or - '--xunit-file' in setup['noseopts'])): - logger.warning("Use the --xunit option to bcfg2-test instead of the " - "--with-xunit or --xunit-file options to nosetest") - xunitfile = None - if '--with-xunit' in setup['noseopts']: - setup['noseopts'].remove('--with-xunit') - xunitfile = "nosetests.xml" - if '--xunit-file' in setup['noseopts']: - idx = setup['noseopts'].index('--xunit-file') - try: - setup['noseopts'].pop(idx) # remove --xunit-file - # remove the argument to it - xunitfile = setup['noseopts'].pop(idx) - except IndexError: - pass - if xunitfile and not setup['xunit']: - setup['xunit'] = xunitfile - return setup - - -def main(): - setup = parse_args() - logger = logging.getLogger(sys.argv[0]) - core = get_core(setup) - signal.signal(signal.SIGINT, get_sigint_handler(core)) - - if setup['args']: - clients = setup['args'] - else: - clients = core.metadata.clients - - ignore = get_ignore(setup) - - if setup['children']: - if setup['children'] > len(clients): - logger.info("Refusing to spawn more children than clients to test," - " setting children=%s" % len(clients)) - setup['children'] = len(clients) - perchild = int(ceil(len(clients) / float(setup['children'] + 1))) - queue = Queue() - for child in range(setup['children']): - start = child * perchild - end = (child + 1) * perchild - child = Process(target=run_child, - args=(setup, clients[start:end], queue)) - child.start() - - def generate_tests(): - """ Read test results for the clients """ - start = setup['children'] * perchild - for client in clients[start:]: - yield ClientTest(core, client, ignore) - - for i in range(start): # pylint: disable=W0612 - yield ClientTestFromQueue(*queue.get()) - else: - def generate_tests(): - """ Run tests for the clients """ - for client in clients: - yield ClientTest(core, client, ignore) - - result = TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], - suite=LazySuite(generate_tests), exit=False) - - # block until all children have completed -- should be - # immediate since we've already gotten all the results we - # expect - for child in active_children(): - child.join() - - core.shutdown() - if result.success: - os._exit(0) # pylint: disable=W0212 - else: - os._exit(1) # pylint: disable=W0212 - +from Bcfg2.Server.Test import CLI if __name__ == "__main__": - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 49baeb9c3..95fb9889e 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -5,358 +5,8 @@ the right way to get around that in long-running processes it to have a short-lived helper. No, seriously -- check out the yum-updatesd code. It's pure madness. """ -import os import sys -import yum -import logging -import Bcfg2.Logger -from Bcfg2.Compat import wraps -from lockfile import FileLock, LockTimeout -from optparse import OptionParser -try: - import json -except ImportError: - import simplejson as json - - -def pkg_to_tuple(package): - """ json doesn't distinguish between tuples and lists, but yum - does, so we convert a package in list format to one in tuple - format """ - if isinstance(package, list): - return tuple(package) - else: - return package - - -def pkgtup_to_string(package): - """ given a package tuple, return a human-readable string - describing the package """ - if package[3] in ['auto', 'any']: - return package[0] - - rv = [package[0], "-"] - if package[2]: - rv.extend([package[2], ':']) - rv.extend([package[3], '-', package[4]]) - if package[1]: - rv.extend(['.', package[1]]) - return ''.join(str(e) for e in rv) - - -class YumHelper(object): - """ Yum helper base object """ - - def __init__(self, cfgfile, verbose=1): - self.cfgfile = cfgfile - self.yumbase = yum.YumBase() - # pylint: disable=E1121,W0212 - try: - self.yumbase.preconf.debuglevel = verbose - self.yumbase.preconf.fn = cfgfile - self.yumbase._getConfig() - except AttributeError: - self.yumbase._getConfig(cfgfile, debuglevel=verbose) - # pylint: enable=E1121,W0212 - self.logger = logging.getLogger(self.__class__.__name__) - - -class DepSolver(YumHelper): - """ Yum dependency solver. This is used for operations that only - read from the yum cache, and thus operates in cacheonly mode. """ - - def __init__(self, cfgfile, verbose=1): - YumHelper.__init__(self, cfgfile, verbose=verbose) - # internally, yum uses an integer, not a boolean, for conf.cache - self.yumbase.conf.cache = 1 - self._groups = None - - def get_groups(self): - """ getter for the groups property """ - if self._groups is not None: - return self._groups - else: - return ["noarch"] - - def set_groups(self, groups): - """ setter for the groups property """ - self._groups = set(groups).union(["noarch"]) - - groups = property(get_groups, set_groups) - - def get_package_object(self, pkgtup, silent=False): - """ given a package tuple, get a yum package object """ - try: - matches = yum.packageSack.packagesNewestByName( - self.yumbase.pkgSack.searchPkgTuple(pkgtup)) - except yum.Errors.PackageSackError: - if not silent: - self.logger.warning("Package '%s' not found" % - self.get_package_name(pkgtup)) - matches = [] - except yum.Errors.RepoError: - err = sys.exc_info()[1] - self.logger.error("Temporary failure loading metadata for %s: %s" % - (self.get_package_name(pkgtup), err)) - matches = [] - - pkgs = self._filter_arch(matches) - if pkgs: - return pkgs[0] - else: - return None - - def get_group(self, group, ptype="default"): - """ Resolve a package group name into a list of packages """ - if group.startswith("@"): - group = group[1:] - - try: - if self.yumbase.comps.has_group(group): - group = self.yumbase.comps.return_group(group) - else: - self.logger.error("%s is not a valid group" % group) - return [] - except yum.Errors.GroupsError: - err = sys.exc_info()[1] - self.logger.warning(err) - return [] - - if ptype == "default": - return [p - for p, d in list(group.default_packages.items()) - if d] - elif ptype == "mandatory": - return [p - for p, m in list(group.mandatory_packages.items()) - if m] - elif ptype == "optional" or ptype == "all": - return group.packages - else: - self.logger.warning("Unknown group package type '%s'" % ptype) - return [] - - def _filter_arch(self, packages): - """ filter packages in the given list that do not have an - architecture in the list of groups for this client """ - matching = [] - for pkg in packages: - if pkg.arch in self.groups: - matching.append(pkg) - else: - self.logger.debug("%s has non-matching architecture (%s)" % - (pkg, pkg.arch)) - if matching: - return matching - else: - # no packages match architecture; we'll assume that the - # user knows what s/he is doing and this is a multiarch - # box. - return packages - - def get_package_name(self, package): - """ get the name of a package or virtual package from the - internal representation used by this Collection class """ - if isinstance(package, tuple): - if len(package) == 3: - return yum.misc.prco_tuple_to_string(package) - else: - return pkgtup_to_string(package) - else: - return str(package) - - def complete(self, packagelist): - """ resolve dependencies and generate a complete package list - from the given list of initial packages """ - packages = set() - unknown = set() - for pkg in packagelist: - if isinstance(pkg, tuple): - pkgtup = pkg - else: - pkgtup = (pkg, None, None, None, None) - pkgobj = self.get_package_object(pkgtup) - if not pkgobj: - self.logger.debug("Unknown package %s" % - self.get_package_name(pkg)) - unknown.add(pkg) - else: - if self.yumbase.tsInfo.exists(pkgtup=pkgobj.pkgtup): - self.logger.debug("%s added to transaction multiple times" - % pkgobj) - else: - self.logger.debug("Adding %s to transaction" % pkgobj) - self.yumbase.tsInfo.addInstall(pkgobj) - self.yumbase.resolveDeps() - - for txmbr in self.yumbase.tsInfo: - packages.add(txmbr.pkgtup) - return list(packages), list(unknown) - - -def acquire_lock(func): - """ decorator for CacheManager methods that gets and release a - lock while the method runs """ - @wraps(func) - def inner(self, *args, **kwargs): - """ Get and release a lock while running the function this - wraps. """ - self.logger.debug("Acquiring lock at %s" % self.lockfile) - while not self.lock.i_am_locking(): - try: - self.lock.acquire(timeout=60) # wait up to 60 seconds - except LockTimeout: - self.lock.break_lock() - self.lock.acquire() - try: - func(self, *args, **kwargs) - finally: - self.lock.release() - self.logger.debug("Released lock at %s" % self.lockfile) - - return inner - - -class CacheManager(YumHelper): - """ Yum cache manager. Unlike :class:`DepSolver`, this can write - to the yum cache, and so is used for operations that muck with the - cache. (Technically, :func:`CacheManager.clean_cache` could be in - either DepSolver or CacheManager, but for consistency I've put it - here.) """ - - def __init__(self, cfgfile, verbose=1): - YumHelper.__init__(self, cfgfile, verbose=verbose) - self.lockfile = \ - os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), - "lock") - self.lock = FileLock(self.lockfile) - - @acquire_lock - def clean_cache(self): - """ clean the yum cache """ - for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", - "ExpireCache"]: - # for reasons that are entirely obvious, all of the yum - # API clean* methods return a tuple of 0 (zero, always - # zero) and a list containing a single message about how - # many files were deleted. so useful. thanks, yum. - msg = getattr(self.yumbase, "clean%s" % mdtype)()[1][0] - if not msg.startswith("0 "): - self.logger.info(msg) - - @acquire_lock - def populate_cache(self): - """ populate the yum cache """ - for repo in self.yumbase.repos.findRepos('*'): - repo.metadata_expire = 0 - repo.mdpolicy = "group:all" - self.yumbase.doRepoSetup() - self.yumbase.repos.doSetup() - for repo in self.yumbase.repos.listEnabled(): - # this populates the cache as a side effect - repo.repoXML # pylint: disable=W0104 - try: - repo.getGroups() - except yum.Errors.RepoMDError: - pass # this repo has no groups - self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) - self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) - self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) - # this does something with the groups cache as a side effect - self.yumbase.comps # pylint: disable=W0104 - - -def main(): - parser = OptionParser() - parser.add_option("-c", "--config", help="Config file") - parser.add_option("-v", "--verbose", help="Verbosity level", - action="count") - (options, args) = parser.parse_args() - - if options.verbose: - level = logging.DEBUG - clevel = logging.DEBUG - else: - level = logging.WARNING - clevel = logging.INFO - Bcfg2.Logger.setup_logging('bcfg2-yum-helper', to_syslog=True, - to_console=clevel, level=level) - logger = logging.getLogger('bcfg2-yum-helper') - - try: - cmd = args[0] - except IndexError: - logger.error("No command given") - return 1 - - if not os.path.exists(options.config): - logger.error("Config file %s not found" % options.config) - return 1 - - # pylint: disable=W0702 - rv = 0 - if cmd == "clean": - cachemgr = CacheManager(options.config, options.verbose) - try: - cachemgr.clean_cache() - print(json.dumps(True)) - except: - logger.error("Unexpected error cleaning cache: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(False)) - rv = 2 - elif cmd == "makecache": - cachemgr = CacheManager(options.config, options.verbose) - try: - # this code copied from yumcommands.py - cachemgr.populate_cache() - print(json.dumps(True)) - except yum.Errors.YumBaseError: - logger.error("Unexpected error creating cache: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(False)) - elif cmd == "complete": - depsolver = DepSolver(options.config, options.verbose) - try: - data = json.loads(sys.stdin.read()) - except: - logger.error("Unexpected error decoding JSON input: %s" % - sys.exc_info()[1]) - rv = 2 - try: - depsolver.groups = data['groups'] - (packages, unknown) = depsolver.complete( - [pkg_to_tuple(p) for p in data['packages']]) - print(json.dumps(dict(packages=list(packages), - unknown=list(unknown)))) - except: - logger.error("Unexpected error completing package set: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(dict(packages=[], unknown=data['packages']))) - rv = 2 - elif cmd == "get_groups": - depsolver = DepSolver(options.config, options.verbose) - try: - data = json.loads(sys.stdin.read()) - rv = dict() - for gdata in data: - if "type" in gdata: - packages = depsolver.get_group(gdata['group'], - ptype=gdata['type']) - else: - packages = depsolver.get_group(gdata['group']) - rv[gdata['group']] = list(packages) - print(json.dumps(rv)) - except: - logger.error("Unexpected error getting groups: %s" % - sys.exc_info()[1], exc_info=1) - print(json.dumps(dict())) - rv = 2 - else: - logger.error("Unknown command %s" % cmd) - print(json.dumps(None)) - rv = 2 - return rv +from Bcfg2.Server.Plugins.Packages.YumHelper import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 87b63a6ab..17fac4fe4 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -70,9 +70,9 @@ no_checks = { "lib/Bcfg2/Server/Reports": ["manage.py"], "lib/Bcfg2/Server/Plugins": ["Base.py"], } -if sys.version_info > (2, 5): +if sys.version_info < (2, 6): # multiprocessing core requires py2.6 - no_checks['lib/Bcfg2/Server'].append('MultiprocessingCore.py') + no_checks['lib/Bcfg2/Server'] = ['MultiprocessingCore.py'] try: any diff --git a/testsuite/pylintrc.conf b/testsuite/pylintrc.conf index 653c68426..e13a51d0d 100644 --- a/testsuite/pylintrc.conf +++ b/testsuite/pylintrc.conf @@ -147,7 +147,7 @@ ignore-mixin-members=yes # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). -ignored-classes=ForeignKey,Interaction,git.cmd.Git +ignored-classes=ForeignKey,Interaction,git.cmd.Git,argparse.Namespace,Namespace # When zope mode is activated, add a predefined set of Zope acquired attributes # to generated-members. diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py deleted file mode 100755 index 2b0ca6d63..000000000 --- a/tools/bcfg2-profile-templates.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/python -Ott -# -*- coding: utf-8 -*- -""" Benchmark template rendering times """ - -import sys -import time -import math -import signal -import logging -import operator -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Core - - -def stdev(nums): - mean = float(sum(nums)) / len(nums) - return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums))) - - -def get_sigint_handler(core): - """ Get a function that handles SIGINT/Ctrl-C by shutting down the - core and exiting properly.""" - - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - core.shutdown() - os._exit(1) # pylint: disable=W0212 - - return hdlr - - -def main(): - optinfo = dict( - client=Bcfg2.Options.Option("Benchmark templates for one client", - cmd="--client", - odesc="<client>", - long_arg=True, - default=None), - runs=Bcfg2.Options.Option("Number of rendering passes per template", - cmd="--runs", - odesc="<runs>", - long_arg=True, - default=5, - cook=int)) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging("bcfg2-test", - to_console=setup['verbose'] or setup['debug'], - to_syslog=False, - to_file=setup['logging'], - level=level) - logger = logging.getLogger(sys.argv[0]) - - core = Bcfg2.Server.Core.BaseCore(setup) - signal.signal(signal.SIGINT, get_sigint_handler(core)) - logger.info("Bcfg2 server core loaded") - core.load_plugins() - logger.debug("Plugins loaded") - core.block_for_fam_events(handle_events=True) - logger.debug("Repository events processed") - - if setup['args']: - templates = setup['args'] - else: - templates = [] - - if setup['client'] is None: - clients = [core.build_metadata(c) for c in core.metadata.clients] - else: - clients = [core.build_metadata(setup['client'])] - - times = dict() - client_count = 0 - for metadata in clients: - client_count += 1 - logger.info("Rendering templates for client %s (%s/%s)" % - (metadata.hostname, client_count, len(clients))) - structs = core.GetStructures(metadata) - struct_count = 0 - for struct in structs: - struct_count += 1 - logger.info("Rendering templates from structure %s:%s (%s/%s)" % - (struct.tag, struct.get("name"), struct_count, - len(structs))) - entries = struct.xpath("//Path") - entry_count = 0 - for entry in entries: - entry_count += 1 - if templates and entry.get("name") not in templates: - continue - logger.info("Rendering Path:%s (%s/%s)..." % - (entry.get("name"), entry_count, len(entries))) - ptimes = times.setdefault(entry.get("name"), []) - for i in range(setup['runs']): - start = time.time() - try: - core.Bind(entry, metadata) - ptimes.append(time.time() - start) - except: - break - if ptimes: - avg = sum(ptimes) / len(ptimes) - if avg: - logger.debug(" %s: %.02f sec" % - (metadata.hostname, avg)) - - # print out per-file results - tmpltimes = [] - for tmpl, ptimes in times.items(): - try: - mean = float(sum(ptimes)) / len(ptimes) - except ZeroDivisionError: - continue - ptimes.sort() - median = ptimes[len(ptimes) / 2] - std = stdev(ptimes) - if mean > 0.01 or median > 0.01 or std > 1 or templates: - tmpltimes.append((tmpl, mean, median, std)) - print("%-50s %-9s %-11s %6s" % - ("Template", "Mean Time", "Median Time", "σ")) - for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): - print("%-50s %9.02f %11.02f %6.02f" % info) - core.shutdown() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index 3c90a3ea5..21b5ad8d4 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -6,19 +6,19 @@ the server core, then uses that to get probes, run them, and so on.""" import sys import socket import Bcfg2.Options -from Bcfg2.Client.Client import Client -from Bcfg2.Server.Core import BaseCore +from Bcfg2.Client import Client +from Bcfg2.Server.Core import Core -class LocalCore(BaseCore): +class LocalCore(Core): """ Local server core similar to the one started by bcfg2-info """ - def __init__(self, setup): - saved = (setup['syslog'], setup['logging']) - setup['syslog'] = False - setup['logging'] = None - Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) - setup['syslog'], setup['logging'] = saved + def __init__(self): + #saved = (setup['syslog'], setup['logging']) + #setup['syslog'] = False + #setup['logging'] = None + Bcfg2.Server.Core.BaseCore.__init__(self) + #setup['syslog'], setup['logging'] = saved self.load_plugins() self.block_for_fam_events(handle_events=True) @@ -57,26 +57,22 @@ class LocalClient(Client): """ A version of the Client class that uses LocalProxy instead of an XML-RPC proxy to make its calls """ - def __init__(self, setup, proxy): - Client.__init__(self, setup) + def __init__(self, proxy): + Client.__init__(self) self._proxy = proxy def main(): - optinfo = Bcfg2.Options.CLIENT_COMMON_OPTIONS - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - if 'bundle_quick' in optinfo: - # CLIENT_BUNDLEQUICK option uses -Q, just like the server repo - # option. the server repo is more important for this - # application. - optinfo['bundle_quick'] = Bcfg2.Options.Option('bundlequick', - default=False) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - core = LocalCore(setup) + parser = Bcfg2.Options.Parser( + description="Run a Bcfg2 client against a local repository without a " + "server", + conflict_handler="resolve", + components=[LocalCore, LocalProxy, LocalClient]) + parser.parse() + + core = LocalCore() try: - LocalClient(setup, LocalProxy(core)).run() + LocalClient(LocalProxy(core)).run() finally: core.shutdown() diff --git a/tools/posixusers_baseline.py b/tools/posixusers_baseline.py index c45e54f1a..1f89c7cb6 100755 --- a/tools/posixusers_baseline.py +++ b/tools/posixusers_baseline.py @@ -2,72 +2,46 @@ import grp import sys -import logging import lxml.etree import Bcfg2.Logger +import Bcfg2.Options from Bcfg2.Client.Tools.POSIXUsers import POSIXUsers -from Bcfg2.Options import OptionParser, Option, get_bool, CLIENT_COMMON_OPTIONS -def get_setup(): - optinfo = CLIENT_COMMON_OPTIONS - optinfo['nouids'] = Option("Do not include UID numbers for users", - default=False, - cmd='--no-uids', - long_arg=True, - cook=get_bool) - optinfo['nogids'] = Option("Do not include GID numbers for groups", - default=False, - cmd='--no-gids', - long_arg=True, - cook=get_bool) - setup = OptionParser(optinfo) - setup.parse(sys.argv[1:]) +class CLI(object): + options = [ + Bcfg2.Options.BooleanOption( + "--no-uids", help="Do not include UID numbers for users"), + Bcfg2.Options.BooleanOption( + "--no-gids", help="Do not include GID numbers for groups")] - if setup['args']: - print("posixuser_[baseline.py takes no arguments, only options") - print(setup.buildHelpMessage()) - raise SystemExit(1) - level = 30 - if setup['verbose']: - level = 20 - if setup['debug']: - level = 0 - Bcfg2.Logger.setup_logging('posixusers_baseline.py', - to_syslog=False, - level=level, - to_file=setup['logging']) - return setup - - -def main(): - setup = get_setup() - if setup['file']: - config = lxml.etree.parse(setup['file']).getroot() - else: + def __init__(self): + Bcfg2.Options.get_parser( + description="Generate a bundle with a baseline of POSIX users and " + "groups", + components=[self, POSIXUsers]).parse() config = lxml.etree.Element("Configuration") - users = POSIXUsers(logging.getLogger('posixusers_baseline.py'), - setup, config) - - baseline = lxml.etree.Element("Bundle", name="posixusers_baseline") - for entry in users.FindExtra(): - data = users.existing[entry.tag][entry.get("name")] - for attr, idx in users.attr_mapping[entry.tag].items(): - if (entry.get(attr) or - (attr == 'uid' and setup['nouids']) or - (attr == 'gid' and setup['nogids'])): - continue - entry.set(attr, str(data[idx])) - if entry.tag == 'POSIXUser': - entry.set("group", grp.getgrgid(data[3])[0]) - for group in users.user_supplementary_groups(entry): - memberof = lxml.etree.SubElement(entry, "MemberOf", - group=group[0]) - - entry.tag = "Bound" + entry.tag - baseline.append(entry) - - print(lxml.etree.tostring(baseline, pretty_print=True)) + self.users = POSIXUsers(config) + + def run(self): + baseline = lxml.etree.Element("Bundle", name="posixusers_baseline") + for entry in self.users.FindExtra(): + data = self.users.existing[entry.tag][entry.get("name")] + for attr, idx in self.users.attr_mapping[entry.tag].items(): + if (entry.get(attr) or + (attr == 'uid' and Bcfg2.Options.setup.no_uids) or + (attr == 'gid' and Bcfg2.Options.setup.no_gids)): + continue + entry.set(attr, str(data[idx])) + if entry.tag == 'POSIXUser': + entry.set("group", grp.getgrgid(data[3])[0]) + for group in self.users.user_supplementary_groups(entry): + lxml.etree.SubElement(entry, "MemberOf", group=group[0]) + + entry.tag = "Bound" + entry.tag + baseline.append(entry) + + print(lxml.etree.tostring(baseline, pretty_print=True)) if __name__ == "__main__": - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py index 507a16f43..ad2a40426 100755 --- a/tools/selinux_baseline.py +++ b/tools/selinux_baseline.py @@ -1,41 +1,18 @@ #!/usr/bin/env python import sys -import logging import lxml.etree - import Bcfg2.Logger import Bcfg2.Options -from Bcfg2.Client.Tools.SELinux import * - -LOGGER = None - -def get_setup(): - global LOGGER - optinfo = Bcfg2.Options.CLIENT_COMMON_OPTIONS - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) +from Bcfg2.Client.Tools.SELinux import SELinux - if setup['args']: - print("selinux_baseline.py takes no arguments, only options") - print(setup.buildHelpMessage()) - raise SystemExit(1) - level = 30 - if setup['verbose']: - level = 20 - if setup['debug']: - level = 0 - Bcfg2.Logger.setup_logging('selinux_base', - to_syslog=False, - level=level, - to_file=setup['logging']) - LOGGER = logging.getLogger('bcfg2') - return setup def main(): - setup = get_setup() + Bcfg2.Options.get_parser( + description="Get a baseline bundle of SELinux entries", + components=[SELinux]).parse() config = lxml.etree.Element("Configuration") - selinux = SELinux(LOGGER, setup, config) + selinux = SELinux(config) baseline = lxml.etree.Element("Bundle", name="selinux_baseline") for etype, handler in selinux.handlers.items(): diff --git a/tools/upgrade/1.1/posixunified.py b/tools/upgrade/1.1/posixunified.py index 8eb4ed734..b6ce7bc90 100644..100755 --- a/tools/upgrade/1.1/posixunified.py +++ b/tools/upgrade/1.1/posixunified.py @@ -17,12 +17,13 @@ NOTE: This script takes a conservative approach when it comes to """ if __name__ == '__main__': - opts = { - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - } - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) - repo = setup['repo'] + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.0-style POSIX entries to 1.1-style " + "unified Path entries") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + repo = Bcfg2.Options.setup.repository unifiedposixrules = "%s/Rules/unified-rules.xml" % repo rulesroot = lxml.etree.Element("Rules") diff --git a/tools/upgrade/1.2/nagiosgen-convert.py b/tools/upgrade/1.2/nagiosgen-convert.py index 2c2142735..eb10cd4ea 100755 --- a/tools/upgrade/1.2/nagiosgen-convert.py +++ b/tools/upgrade/1.2/nagiosgen-convert.py @@ -7,10 +7,13 @@ import lxml.etree import Bcfg2.Options def main(): - opts = {'repo': Bcfg2.Options.SERVER_REPOSITORY} - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) - repo = setup['repo'] + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.1-style Properties-based NagiosGen " + "configuration to standalone 1.2-style") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + repo = Bcfg2.Options.setup.repository oldconfigfile = os.path.join(repo, 'Properties', 'NagiosGen.xml') newconfigpath = os.path.join(repo, 'NagiosGen') newconfigfile = os.path.join(newconfigpath, 'config.xml') @@ -32,11 +35,11 @@ def main(): if host.tag == lxml.etree.Comment: # skip comments continue - + if host.tag == 'default': print("default tag will not be converted; use a suitable Group tag instead") continue - + newhost = lxml.etree.Element("Client", name=host.tag) for opt in host: newopt = lxml.etree.Element("Option", name=opt.tag) diff --git a/tools/upgrade/1.2/packages-convert.py b/tools/upgrade/1.2/packages-convert.py index d65ce90a2..eb1f2f7de 100755 --- a/tools/upgrade/1.2/packages-convert.py +++ b/tools/upgrade/1.2/packages-convert.py @@ -30,10 +30,13 @@ def place_source(xdata, source, groups): return xdata def main(): - opts = {'repo': Bcfg2.Options.SERVER_REPOSITORY} - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) - repo = setup['repo'] + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.1-style Packages configuration to " + "1.2-style") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + repo = Bcfg2.Options.setup.repository configpath = os.path.join(repo, 'Packages') oldconfigfile = os.path.join(configpath, 'config.xml') newconfigfile = os.path.join(configpath, 'packages.conf') @@ -78,7 +81,7 @@ def main(): if el.tag == lxml.etree.Comment or el.tag == 'Config': # skip comments and Config continue - + if el.tag == XI + 'include': oldsources.append(os.path.join(configpath, el.get('href'))) newsource.append(el) @@ -98,7 +101,7 @@ def main(): newel.set(tag.lower(), el.find(tag).text) except AttributeError: pass - + for child in el.getchildren(): if child.tag in ['Component', 'Blacklist', 'Whitelist', 'Arch']: newel.append(child) diff --git a/tools/upgrade/1.3/migrate_configs.py b/tools/upgrade/1.3/migrate_configs.py index b7adb2528..9fa362acf 100755 --- a/tools/upgrade/1.3/migrate_configs.py +++ b/tools/upgrade/1.3/migrate_configs.py @@ -16,13 +16,13 @@ def copy_section(src_file, tgt_cfg, section, newsection=None): tgt_cfg.add_section(newsection) except ConfigParser.DuplicateSectionError: print("[%s] section already exists in %s, adding options" % - (newsection, setup['cfile'])) + (newsection, Bcfg2.Options.setup.config)) for opt in cfg.options(section): val = cfg.get(section, opt) if tgt_cfg.has_option(newsection, opt): print("%s in [%s] already populated in %s, skipping" % - (opt, newsection, setup['cfile'])) - print(" %s: %s" % (setup['cfile'], + (opt, newsection, Bcfg2.Options.setup.config)) + print(" %s: %s" % (Bcfg2.Options.setup.config, tgt_cfg.get(newsection, opt))) print(" %s: %s" % (src_file, val)) else: @@ -30,47 +30,50 @@ def copy_section(src_file, tgt_cfg, section, newsection=None): tgt_cfg.set(newsection, opt, val) def main(): - opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY, - configfile=Bcfg2.Options.CFILE) - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.2 per-plugin config files to 1.3 " + "unified config file") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + repo = Bcfg2.Options.setup.repository + cfp = ConfigParser.ConfigParser() + cfp.read(Bcfg2.Options.setup.config) # files that you should remove manually remove = [] # move rules config out of rules.conf and into bcfg2.conf - rules_conf = os.path.join(setup['repo'], 'Rules', 'rules.conf') + rules_conf = os.path.join(repo, 'Rules', 'rules.conf') if os.path.exists(rules_conf): remove.append(rules_conf) - copy_section(rules_conf, setup.cfp, "rules") - + copy_section(rules_conf, cfp, "rules") + # move packages config out of packages.conf and into bcfg2.conf - pkgs_conf = os.path.join(setup['repo'], 'Packages', 'packages.conf') + pkgs_conf = os.path.join(repo, 'Packages', 'packages.conf') if os.path.exists(pkgs_conf): remove.append(pkgs_conf) - copy_section(pkgs_conf, setup.cfp, "global", newsection="packages") + copy_section(pkgs_conf, cfp, "global", newsection="packages") for section in ["apt", "yum", "pulp"]: - copy_section(pkgs_conf, setup.cfp, section, + copy_section(pkgs_conf, cfp, section, newsection="packages:" + section) # move reports database config into [database] section - if setup.cfp.has_section("statistics"): - if not setup.cfp.has_section("database"): - setup.cfp.add_section("database") - for opt in setup.cfp.options("statistics"): + if cfp.has_section("statistics"): + if not cfp.has_section("database"): + cfp.add_section("database") + for opt in cfp.options("statistics"): if opt.startswith("database_"): newopt = opt[9:] - if setup.cfp.has_option("database", newopt): + if cfp.has_option("database", newopt): print("%s in [database] already populated, skipping" % newopt) else: - setup.cfp.set("database", newopt, - setup.cfp.get("statistics", opt)) - setup.cfp.remove_option("statistics", opt) + cfp.set("database", newopt, cfp.get("statistics", opt)) + cfp.remove_option("statistics", opt) - print("Writing %s" % setup['configfile']) + print("Writing %s" % Bcfg2.Options.setup.config) try: - setup.cfp.write(open(setup['configfile'], "w")) + cfp.write(open(Bcfg2.Options.setup.config, "w")) if len(remove): print("Settings were migrated, but you must remove these files " "manually:") @@ -78,7 +81,7 @@ def main(): print(" %s" % path) except IOError: err = sys.exc_info()[1] - print("Could not write %s: %s" % (setup['configfile'], err)) + print("Could not write %s: %s" % (Bcfg2.Options.setup.config, err)) if __name__ == '__main__': sys.exit(main()) diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py index 07def2ac8..f52ccab08 100755 --- a/tools/upgrade/1.3/migrate_dbstats.py +++ b/tools/upgrade/1.3/migrate_dbstats.py @@ -9,10 +9,9 @@ import logging import time import Bcfg2.Logger import Bcfg2.Options -from django.core.cache import cache from django.db import connection, transaction, backend -from Bcfg2.Server.Admin.Reports import Reports +from Bcfg2.Server.Admin import UpdateReports from Bcfg2.Reporting import models as new_models from Bcfg2.Reporting.utils import BatchFetch from Bcfg2.Server.Reports.reports import models as legacy_models @@ -281,17 +280,10 @@ def _restructure(): if __name__ == '__main__': - Bcfg2.Logger.setup_logging('bcfg2-report-collector', - to_console=logging.INFO, - level=logging.INFO) - - optinfo = dict() - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - #sync! - Reports(setup).__call__(['update']) + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.2 DBStats plugin to 1.3 Reporting " + "subsystem", + components=[UpdateReports]) + UpdateReports().run(Bcfg2.Options.setup) _restructure() diff --git a/tools/upgrade/1.3/migrate_info.py b/tools/upgrade/1.3/migrate_info.py index 3ccbf0285..7f3bb9a29 100755 --- a/tools/upgrade/1.3/migrate_info.py +++ b/tools/upgrade/1.3/migrate_info.py @@ -5,7 +5,16 @@ import re import sys import lxml.etree import Bcfg2.Options -from Bcfg2.Server.Plugin import INFO_REGEX + +INFO_REGEX = re.compile(r'owner:\s*(?P<owner>\S+)|' + + r'group:\s*(?P<group>\S+)|' + + r'mode:\s*(?P<mode>\w+)|' + + r'secontext:\s*(?P<secontext>\S+)|' + + r'paranoid:\s*(?P<paranoid>\S+)|' + + r'sensitive:\s*(?P<sensitive>\S+)|' + + r'encoding:\s*(?P<encoding>\S+)|' + + r'important:\s*(?P<important>\S+)|' + + r'mtime:\s*(?P<mtime>\w+)') PERMS_REGEX = re.compile(r'perms:\s*(?P<perms>\w+)') @@ -32,16 +41,17 @@ def convert(info_file): def main(): - opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY, - configfile=Bcfg2.Options.CFILE, - plugins=Bcfg2.Options.SERVER_PLUGINS) - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.2 info/:info files to 1.3 info.xml") + parser.add_options([Bcfg2.Options.Common.repository, + Bcfg2.Options.Common.plugins]) + parser.parse() - for plugin in setup['plugins']: + for plugin in Bcfg2.Options.setup.plugins: if plugin not in ['SSLCA', 'Cfg', 'TGenshi', 'TCheetah', 'SSHbase']: continue - for root, dirs, files in os.walk(os.path.join(setup['repo'], plugin)): + datastore = os.path.join(Bcfg2.Options.setup.repository, plugin) + for root, dirs, files in os.walk(datastore): for fname in files: if fname in [":info", "info"]: convert(os.path.join(root, fname)) diff --git a/tools/upgrade/1.3/migrate_perms_to_mode.py b/tools/upgrade/1.3/migrate_perms_to_mode.py index 18abffec2..786df0de6 100755 --- a/tools/upgrade/1.3/migrate_perms_to_mode.py +++ b/tools/upgrade/1.3/migrate_perms_to_mode.py @@ -54,16 +54,17 @@ def convertstructure(structfile): def main(): - opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY, - configfile=Bcfg2.Options.CFILE, - plugins=Bcfg2.Options.SERVER_PLUGINS) - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) - repo = setup['repo'] + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.2 'perms' attribute to 1.3 'mode' " + "attribute") + parser.add_options([Bcfg2.Options.Common.repository, + Bcfg2.Options.Common.plugins]) + parser.parse() + repo = Bcfg2.Options.setup.repository - for plugin in setup['plugins']: + for plugin in Bcfg2.Options.setup.plugins: if plugin in ['Base', 'Bundler', 'Rules']: - for root, dirs, files in os.walk(os.path.join(repo, plugin)): + for root, _, files in os.walk(os.path.join(repo, plugin)): for fname in files: convertstructure(os.path.join(root, fname)) if plugin not in ['Cfg', 'TGenshi', 'TCheetah', 'SSHbase', 'SSLCA']: diff --git a/tools/upgrade/1.3/service_modes.py b/tools/upgrade/1.3/service_modes.py index 0c458c3a9..d8e3c9e6f 100755 --- a/tools/upgrade/1.3/service_modes.py +++ b/tools/upgrade/1.3/service_modes.py @@ -6,14 +6,18 @@ import glob import lxml.etree import Bcfg2.Options + def main(): - opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY) - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.2 Service modes to 1.3-style " + "granular Service specification") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() files = [] for plugin in ['Bundler', 'Rules', 'Default']: - files.extend(glob.glob(os.path.join(setup['repo'], plugin, "*"))) + files.extend(glob.glob(os.path.join(Bcfg2.Options.setup.repository, + plugin, "*"))) for bfile in files: bdata = lxml.etree.parse(bfile) diff --git a/tools/upgrade/1.4/README b/tools/upgrade/1.4/README index b6ff8d8c8..58786966b 100644 --- a/tools/upgrade/1.4/README +++ b/tools/upgrade/1.4/README @@ -4,3 +4,6 @@ to 1.4. migrate_decisions.py - Convert old group- and host-specific whitelist and blacklist files into structured XML + +remove_bundle_names.py + - Remove deprecated explicit bundle names diff --git a/tools/upgrade/1.4/convert_bundles.py b/tools/upgrade/1.4/convert_bundles.py new file mode 100755 index 000000000..b9cb483f2 --- /dev/null +++ b/tools/upgrade/1.4/convert_bundles.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import os +import sys +import lxml.etree +import Bcfg2.Options + + +def main(): + parser = Bcfg2.Options.get_parser("Tool to remove bundle names") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + bundler_dir = os.path.join(Bcfg2.Options.setup.repository, "Bundler") + if os.path.exists(bundler_dir): + for root, _, files in os.walk(bundler_dir): + for fname in files: + bpath = os.path.join(root, fname) + newpath = bpath + if newpath.endswith(".genshi"): + newpath = newpath[:-6] + "xml" + print("Converting %s to %s" % (bpath, newpath)) + else: + print("Converting %s" % bpath) + xroot = lxml.etree.parse(bpath) + xdata = xroot.getroot() + if 'name' in xdata.attrib: + del xdata.attrib['name'] + xroot.write(bpath) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/upgrade/1.4/migrate_decisions.py b/tools/upgrade/1.4/migrate_decisions.py index f7072783a..d0915f202 100755 --- a/tools/upgrade/1.4/migrate_decisions.py +++ b/tools/upgrade/1.4/migrate_decisions.py @@ -6,7 +6,6 @@ import sys import glob import lxml.etree import Bcfg2.Options -from Bcfg2.Server import XMLParser SPECIFIC = re.compile(r'.*\/(white|black)list' @@ -56,12 +55,13 @@ def convert(files, xdata): def main(): - opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY, - configfile=Bcfg2.Options.CFILE) - setup = Bcfg2.Options.load_option_parser(opts) - setup.parse(sys.argv[1:]) + parser = Bcfg2.Options.get_parser( + description="Migrate from Bcfg2 1.3 Decisions list format to 1.4 " + "format") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() - datadir = os.path.join(setup['repo'], 'Decisions') + datadir = os.path.join(Bcfg2.Options.setup.repository, 'Decisions') whitelist = lxml.etree.Element("Decisions") blacklist = lxml.etree.Element("Decisions") if os.path.exists(datadir): |