diff options
74 files changed, 2366 insertions, 589 deletions
diff --git a/.travis.yml b/.travis.yml index 54f2215de..8b336e7f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - "2.6" - - "2.7" + - "2.7_with_system_site_packages" env: - WITH_OPTIONAL_DEPS=yes - WITH_OPTIONAL_DEPS=no diff --git a/debian/bcfg2-server.init b/debian/bcfg2-server.init index b1c3aba21..cd2ad858e 100755 --- a/debian/bcfg2-server.init +++ b/debian/bcfg2-server.init @@ -33,8 +33,8 @@ BCFG2_SERVER_ENABLED=0 test -f "/etc/default/bcfg2-server" && . /etc/default/bcfg2-server if [ "$BCFG2_SERVER_ENABLED" -eq 0 ] ; then - log_failure_msg "bcfg2-server is disabled - see /etc/default/bcfg2-server" - exit 0 + log_failure_msg "bcfg2-server is disabled - see /etc/default/bcfg2-server" + exit 0 fi # Exit if $DAEMON doesn't exist and is not executable @@ -63,10 +63,11 @@ stop () { killproc -p $PIDFILE ${BINARY} STATUS=$? if [ "$STATUS" = 0 ]; then - log_success_msg "bcfg2-server" - test -d /var/lock/subsys && touch /var/lock/subsys/bcfg2-server + [ -e $PIDFILE ] && rm -f $PIDFILE + log_success_msg "bcfg2-server" + test -d /var/lock/subsys && touch /var/lock/subsys/bcfg2-server else - log_failure_msg "bcfg2-server" + log_failure_msg "bcfg2-server" fi return $STATUS } @@ -75,15 +76,15 @@ status () { # Inspired by redhat /etc/init.d/functions status() call PID=$(pidof -x $BINARY -o %PPID) if [ -n "$PID" ]; then - echo "$BINARY (pid $PID) is running..." - return 0 + echo "$BINARY (pid $PID) is running..." + return 0 fi if [ -f $PIDFILE ]; then - if [ -n "$PID" ]; then - log_failure_msg "$BINARY dead but pid file exists..." - return 1 - fi + if [ -n "$PID" ]; then + log_failure_msg "$BINARY dead but pid file exists..." + return 1 + fi fi log_failure_msg "$BINARY is not running" diff --git a/debian/changelog b/debian/changelog index e30fba546..c41b2ecc2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,12 @@ bcfg2 (1.4.0pre1-0.0) unstable; urgency=low -- Sol Jerome <sol.jerome@gmail.com> Mon, 16 Jun 2014 09:36:13 -0500 +bcfg2 (1.3.5-0.0) unstable; urgency=low + + * New upstream release + + -- Sol Jerome <sol.jerome@gmail.com> Fri, 05 Sep 2014 07:54:48 -0500 + bcfg2 (1.3.4-0.0) unstable; urgency=low * New upstream release diff --git a/debian/control b/debian/control index 8eb4343c7..452fa3d5f 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Homepage: http://bcfg2.org/ Package: bcfg2 Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, debsums, python-apt, ucf, lsb-base (>= 3.1-9), python (>= 2.6) +Depends: ${python:Depends}, ${misc:Depends}, debsums, python-apt (>= 0.7.91), ucf, lsb-base (>= 3.1-9), python (>= 2.6) Description: Configuration management client Bcfg2 is a configuration management system that generates configuration sets for clients bound by client profiles. diff --git a/doc/client/tools.txt b/doc/client/tools.txt index 09ea76230..93eb11925 100644 --- a/doc/client/tools.txt +++ b/doc/client/tools.txt @@ -152,7 +152,36 @@ Systemd service support. SYSV ---- -Handles System V Packaging format that is available on Solaris. +Handles `System V Packaging <http://docs.oracle.com/cd/E19683-01/806-7008/index.html>`_ format that is available on Solaris. + +.. note:: + + If the Packages specified in the PackageList are datastream format + packages distributed via HTTP, you must specify a simplefile attribute. + Such packages will be downloaded and installed from a local path. + + Note the use of the uri attribute in the datastream format example. If + the simplefile attribute exists, the + :ref:`Pkgmgr <server-plugins-generators-pkgmgr>` plugin will + automatically construct the url attribute by concatenating the uri and + simplefile attributes (with an intervening slash). + + Datastream format over HTTP: + + .. code-block:: xml + + <PackageList uri='http://install/packages' type='sysv' priority='0'> + <Package name='SCbcfg2' version='1.3.4' simplefile='bcfg-1.3.4-1' /> + </PackageList> + + File system format over NFS or local path: + + .. code-block:: xml + + <PackageList url='/mnt/install/packages' type='sysv' priority='0'> + <Package name='SCbcfg2' version='1.3.4' /> + </PackageList> + Upstart ------- diff --git a/doc/development/compat.txt b/doc/development/compat.txt index f90274ce5..8700c46d3 100644 --- a/doc/development/compat.txt +++ b/doc/development/compat.txt @@ -60,6 +60,8 @@ behavior (e.g., :func:`input`) do not cause unexpected side-effects. +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | urlparse | :func:`urlparse.urlparse` | :func:`urllib.parse.urlparse` | +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ +| urlretrieve | :func:`urllib.urlretrieve` | :func:`urllib.request.urlretrieve` | ++---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | HTTPBasicAuthHandler | :class:`urllib2.HTTPBasicAuthHandler` | :class:`urllib.request.HTTPBasicAuthHandler` | +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | HTTPPasswordMgrWithDefaultRealm | :class:`urllib2.HTTPPasswordMgrWithDefaultRealm` | :class:`urllib.request.HTTPPasswordMgrWithDefaultRealm` | diff --git a/doc/development/option_parsing.txt b/doc/development/option_parsing.txt index 091f43cdd..e14031e1e 100644 --- a/doc/development/option_parsing.txt +++ b/doc/development/option_parsing.txt @@ -174,28 +174,32 @@ The normal implementation pattern is this: #. Define all of your subcommands as children of :class:`Bcfg2.Options.Subcommand`. -#. Define a :class:`Bcfg2.Options.CommandRegistry` object that will be +#. Create a :class:`Bcfg2.Options.CommandRegistry` object that will be used to register all of the commands. Registering a command collect its options and adds it as a :class:`Bcfg2.Options.Subparser` option group to the main option parser. -#. Register your commands with - :func:`Bcfg2.Options.register_commands`, parse options, and run. +#. Register your commands with the + :func:`Bcfg2.Options.CommandRegistry.register_commands` method of + your ``CommandRegistry`` object. +#. Add options from the + :attr:`Bcfg2.Options.CommandRegistry.command_options` + attribute to the option parser. +#. Parse options, and run. :mod:`Bcfg2.Server.Admin` provides a fairly simple implementation, -where the CLI class is itself the command registry: +where the CLI class subclasses the command registry: .. code-block:: python class CLI(Bcfg2.Options.CommandRegistry): def __init__(self): Bcfg2.Options.CommandRegistry.__init__(self) - Bcfg2.Options.register_commands(self.__class__, - globals().values(), - parent=AdminCmd) + self.register_commands(globals().values(), parent=AdminCmd) parser = Bcfg2.Options.get_parser( description="Manage a running Bcfg2 server", components=[self]) + parser.add_options(self.subcommand_options) parser.parse() In this case, commands are collected from amongst all global variables @@ -208,9 +212,7 @@ At a minimum, the :func:`Bcfg2.Options.Subcommand.run` method must be overridden, and a docstring written. .. autoclass:: Subcommand -.. autoclass:: HelpCommand .. autoclass:: CommandRegistry -.. autofunction:: register_commands Actions ------- diff --git a/doc/installation/distributions.txt b/doc/installation/distributions.txt index 306439485..5dad4d860 100644 --- a/doc/installation/distributions.txt +++ b/doc/installation/distributions.txt @@ -36,9 +36,9 @@ Just use `pacman` to perform the installation :: Debian ====== -Packages of Bcfg2 are available for Debian Lenny, Debian Squeeze, and -Debian Sid. The fastest way to get Bcfg2 onto your Debian system -is to use ``apt-get`` or ``aptitude``. :: +Packages of Bcfg2 are available for all current versions of Debian. +The fastest way to get Bcfg2 onto your Debian system is to use ``apt-get`` +or ``aptitude``. :: sudo aptitude install bcfg2 bcfg2-server diff --git a/doc/installation/prerequisites.txt b/doc/installation/prerequisites.txt index 8119be06b..d89d44894 100644 --- a/doc/installation/prerequisites.txt +++ b/doc/installation/prerequisites.txt @@ -25,7 +25,7 @@ Bcfg2 Client +----------------------------+------------------------+--------------------------------+ | lxml or elementtree [#f2]_ | Any | lxml: libxml2, libxslt, python | +----------------------------+------------------------+--------------------------------+ -| python-apt [#f3]_ | Any | python | +| python-apt [#f3]_ | 0.7.91 and greater | python | +----------------------------+------------------------+--------------------------------+ | debsums (if APT tool | Any | | | driver is used) | | | diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index 825ab2121..62c4ac1a8 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -662,9 +662,8 @@ control the database connection of the server. Port for database connections. Not used for sqlite3. options - Various options for the database connection. The value is - expected as multiple key=value pairs, separated with commas. - The concrete value depends on the database engine. + Various options for the database connection. The value expected + is the literal value of the django OPTIONS setting. reporting_engine The database engine used by the Reporting plugin. One of the @@ -697,9 +696,8 @@ control the database connection of the server. Port for reporting database connections. Not used for sqlite3. reporting_options - Various options for the reporting database connection. The value - is expected as multiple key=value pairs, separated with commas. - The concrete value depends on the database engine. + Various options for the database connection. The value expected + is the literal value of the django OPTIONS setting. Reporting options ----------------- @@ -716,6 +714,10 @@ Reporting options web_debug Turn on Django debugging. + max_children + Maximum number of children for the reporting collector. Use 0 to + disable the limit. (default is 0) + See Also -------- diff --git a/doc/releases/1.3.5.txt b/doc/releases/1.3.5.txt new file mode 100644 index 000000000..893cdbf39 --- /dev/null +++ b/doc/releases/1.3.5.txt @@ -0,0 +1,33 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.3.5: + +1.3.5 +===== + +We are happy to announce the release of Bcfg2 1.3.5. It is available for +download at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +This is primarily a bugfix release. + +* Properly close db connections +* Improved error messages +* Fix yum upgrade/downgrade +* Enable bcfg2-yum-helper to depsolve for arches incompatible with + server +* Spec file fixes +* bcfg2-crypet: Default to only (En|De)crypt vars that need it +* Fix email reporting bug +* Fix debsums parsing +* Fix solaris makefile +* SYSV: Implement downloading and installing SYSV packages from HTTP: + http://docs.bcfg2.org/client/tools.html#sysv +* Fix debian bcfg2-server init script + + +Special thanks to the following contributors for this release: John +Morris, Jonathan Billings, Chris Brinker, Tim Laszlo, Matt Kemp, Michael +Fenn, Pavel Labushev, Nathan Olla, Alexander Sulfrian. diff --git a/doc/releases/1.3.6.txt b/doc/releases/1.3.6.txt new file mode 100644 index 000000000..757fbf6f5 --- /dev/null +++ b/doc/releases/1.3.6.txt @@ -0,0 +1,34 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + +.. _releases-1.3.6: + +1.3.6 +===== + +We are happy to announce the release of Bcfg2 1.3.6. It is available for +download at: + + ftp://ftp.mcs.anl.gov/pub/bcfg + +This is primarily a bugfix release. + +* Fix python 2.4 compatibility +* Fix stale lockfile detection and behavior +* Reporting: fix filter urls +* Fix client protocol option handling +* YUM: Add options to enable and disable Yum plugins +* Packages: add name to sources +* Reporting: better exception handling +* Various interrupt handling fixes +* Fix client decision whitelist/blacklist handling +* Fix database OPTIONS parsing + + This change requires you to set the *options* value of the + ``[database`` section in ``bcfg2.conf`` to the literal value which is + passed through to the django OPTIONS setting. + + https://docs.djangoproject.com/en/1.7/ref/settings/#std:setting-OPTIONS + +Special thanks to the following contributors for this release: Michael +Fenn, Matt Kemp, Alexander Sulfrian, Jonathan Billings. diff --git a/doc/releases/index.txt b/doc/releases/index.txt index 479aa19de..271fc23cc 100644 --- a/doc/releases/index.txt +++ b/doc/releases/index.txt @@ -10,4 +10,5 @@ Release Announcements .. toctree:: 1.4.0pre1 + 1.3.5 1.3.4 diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 53bdef24e..38d4c7e3a 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -270,6 +270,9 @@ reporting * web_prefix: Prefix to be added to Django's MEDIA_URL * file_limit: The maximum size of a diff or binary data to store in the database. +* max_children: Maximum number of children for the reporting + collector. Use 0 to disable the limit and spawn a thread + as soon as a working file is available. .. _dynamic_transports: diff --git a/doc/server/database.txt b/doc/server/database.txt index 986914171..c90dcb710 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -69,8 +69,8 @@ of ``/etc/bcfg2.conf``. +--------------------+------------------------------------------------------------+---------------------------------------+ | options | Extra parameters to use when connecting to the database. | None | | | Available parameters vary depending on your database | | -| | backend. The parameters are supplied as comma separated | | -| | key=value pairs. | | +| | backend. The parameters are supplied as the value of the | | +| | django OPTIONS setting. | | +--------------------+------------------------------------------------------------+---------------------------------------+ | reporting_engine | The name of the Django database backend to use for the | None | | | reporting database. Takes the same values as ``engine``. | | @@ -86,9 +86,9 @@ of ``/etc/bcfg2.conf``. | reporting_port | The port to connect to for the reporting database | None | +--------------------+------------------------------------------------------------+---------------------------------------+ | reporting_options | Extra parameters to use when connecting to the reporting | None | -| | database. Available parameters vary depending on your | | -| | database backend. The parameters are supplied as comma | | -| | separated key=value pairs. | | +| | database. Available parameters vary depending on your | | +| | database backend. The parameters are supplied as the | | +| | value of the django OPTIONS setting. | | +--------------------+------------------------------------------------------------+---------------------------------------+ diff --git a/doc/server/plugins/connectors/awstags.txt b/doc/server/plugins/connectors/awstags.txt index b884ca065..5ac2fbc28 100644 --- a/doc/server/plugins/connectors/awstags.txt +++ b/doc/server/plugins/connectors/awstags.txt @@ -7,8 +7,8 @@ ========= The AWSTags plugin is a connector that retrieves tags from instances -in EC2, and can assign optionally assign -group membership pased on patterns in the tags. See `Using Tags +in EC2, and can optionally assign group membership based on patterns +in the tags. See `Using Tags <http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html>`_ for details on using tags in EC2. diff --git a/doc/server/plugins/connectors/templatehelper.txt b/doc/server/plugins/connectors/templatehelper.txt index c719f93cb..d113dcab7 100644 --- a/doc/server/plugins/connectors/templatehelper.txt +++ b/doc/server/plugins/connectors/templatehelper.txt @@ -56,7 +56,7 @@ Usage Specific helpers can be referred to in templates as ``metadata.TemplateHelper[<modulename>]``. That accesses -a HelperModule object will has, as attributes, all symbols listed in +a HelperModule object will have, as attributes, all symbols listed in ``__export__``. For example, consider this helper module:: __export__ = ["hello"] diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index 8b49e244b..c991f20c9 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -321,15 +321,6 @@ processed for any Genshi, Cheetah, or Jinja2 base file. Cfg/etc/fstab/fstab.H_host.example.com.genshi Cfg/etc/fstab/fstab.G50_server.cheetah -Genshi templates take precence over cheetah templates. For example, if -two files exist named:: - - Cfg/etc/fstab/fstab.genshi - Cfg/etc/fstab/fstab.cheetah - -The Cheetah template is ignored. Exploiting this fact is probably a -pretty bad idea in practice. - You can mix Genshi and Cheetah when using different host-specific or group-specific files. For example:: diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 2772fe78d..13000a719 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,6 @@ -.TH "BCFG2.CONF" "5" "April 06, 2014" "1.3" "Bcfg2" +.\" Man page generated from reStructuredText. +. +.TH "BCFG2.CONF" "5" "November 04, 2014" "1.4" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -28,8 +30,6 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.\" Man page generated from reStructuredText. -. .SH DESCRIPTION .sp bcfg2.conf includes configuration parameters for the Bcfg2 server and @@ -107,7 +107,7 @@ SCCS .B listen_all This setting tells the server to listen on all available interfaces. The default is to only listen on those interfaces specified by the -bcfg2 setting in the components section of \fBbcfg2.conf\fP. +bcfg2 setting in the components section of \fBbcfg2.conf\fP\&. .TP .B plugins A comma\-delimited list of enabled server plugins. Currently @@ -179,24 +179,24 @@ best .UNINDENT .UNINDENT .sp -The default is \fIbest\fP, which is currently an alias for \fIbuiltin\fP. +The default is \fIbest\fP, which is currently an alias for \fIbuiltin\fP\&. More details on the backends can be found in the official documentation. .TP .B user -The username or UID to run the daemon as. Default is \fI0\fP. +The username or UID to run the daemon as. Default is \fI0\fP\&. .TP .B group -The group name or GID to run the daemon as. Default is \fI0\fP. +The group name or GID to run the daemon as. Default is \fI0\fP\&. .TP .B vcs_root Specifies the path to the root of the VCS working copy that holds -your Bcfg2 specification, if it is different from \fIrepository\fP. +your Bcfg2 specification, if it is different from \fIrepository\fP\&. E.g., if the VCS repository does not hold the bcfg2 data at the top level, you may need to set this option. .TP .B umask -The umask to set for the server. Default is \fI0077\fP. +The umask to set for the server. Default is \fI0077\fP\&. .UNINDENT .SH SERVER PLUGINS .sp @@ -573,7 +573,7 @@ The path at which to generate APT configs. No default. .TP .B gpg_keypath The path on the client where RPM GPG keys will be copied before -they are imported on the client. Default is \fB/etc/pki/rpm\-gpg\fP. +they are imported on the client. Default is \fB/etc/pki/rpm\-gpg\fP\&. .TP .B version Set the version attribute used when binding Packages. Default is @@ -632,7 +632,7 @@ the configuration file. .TP .B path Custom path for backups created in paranoid mode. The default is -in \fB/var/cache/bcfg2\fP. +in \fB/var/cache/bcfg2\fP\&. .TP .B max_copies Specify a maximum number of copies for the server to keep when @@ -696,7 +696,7 @@ ado_mssql .B name The name of the database to use for server data. If \(aqdatabase_engine\(aq is set to \(aqsqlite3\(aq this is a file path to -the sqlite file and defaults to \fB$REPOSITORY_DIR/etc/bcfg2.sqlite\fP. +the sqlite file and defaults to \fB$REPOSITORY_DIR/etc/bcfg2.sqlite\fP\&. .TP .B user User for database connections. Not used for sqlite3. @@ -711,9 +711,8 @@ Host for database connections. Not used for sqlite3. Port for database connections. Not used for sqlite3. .TP .B options -Various options for the database connection. The value is -expected as multiple key=value pairs, separated with commas. -The concrete value depends on the database engine. +Various options for the database connection. The value expected +is the literal value of the django OPTIONS setting. .TP .B reporting_engine The database engine used by the Reporting plugin. One of the @@ -755,8 +754,7 @@ Port for reporting database connections. Not used for sqlite3. .TP .B reporting_options Various options for the reporting database connection. The value -is expected as multiple key=value pairs, separated with commas. -The concrete value depends on the database engine. +expected is the literal value of the django OPTIONS setting. .UNINDENT .UNINDENT .UNINDENT @@ -776,6 +774,10 @@ time zone as well). .TP .B web_debug Turn on Django debugging. +.TP +.B max_children +Maximum number of children for the reporting collector. Use 0 to +disable the limit. (default is 0) .UNINDENT .UNINDENT .UNINDENT diff --git a/schemas/packages.xsd b/schemas/packages.xsd index ae7b0430a..fc5a1356c 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -173,7 +173,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute type="xsd:boolean" name="debsrc"> + <xsd:attribute type="xsd:boolean" name="debsrc" default="false"> <xsd:annotation> <xsd:documentation> Include ``deb-src`` lines in the generated APT @@ -217,6 +217,14 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:string" name="name"> + <xsd:annotation> + <xsd:documentation> + Specifiy an explicit name for the source and do not generate + it automatically. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/solaris/Makefile b/solaris/Makefile index cdf61d8f7..38cf2547d 100644 --- a/solaris/Makefile +++ b/solaris/Makefile @@ -9,7 +9,7 @@ default: clean package package: -mkdir tmp tmp/bcfg2-server tmp/bcfg2 -mkdir -p build/lib/$(PYVERSION)/site-packages - -cd ../ && PYTHONPATH=$(PYTHONPATH):$(PWD)/build/lib/python2.6/site-packages/ $(PYTHON) setup.py install --single-version-externally-managed --record=/dev/null --prefix=$(PWD)/build + -cd ../ && PYTHONPATH=$(PYTHONPATH):$(PWD)/build/lib/python$(PYVERSION)/site-packages/ $(PYTHON) setup.py install --single-version-externally-managed --record=/dev/null --prefix=$(PWD)/build #setuptools appears to use a restictive umask -chmod -R o+r build/ -cat build/bin/bcfg2 | sed -e 's!/usr/bin/python!$(PYTHON)!' > build/bin/bcfg2.new && mv build/bin/bcfg2.new build/bin/bcfg2 diff --git a/solaris/gen-prototypes.sh b/solaris/gen-prototypes.sh index 64aff9edb..b0006df16 100644 --- a/solaris/gen-prototypes.sh +++ b/solaris/gen-prototypes.sh @@ -1,6 +1,6 @@ #!/bin/sh cd build -PP="./lib/python/site-packages/" +PP="./lib/python${PYVERSION}/site-packages/" #bcfg2 echo "i pkginfo=./pkginfo.bcfg2" > ../prototype.tmp diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index cf4e7c7ea..739ba013f 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -5,6 +5,7 @@ import warnings warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) import os +import sys import apt.cache import Bcfg2.Options import Bcfg2.Client.Tools @@ -12,7 +13,7 @@ import Bcfg2.Client.Tools class APT(Bcfg2.Client.Tools.Tool): """The Debian toolset implements package and service operations - and inherits the rest from Tools.Tool. """ + and inherits the rest from Tools.Tool.""" options = Bcfg2.Client.Tools.Tool.options + [ Bcfg2.Options.PathOption( @@ -79,33 +80,30 @@ class APT(Bcfg2.Client.Tools.Tool): try: self.pkg_cache = apt.cache.Cache() except SystemError: - e = sys.exc_info()[1] - self.logger.info("Failed to initialize APT cache: %s" % e) + err = sys.exc_info()[1] + self.logger.info("Failed to initialize APT cache: %s" % err) raise Bcfg2.Client.Tools.ToolInstantiationError - self.pkg_cache.update() + try: + self.pkg_cache.update() + except apt.cache.FetchFailedException: + err = sys.exc_info()[1] + self.logger.info("Failed to update APT cache: %s" % err) self.pkg_cache = apt.cache.Cache() - if 'req_reinstall_pkgs' in dir(self.pkg_cache): - self._newapi = True - else: - self._newapi = False def FindExtra(self): """Find extra packages.""" packages = [entry.get('name') for entry in self.getSupportedEntries()] - if self._newapi: - extras = [(p.name, p.installed.version) for p in self.pkg_cache - if p.is_installed and p.name not in packages] - else: - extras = [(p.name, p.installedVersion) for p in self.pkg_cache - if p.isInstalled and p.name not in packages] + extras = [(p.name, p.installed.version) for p in self.pkg_cache + if p.is_installed and p.name not in packages] return [Bcfg2.Client.XML.Element('Package', name=name, type='deb', version=version) for (name, version) in extras] def VerifyDebsums(self, entry, modlist): + """Verify the package contents with debsum information.""" output = \ self.cmd.run("%s -as %s" % - (self.debsums, entry.get('name'))).stdout.splitlines() + (self.debsums, entry.get('name'))).stderr.splitlines() if len(output) == 1 and "no md5sums for" in output[0]: self.logger.info("Package %s has no md5sums. Cannot verify" % entry.get('name')) @@ -127,11 +125,11 @@ class APT(Bcfg2.Client.Tools.Tool): # these files should not exist continue elif "is not installed" in item or "missing file" in item: - self.logger.error("Package %s is not fully installed" % - entry.get('name')) + self.logger.error("Package %s is not fully installed" + % entry.get('name')) else: - self.logger.error("Got Unsupported pattern %s from debsums" % - item) + self.logger.error("Got Unsupported pattern %s from debsums" + % item) files.append(item) files = list(set(files) - set(self.ignores)) # We check if there is file in the checksum to do @@ -142,67 +140,53 @@ class APT(Bcfg2.Client.Tools.Tool): bad = [filename for filename in files if filename not in modlist] if bad: self.logger.debug("It is suggested that you either manage " - "these files, revert the changes, or ignore " - "false failures:") - self.logger.info("Package %s failed validation. Bad files " - "are:" % entry.get('name')) + "these files, revert the changes, or " + "ignore false failures:") + self.logger.info("Package %s failed validation. Bad files are:" + % entry.get('name')) self.logger.info(bad) - entry.set('qtext', - "Reinstall Package %s-%s to fix failing files? " - "(y/N) " % (entry.get('name'), entry.get('version'))) + entry.set( + 'qtext', + "Reinstall Package %s-%s to fix failing files? (y/N) " + % (entry.get('name'), entry.get('version'))) return False return True def VerifyPackage(self, entry, modlist, checksums=True): """Verify package for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (entry.attrib['name'])) return False pkgname = entry.get('name') - if self.pkg_cache.has_key(pkgname): # nopep8 - if self._newapi: - is_installed = self.pkg_cache[pkgname].is_installed - else: - is_installed = self.pkg_cache[pkgname].isInstalled - if not self.pkg_cache.has_key(pkgname) or not is_installed: # nopep8 + if pkgname not in self.pkg_cache or \ + not self.pkg_cache[pkgname].is_installed: self.logger.info("Package %s not installed" % (entry.get('name'))) entry.set('current_exists', 'false') return False pkg = self.pkg_cache[pkgname] - if self._newapi: - installed_version = pkg.installed.version - candidate_version = pkg.candidate.version - else: - installed_version = pkg.installedVersion - candidate_version = pkg.candidateVersion + installed_version = pkg.installed.version if entry.get('version') == 'auto': - if self._newapi: - is_upgradable = \ - self.pkg_cache._depcache.is_upgradable(pkg._pkg) - else: - is_upgradable = \ - self.pkg_cache._depcache.IsUpgradable(pkg._pkg) - if is_upgradable: - desiredVersion = candidate_version + if pkg.is_upgradable: + desired_version = pkg.candidate.version else: - desiredVersion = installed_version + desired_version = installed_version elif entry.get('version') == 'any': - desiredVersion = installed_version + desired_version = installed_version else: - desiredVersion = entry.get('version') - if desiredVersion != installed_version: + desired_version = entry.get('version') + if desired_version != installed_version: entry.set('current_version', installed_version) entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % (entry.get('name'), entry.get('current_version'), - desiredVersion)) + desired_version)) return False else: # version matches - if (not Bcfg2.Options.setup.quick and - entry.get('verify', 'true') == 'true' - and checksums): + if not Bcfg2.Options.setup.quick \ + and entry.get('verify', 'true') == 'true' \ + and checksums: pkgsums = self.VerifyDebsums(entry, modlist) return pkgsums return True @@ -215,21 +199,8 @@ class APT(Bcfg2.Client.Tools.Tool): self.logger.info('Removing packages:') self.logger.info(pkgnames) for pkg in pkgnames.split(" "): - try: - if self._newapi: - self.pkg_cache[pkg].mark_delete(purge=True) - else: - self.pkg_cache[pkg].markDelete(purge=True) - except: - if self._newapi: - self.pkg_cache[pkg].mark_delete() - else: - self.pkg_cache[pkg].markDelete() - try: - self.pkg_cache.commit() - except SystemExit: - # thank you python-apt 0.6 - pass + self.pkg_cache[pkg].mark_delete(purge=True) + self.pkg_cache.commit() self.pkg_cache = apt.cache.Cache() self.modified += packages self.extra = self.FindExtra() @@ -240,40 +211,28 @@ class APT(Bcfg2.Client.Tools.Tool): ipkgs = [] bad_pkgs = [] for pkg in packages: - if not self.pkg_cache.has_key(pkg.get('name')): # nopep8 - self.logger.error("APT has no information about package %s" % - (pkg.get('name'))) + pkgname = pkg.get('name') + if pkgname not in self.pkg_cache: + self.logger.error("APT has no information about package %s" + % pkgname) continue if pkg.get('version') in ['auto', 'any']: - if self._newapi: - try: - cversion = \ - self.pkg_cache[pkg.get('name')].candidate.version - ipkgs.append("%s=%s" % (pkg.get('name'), cversion)) - except AttributeError: - self.logger.error("Failed to find %s in apt package " - "cache" % pkg.get('name')) - continue - else: - cversion = self.pkg_cache[pkg.get('name')].candidateVersion - ipkgs.append("%s=%s" % (pkg.get('name'), cversion)) - continue - if self._newapi: - avail_vers = [ - x.ver_str for x in - self.pkg_cache[pkg.get('name')]._pkg.version_list] - else: - avail_vers = [ - x.VerStr for x in - self.pkg_cache[pkg.get('name')]._pkg.VersionList] + try: + ipkgs.append("%s=%s" % ( + pkgname, + self.pkg_cache[pkgname].candidate.version)) + except AttributeError: + self.logger.error("Failed to find %s in apt package " + "cache" % pkgname) + continue + avail_vers = self.pkg_cache[pkgname].versions.keys() if pkg.get('version') in avail_vers: - ipkgs.append("%s=%s" % (pkg.get('name'), pkg.get('version'))) + ipkgs.append("%s=%s" % (pkgname, pkg.get('version'))) continue else: - self.logger.error("Package %s: desired version %s not in %s" % - (pkg.get('name'), pkg.get('version'), - avail_vers)) - bad_pkgs.append(pkg.get('name')) + self.logger.error("Package %s: desired version %s not in %s" + % (pkgname, pkg.get('version'), avail_vers)) + bad_pkgs.append(pkgname) if bad_pkgs: self.logger.error("Cannot find correct versions of packages:") self.logger.error(bad_pkgs) @@ -290,6 +249,6 @@ class APT(Bcfg2.Client.Tools.Tool): self.modified.append(package) return states - def VerifyPath(self, entry, _): + def VerifyPath(self, entry, _): # pylint: disable=W0613 """Do nothing here since we only verify Path type=ignore.""" return True diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index dedc50d89..ca0502b75 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -2,7 +2,6 @@ import Bcfg2.Client.Tools from Bcfg2.Utils import safe_input -from Bcfg2.Client import matches_white_list, passes_black_list class Action(Bcfg2.Client.Tools.Tool): @@ -11,23 +10,6 @@ class Action(Bcfg2.Client.Tools.Tool): __handles__ = [('Action', None)] __req__ = {'Action': ['name', 'timing', 'when', 'command', 'status']} - def _action_allowed(self, action): - """ Return true if the given action is allowed to be run by - the whitelist or blacklist """ - if (Bcfg2.Options.setup.decision == 'whitelist' and - not matches_white_list(action, - Bcfg2.Options.setup.decision_list)): - self.logger.info("In whitelist mode: suppressing Action: %s" % - action.get('name')) - return False - if (Bcfg2.Options.setup.decision == 'blacklist' and - not passes_black_list(action, - Bcfg2.Options.setup.decision_list)): - self.logger.info("In blacklist mode: suppressing Action: %s" % - action.get('name')) - return False - return True - def RunAction(self, entry): """This method handles command execution and status return.""" shell = False @@ -76,7 +58,7 @@ class Action(Bcfg2.Client.Tools.Tool): states = dict() for action in bundle.findall("Action"): if action.get('timing') in ['post', 'both']: - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) return states @@ -87,7 +69,7 @@ class Action(Bcfg2.Client.Tools.Tool): for action in bundle.findall("Action"): if (action.get('timing') in ['post', 'both'] and action.get('when') != 'modified'): - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) return states diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index a7fcb6709..7200b0fc2 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -160,7 +160,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group") + if entry.get("name") in g[3] and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 5698f237a..332638de4 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,6 +4,8 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML +from Bcfg2.Compat import urlretrieve + # pylint: disable=C0103 noask = ''' @@ -37,6 +39,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): # noaskfile needs to live beyond __init__ otherwise file is removed self.noaskfile = tempfile.NamedTemporaryFile() self.noaskname = self.noaskfile.name + # for any pkg files downloaded + self.tmpfiles = [] try: self.noaskfile.write(noask) # flush admin file contents to disk @@ -45,6 +49,41 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.pkgtool[1]) except: # pylint: disable=W0702 self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1]) + self.origpkgtool = self.pkgtool + + def pkgmogrify(self, packages): + """ Take a list of pkg objects, check for a 'simplefile' attribute. + If present, insert a _sysv_pkg_path attribute to the package and + download the datastream format SYSV package to a temporary file. + """ + for pkg in packages: + if pkg.get('simplefile'): + tmpfile = tempfile.NamedTemporaryFile() + self.tmpfiles.append(tmpfile) + self.logger.info("Downloading %s to %s" % (pkg.get('url'), + tmpfile.name)) + urlretrieve(pkg.get('url'), tmpfile.name) + pkg.set('_sysv_pkg_path', tmpfile.name) + + def _get_package_command(self, packages): + """Override the default _get_package_command, replacing the attribute + 'url' if '_sysv_pkg_path' if necessary in the returned command + string + """ + if hasattr(self, 'origpkgtool'): + if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): + self.pkgtool = (self.pkgtool[0], ('%s %s', + ['_sysv_pkg_path', 'name'])) + else: + self.pkgtool = self.origpkgtool + + pkgcmd = super(SYSV, self)._get_package_command(packages) + self.logger.debug("Calling install command: %s" % pkgcmd) + return pkgcmd + + def Install(self, packages): + self.pkgmogrify(packages) + super(SYSV, self).Install(packages) def RefreshPackages(self): """Refresh memory hashes of packages.""" @@ -80,8 +119,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Package %s not installed" % entry.get("name")) else: - if (Bcfg2.Options.setup.quick or - entry.attrib.get('verify', 'true') == 'false'): + if Bcfg2.Options.setup.quick or \ + entry.attrib.get('verify', 'true') == 'false': return True rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name')) if rv.success: diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 027d91c71..3b60c8285 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,15 +13,25 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} + def get_svc_name(self, service): + """Append .service to name if name doesn't specify a unit type.""" + svc = service.get('name') + if svc.endswith(('.service', '.socket', '.device', '.mount', + '.automount', '.swap', '.target', '.path', + '.timer', '.snapshot', '.slice', '.scope')): + return svc + else: + return '%s.service' % svc + def get_svc_command(self, service, action): - return "/bin/systemctl %s %s.service" % (action, service.get('name')) + return "/bin/systemctl %s %s" % (action, self.get_svc_name(service)) def VerifyService(self, entry, _): """Verify Service status for entry.""" if entry.get('status') == 'ignore': return True - cmd = "/bin/systemctl status %s.service " % (entry.get('name')) + cmd = "/bin/systemctl status %s" % (self.get_svc_name(entry)) rv = self.cmd.run(cmd) if 'Loaded: error' in rv.stdout: diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 21fc05b0d..a8a80974a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -11,6 +11,7 @@ import yum.callbacks import yum.Errors import yum.misc import rpmUtils.arch +import rpmUtils.miscutils import Bcfg2.Client.XML import Bcfg2.Client.Tools import Bcfg2.Options @@ -148,12 +149,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): dest="yum_verify_flags", type=Bcfg2.Options.Types.comma_list, help="YUM verify flags"), Bcfg2.Options.Option( - cf=('YUM', 'disabled_plugins'), default=[], - type=Bcfg2.Options.Types.comma_list, dest="yum_disabled_plugins", + cf=('YUM', 'disabled_plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, dest="yum_disabled_plugins", help="YUM disabled plugins"), Bcfg2.Options.Option( - cf=('YUM', 'enabled_plugins'), default=[], - type=Bcfg2.Options.Types.comma_list, dest="yum_enabled_plugins", + cf=('YUM', 'enabled_plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, dest="yum_enabled_plugins", help="YUM enabled plugins")] pkgtype = 'yum' @@ -660,7 +661,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra.get('release', 'any')) entry.set('current_version', "%s:%s-%s" % current_evr) entry.set('version', "%s:%s-%s" % wanted_evr) - if yum.compareEVR(current_evr, wanted_evr) == 1: + if rpmUtils.miscutils.compareEVR(current_evr, wanted_evr) == 1: entry.set("package_fail_action", "downgrade") else: entry.set("package_fail_action", "update") @@ -976,8 +977,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra2string(build_yname(pkg.get('name'), inst))) continue status = self.instance_status[inst] - if (not status.get('installed', False) and - Bcfg2.Options.setup.yum_install_missing): + if not status.get('installed', False) and \ + Bcfg2.Options.setup.yum_install_missing: queue_pkg(pkg, inst, install_pkgs) elif (status.get('version_fail', False) and Bcfg2.Options.setup.yum_fix_version): diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index cd294db98..ae7fa3aed 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -129,6 +129,23 @@ class Tool(object): raise ToolInstantiationError("%s: %s not executable" % (self.name, filename)) + def _install_allowed(self, entry): + """ Return true if the given entry is allowed to be installed by + the whitelist or blacklist """ + if (Bcfg2.Options.setup.decision == 'whitelist' and + not Bcfg2.Client.matches_white_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In whitelist mode: suppressing Action: %s" % + entry.get('name')) + return False + if (Bcfg2.Options.setup.decision == 'blacklist' and + not Bcfg2.Client.passes_black_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In blacklist mode: suppressing Action: %s" % + entry.get('name')) + return False + return True + def BundleUpdated(self, bundle): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. @@ -587,7 +604,8 @@ class SvcTool(Tool): return for entry in bundle: - if not self.handlesEntry(entry): + if (not self.handlesEntry(entry) + or not self._install_allowed(entry)): continue estatus = entry.get('status') diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 049236e03..b8a75a0c5 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -20,6 +20,7 @@ except ImportError: # urllib imports try: from urllib import quote_plus + from urllib import urlretrieve from urlparse import urljoin, urlparse from urllib2 import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ @@ -27,7 +28,8 @@ try: except ImportError: from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, urlopen + HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ + urlopen, urlretrieve from urllib.error import HTTPError, URLError try: diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index b817ecb94..f5b5d16aa 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -212,7 +212,7 @@ class _OptionContainer(object): Bcfg2.Options.Option( cf=('database', 'engine'), default='sqlite3', help='Database engine', dest='db_engine'), - Bcfg2.Options.Option( + Bcfg2.Options.RepositoryMacroOption( cf=('database', 'name'), default='<repository>/etc/bcfg2.sqlite', help="Database name", dest="db_name"), Bcfg2.Options.Option( diff --git a/src/lib/Bcfg2/Options/Actions.py b/src/lib/Bcfg2/Options/Actions.py index 8b941f2bb..854e6039d 100644 --- a/src/lib/Bcfg2/Options/Actions.py +++ b/src/lib/Bcfg2/Options/Actions.py @@ -2,7 +2,8 @@ import sys import argparse -from Bcfg2.Options.Parser import get_parser +from Bcfg2.Options.Parser import get_parser, OptionParserException +from Bcfg2.Options.Options import _debug __all__ = ["ConfigFileAction", "ComponentAction", "PluginsAction"] @@ -101,7 +102,7 @@ class ComponentAction(FinalizableAction): fail_silently = False def __init__(self, *args, **kwargs): - if self.mapping: + if self.mapping and not self.islist: if 'choices' not in kwargs: kwargs['choices'] = self.mapping.keys() FinalizableAction.__init__(self, *args, **kwargs) @@ -112,9 +113,12 @@ class ComponentAction(FinalizableAction): try: return getattr(__import__(module, fromlist=[name]), name) except (AttributeError, ImportError): + msg = "Failed to load %s from %s: %s" % (name, module, + sys.exc_info()[1]) if not self.fail_silently: - print("Failed to load %s from %s: %s" % - (name, module, sys.exc_info()[1])) + print(msg) + else: + _debug(msg) return None def _load_component(self, name): @@ -143,7 +147,7 @@ class ComponentAction(FinalizableAction): if cls: get_parser().add_component(cls) elif not self.fail_silently: - print("Could not load component %s" % name) + raise OptionParserException("Could not load component %s" % name) return cls def __call__(self, parser, namespace, values, option_string=None): @@ -168,7 +172,10 @@ class ConfigFileAction(FinalizableAction): ``bcfg2-lint.conf``). """ def __call__(self, parser, namespace, values, option_string=None): - parser.add_config_file(self.dest, values, reparse=False) + if values: + parser.add_config_file(self.dest, values, reparse=False) + else: + _debug("No config file passed for %s" % self) FinalizableAction.__call__(self, parser, namespace, values, option_string=option_string) @@ -177,3 +184,4 @@ class PluginsAction(ComponentAction): """ :class:`Bcfg2.Options.ComponentAction` subclass for loading Bcfg2 server plugins. """ bases = ['Bcfg2.Server.Plugins'] + fail_silently = True diff --git a/src/lib/Bcfg2/Options/OptionGroups.py b/src/lib/Bcfg2/Options/OptionGroups.py index 465358fab..49340ab36 100644 --- a/src/lib/Bcfg2/Options/OptionGroups.py +++ b/src/lib/Bcfg2/Options/OptionGroups.py @@ -10,7 +10,7 @@ __all__ = ["OptionGroup", "ExclusiveOptionGroup", "Subparser", "WildcardSectionGroup"] -class OptionContainer(list): +class _OptionContainer(list): """ Parent class of all option groups """ def list_options(self): @@ -29,7 +29,7 @@ class OptionContainer(list): opt.add_to_parser(parser) -class OptionGroup(OptionContainer): +class OptionGroup(_OptionContainer): """ Generic option group that is used only to organize options. This uses :meth:`argparse.ArgumentParser.add_argument_group` behind the scenes. """ @@ -43,16 +43,16 @@ class OptionGroup(OptionContainer): :param description: A longer description of the option group :param description: string """ - OptionContainer.__init__(self, items) + _OptionContainer.__init__(self, items) self.title = kwargs.pop('title') self.description = kwargs.pop('description', None) def add_to_parser(self, parser): group = parser.add_argument_group(self.title, self.description) - OptionContainer.add_to_parser(self, group) + _OptionContainer.add_to_parser(self, group) -class ExclusiveOptionGroup(OptionContainer): +class ExclusiveOptionGroup(_OptionContainer): """ Option group that ensures that only one argument in the group is present. This uses :meth:`argparse.ArgumentParser.add_mutually_exclusive_group` @@ -66,15 +66,15 @@ class ExclusiveOptionGroup(OptionContainer): specified. :type required: boolean """ - OptionContainer.__init__(self, items) + _OptionContainer.__init__(self, items) self.required = kwargs.pop('required', False) def add_to_parser(self, parser): - group = parser.add_mutually_exclusive_group(required=self.required) - OptionContainer.add_to_parser(self, group) + _OptionContainer.add_to_parser( + self, parser.add_mutually_exclusive_group(required=self.required)) -class Subparser(OptionContainer): +class Subparser(_OptionContainer): """ Option group that adds options in it to a subparser. This uses a lot of functionality tied to `argparse Sub-commands <http://docs.python.org/dev/library/argparse.html#sub-commands>`_. @@ -99,7 +99,7 @@ class Subparser(OptionContainer): """ self.name = kwargs.pop('name') self.help = kwargs.pop('help', None) - OptionContainer.__init__(self, items) + _OptionContainer.__init__(self, items) def __repr__(self): return "%s %s(%s)" % (self.__class__.__name__, @@ -111,11 +111,11 @@ class Subparser(OptionContainer): self._subparsers[parser] = parser.add_subparsers(dest='subcommand') subparser = self._subparsers[parser].add_parser(self.name, help=self.help) - OptionContainer.add_to_parser(self, subparser) + _OptionContainer.add_to_parser(self, subparser) -class WildcardSectionGroup(OptionContainer, Option): - """ WildcardSectionGroups contain options that may exist in +class WildcardSectionGroup(_OptionContainer, Option): + """WildcardSectionGroups contain options that may exist in several different sections of the config that match a glob. It works by creating options on the fly to match the sections described in the glob. For example, consider: @@ -134,7 +134,7 @@ class WildcardSectionGroup(OptionContainer, Option): .. code-block:: python >>> Bcfg2.Options.setup - Namespace(myplugin_bar_description='Bar description', myplugin_bar_number=2, myplugin_foo_description='Foo description', myplugin_foo_number=1, myplugin_sections=['myplugin:foo', 'myplugin:bar']) + Namespace(myplugin_bar_description='Bar description', myplugin_myplugin_bar_number=2, myplugin_myplugin_foo_description='Foo description', myplugin_myplugin_foo_number=1, myplugin_sections=['myplugin:foo', 'myplugin:bar']) All options must have the same section glob. @@ -146,10 +146,10 @@ class WildcardSectionGroup(OptionContainer, Option): ``<destination>`` is the original `dest <http://docs.python.org/dev/library/argparse.html#dest>`_ of the option. ``<section>`` is the section that it's found in. - ``<prefix>`` is automatically generated from the section glob by - replacing all consecutive characters disallowed in Python variable - names into underscores. (This can be overridden with the - constructor.) + ``<prefix>`` is automatically generated from the section glob. + (This can be overridden with the constructor.) Both ``<section>`` + and ``<prefix>`` have had all consecutive characters disallowed in + Python variable names replaced with underscores. This group stores an additional option, the sections themselves, in an option given by ``<prefix>sections``. @@ -171,17 +171,17 @@ class WildcardSectionGroup(OptionContainer, Option): that match the glob. :param dest: string """ - OptionContainer.__init__(self, []) + _OptionContainer.__init__(self, []) self._section_glob = items[0].cf[0] # get a default destination self._prefix = kwargs.get("prefix", self._dest_re.sub('_', self._section_glob)) Option.__init__(self, dest=kwargs.get('dest', self._prefix + "sections")) - self._options = items + self.option_templates = items def list_options(self): - return [self] + OptionContainer.list_options(self) + return [self] + _OptionContainer.list_options(self) def from_config(self, cfp): sections = [] @@ -189,10 +189,12 @@ class WildcardSectionGroup(OptionContainer, Option): if fnmatch.fnmatch(section, self._section_glob): sections.append(section) newopts = [] - for opt_tmpl in self._options: + for opt_tmpl in self.option_templates: option = copy.deepcopy(opt_tmpl) option.cf = (section, option.cf[1]) - option.dest = self._prefix + section + "_" + option.dest + option.dest = "%s%s_%s" % (self._prefix, + self._dest_re.sub('_', section), + option.dest) newopts.append(option) self.extend(newopts) for parser in self.parsers: @@ -201,4 +203,17 @@ class WildcardSectionGroup(OptionContainer, Option): def add_to_parser(self, parser): Option.add_to_parser(self, parser) - OptionContainer.add_to_parser(self, parser) + _OptionContainer.add_to_parser(self, parser) + + def __eq__(self, other): + return (_OptionContainer.__eq__(self, other) and + self.option_templates == getattr(other, "option_templates", + None)) + + def __repr__(self): + if len(self) == 0: + return "%s(%s)" % (self.__class__.__name__, + ", ".join(".".join(o.cf) + for o in self.option_templates)) + else: + return _OptionContainer.__repr__(self) diff --git a/src/lib/Bcfg2/Options/Options.py b/src/lib/Bcfg2/Options/Options.py index 3874f810d..752e01b4e 100644 --- a/src/lib/Bcfg2/Options/Options.py +++ b/src/lib/Bcfg2/Options/Options.py @@ -1,17 +1,23 @@ -""" The base :class:`Bcfg2.Options.Option` object represents an -option. Unlike options in :mod:`argparse`, an Option object does not -need to be associated with an option parser; it exists on its own.""" +"""Base :class:`Bcfg2.Options.Option` object to represent an option. -import os +Unlike options in :mod:`argparse`, an Option object does not need to +be associated with an option parser; it exists on its own. +""" + +import argparse import copy import fnmatch -import argparse +import os +import sys + from Bcfg2.Options import Types from Bcfg2.Compat import ConfigParser -__all__ = ["Option", "BooleanOption", "PathOption", "PositionalArgument", - "_debug"] +__all__ = ["Option", "BooleanOption", "RepositoryMacroOption", "PathOption", + "PositionalArgument", "_debug"] + +unit_test = False # pylint: disable=C0103 def _debug(msg): @@ -19,8 +25,11 @@ def _debug(msg): they're options, after all -- so option parsing verbosity is enabled by changing this to True. The verbosity here is primarily of use to developers. """ - if os.environ.get('BCFG2_OPTIONS_DEBUG', '0') == '1': - print(msg) + if unit_test: + print("DEBUG: %s" % msg) + elif os.environ.get('BCFG2_OPTIONS_DEBUG', '0').lower() in ["true", "yes", + "on", "1"]: + sys.stderr.write("%s\n" % msg) #: A dict that records a mapping of argparse action name (e.g., @@ -37,7 +46,7 @@ def _get_action_class(action_name): on. So we just instantiate a dummy parser, add a dummy argument, and determine the class that way. """ if (isinstance(action_name, type) and - issubclass(action_name, argparse.Action)): + issubclass(action_name, argparse.Action)): return action_name if action_name not in _action_map: action = argparse.ArgumentParser().add_argument(action_name, @@ -133,10 +142,11 @@ class Option(object): self._dest = None if 'dest' in self._kwargs: self._dest = self._kwargs.pop('dest') - elif self.cf is not None: - self._dest = self.cf[1] elif self.env is not None: self._dest = self.env + elif self.cf is not None: + self._dest = self.cf[1] + self._dest = self._dest.lower().replace("-", "_") kwargs = copy.copy(self._kwargs) kwargs.pop("action", None) self.actions[None] = action_cls(self._dest, self._dest, **kwargs) @@ -149,9 +159,10 @@ class Option(object): sources.append("%s.%s" % self.cf) if self.env: sources.append("$" + self.env) - spec = ["sources=%s" % sources, "default=%s" % self.default] - spec.append("%d parsers" % (len(self.parsers))) - return 'Option(%s: %s)' % (self.dest, ", ".join(spec)) + spec = ["sources=%s" % sources, "default=%s" % self.default, + "%d parsers" % len(self.parsers)] + return '%s(%s: %s)' % (self.__class__.__name__, + self.dest, ", ".join(spec)) def list_options(self): """ List options contained in this option. This exists to @@ -175,6 +186,17 @@ class Option(object): _debug("Finalizing %s" % self) action.finalize(parser, namespace) + @property + def _type_func(self): + """get a function for converting a value to the option type. + + this always returns a callable, even when ``type`` is None. + """ + if self.type: + return self.type + else: + return lambda x: x + def from_config(self, cfp): """ Get the value of this option from the given :class:`ConfigParser.ConfigParser`. If it is not found in the @@ -201,21 +223,33 @@ class Option(object): self.cf[1]) if o not in exclude]) else: - rv = dict() + rv = {} else: - if self.type: - rtype = self.type - else: - rtype = lambda x: x try: - rv = rtype(cfp.getboolean(*self.cf)) - except ValueError: - rv = rtype(cfp.get(*self.cf)) + rv = self._type_func(self.get_config_value(cfp)) except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): rv = None - _debug("Setting %s from config file(s): %s" % (self, rv)) + _debug("Getting value of %s from config file(s): %s" % (self, rv)) return rv + def get_config_value(self, cfp): + """fetch a value from the config file. + + This is passed the config parser. Its result is passed to the + type function for this option. It can be overridden to, e.g., + handle boolean options. + """ + return cfp.get(*self.cf) + + def get_environ_value(self, value): + """fetch a value from the environment. + + This is passed the raw value from the environment variable, + and its result is passed to the type function for this + option. It can be overridden to, e.g., handle boolean options. + """ + return value + def default_from_config(self, cfp): """ Set the default value of this option from the config file or from the environment. @@ -224,7 +258,8 @@ class Option(object): :type cfp: ConfigParser.ConfigParser """ if self.env and self.env in os.environ: - self.default = os.environ[self.env] + self.default = self._type_func( + self.get_environ_value(os.environ[self.env])) _debug("Setting the default of %s from environment: %s" % (self, self.default)) else: @@ -257,6 +292,13 @@ class Option(object): for action in self.actions.values(): action.dest = value + def early_parsing_hook(self, early_opts): # pylint: disable=C0111 + """Hook called at the end of early option parsing. + + This can be used to save option values for macro fixup. + """ + pass + #: The namespace destination of this option (see `dest #: <http://docs.python.org/dev/library/argparse.html#dest>`_) dest = property(_get_dest, _set_dest) @@ -284,13 +326,65 @@ class Option(object): (self, parser)) -class PathOption(Option): - """ Shortcut for options that expect a path argument. Uses - :meth:`Bcfg2.Options.Types.path` to transform the argument into a - canonical path. +class RepositoryMacroOption(Option): + """Option that does translation of ``<repository>`` macros. - The type of a path option can also be overridden to return an - option file-like object. For example: + Macro translation is done on the fly instead of just fixing up all + values at the end of parsing because macro expansion needs to be + done before path canonicalization for + :class:`Bcfg2.Options.Options.PathOption`. + """ + repository = None + + def __init__(self, *args, **kwargs): + self._original_type = kwargs.pop('type', lambda x: x) + kwargs['type'] = self._type + kwargs.setdefault('metavar', '<path>') + Option.__init__(self, *args, **kwargs) + + def early_parsing_hook(self, early_opts): + if hasattr(early_opts, "repository"): + if self.__class__.repository is None: + _debug("Setting repository to %s for %s" % + (early_opts.repository, self.__class__.__name__)) + self.__class__.repository = early_opts.repository + else: + _debug("Repository is already set for %s" % self.__class__) + + def _get_default(self): + """ Getter for the ``default`` property """ + if not hasattr(self._default, "replace"): + return self._default + else: + return self._type(self._default) + + default = property(_get_default, Option._set_default) + + def transform_value(self, value): + """transform the value after macro expansion. + + this can be overridden to further transform the value set by + the user *after* macros are expanded, but before the user's + ``type`` function is applied. principally exists for + PathOption to canonicalize the path. + """ + return value + + def _type(self, value): + """Type function that fixes up <repository> macros.""" + if self.__class__.repository is None: + return value + else: + return self._original_type(self.transform_value( + value.replace("<repository>", self.__class__.repository))) + + +class PathOption(RepositoryMacroOption): + """Shortcut for options that expect a path argument. + + Uses :meth:`Bcfg2.Options.Types.path` to transform the argument + into a canonical path. The type of a path option can also be + overridden to return a file-like object. For example: .. code-block:: python @@ -298,30 +392,41 @@ class PathOption(Option): Bcfg2.Options.PathOption( "--input", type=argparse.FileType('r'), help="The input file")] - """ - def __init__(self, *args, **kwargs): - kwargs.setdefault('type', Types.path) - kwargs.setdefault('metavar', '<path>') - Option.__init__(self, *args, **kwargs) + PathOptions also do translation of ``<repository>`` macros. + """ + def transform_value(self, value): + return Types.path(value) class _BooleanOptionAction(argparse.Action): - """ BooleanOptionAction sets a boolean value in the following ways: + """BooleanOptionAction sets a boolean value. + - if None is passed, store the default - if the option_string is not None, then the option was passed on the command line, thus store the opposite of the default (this is the argparse store_true and store_false behavior) - if a boolean value is passed, use that + Makes a copy of the initial default, because otherwise the default + can be changed by config file settings or environment + variables. For instance, if a boolean option that defaults to True + was set to False in the config file, specifying the option on the + CLI would then set it back to True. + Defined here instead of :mod:`Bcfg2.Options.Actions` because otherwise - there is a circular import Options -> Actions -> Parser -> Options """ + there is a circular import Options -> Actions -> Parser -> Options. + """ + + def __init__(self, *args, **kwargs): + argparse.Action.__init__(self, *args, **kwargs) + self.original = self.default def __call__(self, parser, namespace, values, option_string=None): if values is None: setattr(namespace, self.dest, self.default) elif option_string is not None: - setattr(namespace, self.dest, not self.default) + setattr(namespace, self.dest, not self.original) else: setattr(namespace, self.dest, bool(values)) @@ -340,9 +445,25 @@ class BooleanOption(Option): kwargs.setdefault('action', _BooleanOptionAction) kwargs.setdefault('nargs', 0) kwargs.setdefault('default', False) - Option.__init__(self, *args, **kwargs) + def get_environ_value(self, value): + if value.lower() in ["false", "no", "off", "0"]: + return False + elif value.lower() in ["true", "yes", "on", "1"]: + return True + else: + raise ValueError("Invalid boolean value %s" % value) + + def get_config_value(self, cfp): + """fetch a value from the config file. + + This is passed the config parser. Its result is passed to the + type function for this option. It can be overridden to, e.g., + handle boolean options. + """ + return cfp.getboolean(*self.cf) + class PositionalArgument(Option): """ Shortcut for positional arguments. """ diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py index 677a69e4c..c846e8093 100644 --- a/src/lib/Bcfg2/Options/Parser.py +++ b/src/lib/Bcfg2/Options/Parser.py @@ -1,13 +1,15 @@ -""" The option parser """ +"""The option parser.""" +import argparse import os import sys -import argparse + from Bcfg2.version import __version__ from Bcfg2.Compat import ConfigParser from Bcfg2.Options import Option, PathOption, BooleanOption, _debug -__all__ = ["setup", "OptionParserException", "Parser", "get_parser"] +__all__ = ["setup", "OptionParserException", "Parser", "get_parser", + "new_parser"] #: The repository option. This is specified here (and imported into @@ -108,13 +110,28 @@ class Parser(argparse.ArgumentParser): for component in components: self.add_component(component) + def _check_duplicate_cf(self, option): + """Check for a duplicate config file option.""" + def add_options(self, options): """ Add an explicit list of options to the parser. When possible, prefer :func:`Bcfg2.Options.Parser.add_component` to add a whole component instead.""" + _debug("Adding options: %s" % options) self.parsed = False for option in options: if option not in self.option_list: + # check for duplicates + if (hasattr(option, "env") and option.env and + option.env in [o.env for o in self.option_list]): + raise OptionParserException( + "Duplicate environment variable option: %s" % + option.env) + if (hasattr(option, "cf") and option.cf and + option.cf in [o.cf for o in self.option_list]): + raise OptionParserException( + "Duplicate config file option: %s" % (option.cf,)) + self.option_list.extend(option.list_options()) option.add_to_parser(self) @@ -151,6 +168,8 @@ class Parser(argparse.ArgumentParser): (opt, value)) action(self, self.namespace, value) else: + _debug("Setting config file-only option %s to %s" % + (opt, value)) setattr(self.namespace, opt.dest, value) def _finalize(self): @@ -169,11 +188,58 @@ class Parser(argparse.ArgumentParser): _debug("Resetting namespace") for attr in dir(self.namespace): if (not attr.startswith("_") and - attr not in ['uri', 'version', 'name'] and - attr not in self._config_files): + attr not in ['uri', 'version', 'name'] and + attr not in self._config_files): _debug("Deleting %s" % attr) delattr(self.namespace, attr) + def _parse_early_options(self): + """Parse early options. + + Early options are options that need to be parsed before other + options for some reason. These fall into two basic cases: + + 1. Database options, which need to be parsed so that Django + modules can be imported, since Django configuration is all + done at import-time; + 2. The repository (``-Q``) option, so that ``<repository>`` + macros in other options can be resolved. + """ + _debug("Option parsing phase 2: Parse early options") + early_opts = argparse.Namespace() + early_parser = Parser(add_help=False, namespace=early_opts, + early=True) + + # add the repo option so we can resolve <repository> + # macros + early_parser.add_options([repository]) + + early_components = [] + for component in self.components: + if getattr(component, "parse_first", False): + early_components.append(component) + early_parser.add_component(component) + early_parser.parse(self.argv) + + _debug("Fixing up <repository> macros in early options") + for attr_name in dir(early_opts): + if not attr_name.startswith("_"): + attr = getattr(early_opts, attr_name) + if hasattr(attr, "replace"): + setattr(early_opts, attr_name, + attr.replace("<repository>", + early_opts.repository)) + + _debug("Early parsing complete, calling hooks") + for component in early_components: + if hasattr(component, "component_parsed_hook"): + _debug("Calling component_parsed_hook on %s" % component) + getattr(component, "component_parsed_hook")(early_opts) + _debug("Calling early parsing hooks; early options: %s" % + early_opts) + for option in self.option_list: + option.early_parsing_hook(early_opts) + def add_config_file(self, dest, cfile, reparse=True): """ Add a config file, which triggers a full reparse of all options. """ @@ -190,10 +256,11 @@ class Parser(argparse.ArgumentParser): def reparse(self, argv=None): """ Reparse options after they have already been parsed. - :param argv: The argument list to parse. By default, + :param argv: The argument list to parse. By default, :attr:`Bcfg2.Options.Parser.argv` is reused. (I.e., the argument list that was initially - parsed.) :type argv: list + parsed.) + :type argv: list """ _debug("Reparsing all options") self._reset_namespace() @@ -210,7 +277,7 @@ class Parser(argparse.ArgumentParser): """ _debug("Parsing options") if argv is None: - argv = sys.argv[1:] + argv = sys.argv[1:] # pragma: nocover if self.parsed and self.argv == argv: _debug("Returning already parsed namespace") return self.namespace @@ -231,25 +298,10 @@ class Parser(argparse.ArgumentParser): # phase 2: re-parse command line for early options; currently, # that's database options - _debug("Option parsing phase 2: Parse early options") if not self._early: - early_opts = argparse.Namespace() - early_parser = Parser(add_help=False, namespace=early_opts, - early=True) - # add the repo option so we can resolve <repository> - # macros - early_parser.add_options([repository]) - early_components = [] - for component in self.components: - if getattr(component, "parse_first", False): - early_components.append(component) - early_parser.add_component(component) - early_parser.parse(self.argv) - _debug("Early parsing complete, calling hooks") - for component in early_components: - if hasattr(component, "component_parsed_hook"): - _debug("Calling component_parsed_hook on %s" % component) - getattr(component, "component_parsed_hook")(early_opts) + self._parse_early_options() + else: + _debug("Skipping parsing phase 2 in early mode") # phase 3: re-parse command line, loading additional # components, until all components have been loaded. On each @@ -273,31 +325,34 @@ class Parser(argparse.ArgumentParser): # _parse_config_options is called, all config file options will get set # to their hardcoded defaults. This process defines the options in the # namespace and _parse_config_options will never look at them again. - self._set_defaults_from_config() - self._parse_config_options() + # + # we have to do the parsing in two loops: first, we squeeze as + # much data out of the config file as we can to ensure that + # all config file settings are read before we use any default + # values. then we can start looking at the command line. while not self.parsed: self.parsed = True self._set_defaults_from_config() - self.parse_known_args(args=self.argv, namespace=self.namespace) + self._parse_config_options() + self.parsed = False + remaining = [] + while not self.parsed: + self.parsed = True + _debug("Parsing known arguments") + try: + _, remaining = self.parse_known_args(args=self.argv, + namespace=self.namespace) + except OptionParserException: + self.error(sys.exc_info()[1]) + self._set_defaults_from_config() self._parse_config_options() self._finalize() + if len(remaining) and not self._early: + self.error("Unknown options: %s" % " ".join(remaining)) - # phase 4: fix up <repository> macros - _debug("Option parsing phase 4: Fix up macros") - repo = getattr(self.namespace, "repository", repository.default) - for attr in dir(self.namespace): - value = getattr(self.namespace, attr) - if (not attr.startswith("_") and - hasattr(value, "replace") and - "<repository>" in value): - setattr(self.namespace, attr, - value.replace("<repository>", repo, 1)) - _debug("Fixing up macros in %s: %s -> %s" % - (attr, value, getattr(self.namespace, attr))) - - # phase 5: call post-parsing hooks - _debug("Option parsing phase 5: Call hooks") + # phase 4: call post-parsing hooks if not self._early: + _debug("Option parsing phase 4: Call hooks") for component in self.components: if hasattr(component, "options_parsed_hook"): _debug("Calling post-parsing hook on %s" % component) @@ -311,23 +366,23 @@ class Parser(argparse.ArgumentParser): _parser = Parser() # pylint: disable=C0103 +def new_parser(): + """Create a new :class:`Bcfg2.Options.Parser` object. + + The new object can be retrieved with + :func:`Bcfg2.Options.get_parser`. This is useful for unit + testing. + """ + global _parser + _parser = Parser() + + def get_parser(description=None, components=None, namespace=None): - """ Get an existing :class:`Bcfg2.Options.Parser` object. (One is - created at the module level when :mod:`Bcfg2.Options` is - imported.) If no arguments are given, then the existing parser is - simply fetched. - - If arguments are given, then one of two things happens: - - * If this is the first ``get_parser`` call with arguments, then - the values given are set accordingly in the parser, and it is - returned. - * If this is not the first such call, then - :class:`Bcfg2.Options.OptionParserException` is raised. - - That is, a ``get_parser`` call with options is considered to - initialize the parser that already exists, and that can only - happen once. + """Get an existing :class:`Bcfg2.Options.Parser` object. + + A Parser is created at the module level when :mod:`Bcfg2.Options` + is imported. If any arguments are given, then the existing parser + is modified before being returned. :param description: Set the parser description :type description: string diff --git a/src/lib/Bcfg2/Options/Subcommands.py b/src/lib/Bcfg2/Options/Subcommands.py index 660bd5077..8972bde00 100644 --- a/src/lib/Bcfg2/Options/Subcommands.py +++ b/src/lib/Bcfg2/Options/Subcommands.py @@ -7,12 +7,13 @@ import sys import copy import shlex import logging + from Bcfg2.Compat import StringIO -from Bcfg2.Options import PositionalArgument +from Bcfg2.Options import PositionalArgument, _debug from Bcfg2.Options.OptionGroups import Subparser from Bcfg2.Options.Parser import Parser, setup as master_setup -__all__ = ["Subcommand", "HelpCommand", "CommandRegistry", "register_commands"] +__all__ = ["Subcommand", "CommandRegistry"] class Subcommand(object): @@ -96,8 +97,8 @@ class Subcommand(object): sio = StringIO() self.parser.print_usage(file=sio) usage = self._ws_re.sub(' ', sio.getvalue()).strip()[7:] - doc = self._ws_re.sub(' ', getattr(self, "__doc__")).strip() - if doc is None: + doc = self._ws_re.sub(' ', getattr(self, "__doc__") or "").strip() + if not doc: self._usage = usage else: self._usage = "%s - %s" % (usage, doc) @@ -119,120 +120,130 @@ class Subcommand(object): command from the interactive shell. :type setup: argparse.Namespace """ - raise NotImplementedError + raise NotImplementedError # pragma: nocover def shutdown(self): - """ Perform any necessary shtudown tasks for this command This + """ Perform any necessary shutdown tasks for this command This is called to when the program exits (*not* when this command is finished executing). """ - pass + pass # pragma: nocover -class HelpCommand(Subcommand): - """ Get help on a specific subcommand. This must be subclassed to - create the actual help command by overriding - :func:`Bcfg2.Options.HelpCommand.command_registry` and giving the - command access to a :class:`Bcfg2.Options.CommandRegistry`. """ +class Help(Subcommand): + """List subcommands and usage, or get help on a specific subcommand.""" options = [PositionalArgument("command", nargs='?')] # the interactive shell has its own help interactive = False - def command_registry(self): - """ Return a :class:`Bcfg2.Options.CommandRegistry` class. - All commands registered with the class will be included in the - help message. """ - raise NotImplementedError + def __init__(self, registry): + Subcommand.__init__(self) + self._registry = registry def run(self, setup): - commands = self.command_registry() + commands = self._registry.commands if setup.command: try: commands[setup.command].parser.print_help() return 0 except KeyError: print("No such command: %s" % setup.command) + return 1 for command in sorted(commands.keys()): print(commands[command].usage()) class CommandRegistry(object): - """ A ``CommandRegistry`` is used to register subcommands and - provides a single interface to run them. It's also used by - :class:`Bcfg2.Options.HelpCommand` to produce help messages for - all available commands. """ + """A ``CommandRegistry`` is used to register subcommands and provides + a single interface to run them. It's also used by + :class:`Bcfg2.Options.Subcommands.Help` to produce help messages + for all available commands. + """ + + def __init__(self): + #: A dict of registered commands. Keys are the class names, + #: lowercased (i.e., the command names), and values are instances + #: of the command objects. + self.commands = dict() - #: A dict of registered commands. Keys are the class names, - #: lowercased (i.e., the command names), and values are instances - #: of the command objects. - commands = dict() + #: A list of options that should be added to the option parser + #: in order to handle registered subcommands. + self.subcommand_options = [] - options = [] + #: the help command + self.help = Help(self) + self.register_command(self.help) def runcommand(self): """ Run the single command named in ``Bcfg2.Options.setup.subcommand``, which is where :class:`Bcfg2.Options.Subparser` groups store the subcommand. """ + _debug("Running subcommand %s" % master_setup.subcommand) try: return self.commands[master_setup.subcommand].run(master_setup) finally: self.shutdown() def shutdown(self): - """ Perform shutdown tasks. This calls the ``shutdown`` - method of all registered subcommands. """ + """Perform shutdown tasks. + + This calls the ``shutdown`` method of the subcommand that was + run. + """ + _debug("Shutting down subcommand %s" % master_setup.subcommand) self.commands[master_setup.subcommand].shutdown() - @classmethod - def register_command(cls, cmdcls): + def register_command(self, cls_or_obj): """ Register a single command. - :param cmdcls: The command class to register - :type cmdcls: type + :param cls_or_obj: The command class or object to register + :type cls_or_obj: type or Subcommand :returns: An instance of ``cmdcls`` """ - cmd_obj = cmdcls() + if isinstance(cls_or_obj, type): + cmdcls = cls_or_obj + cmd_obj = cmdcls() + else: + cmd_obj = cls_or_obj + cmdcls = cmd_obj.__class__ name = cmdcls.__name__.lower() - cls.commands[name] = cmd_obj + self.commands[name] = cmd_obj # py2.5 can't mix *magic and non-magical keyword args, thus # the **dict(...) - cls.options.append( + self.subcommand_options.append( Subparser(*cmdcls.options, **dict(name=name, help=cmdcls.__doc__))) - if issubclass(cls, cmd.Cmd) and cmdcls.interactive: - setattr(cls, "do_%s" % name, cmd_obj) - setattr(cls, "help_%s" % name, cmd_obj.parser.print_help) + if issubclass(self.__class__, cmd.Cmd) and cmdcls.interactive: + setattr(self, "do_%s" % name, cmd_obj) + setattr(self, "help_%s" % name, cmd_obj.parser.print_help) return cmd_obj - -def register_commands(registry, candidates, parent=Subcommand): - """ Register all subcommands in ``candidates`` against the - :class:`Bcfg2.Options.CommandRegistry` subclass given in - ``registry``. A command is registered if and only if: - - * It is a subclass of the given ``parent`` (by default, - :class:`Bcfg2.Options.Subcommand`); - * It is not the parent class itself; and - * Its name does not start with an underscore. - - :param registry: The :class:`Bcfg2.Options.CommandRegistry` - subclass against which commands will be - registered. - :type registry: Bcfg2.Options.CommandRegistry - :param candidates: A list of objects that will be considered for - registration. Only objects that meet the - criteria listed above will be registered. - :type candidates: list - :param parent: Specify a parent class other than - :class:`Bcfg2.Options.Subcommand` that all - registered commands must subclass. - :type parent: type - """ - for attr in candidates: - try: - if (issubclass(attr, parent) and - attr != parent and - not attr.__name__.startswith("_")): - registry.register_command(attr) - except TypeError: - pass + def register_commands(self, candidates, parent=Subcommand): + """ Register all subcommands in ``candidates`` against the + :class:`Bcfg2.Options.CommandRegistry` subclass given in + ``registry``. A command is registered if and only if: + + * It is a subclass of the given ``parent`` (by default, + :class:`Bcfg2.Options.Subcommand`); + * It is not the parent class itself; and + * Its name does not start with an underscore. + + :param registry: The :class:`Bcfg2.Options.CommandRegistry` + subclass against which commands will be + registered. + :type registry: Bcfg2.Options.CommandRegistry + :param candidates: A list of objects that will be considered for + registration. Only objects that meet the + criteria listed above will be registered. + :type candidates: list + :param parent: Specify a parent class other than + :class:`Bcfg2.Options.Subcommand` that all + registered commands must subclass. + :type parent: type + """ + for attr in candidates: + if (isinstance(attr, type) and + issubclass(attr, parent) and + attr != parent and + not attr.__name__.startswith("_")): + self.register_command(attr) diff --git a/src/lib/Bcfg2/Options/Types.py b/src/lib/Bcfg2/Options/Types.py index d11e54fba..ac099e135 100644 --- a/src/lib/Bcfg2/Options/Types.py +++ b/src/lib/Bcfg2/Options/Types.py @@ -19,12 +19,16 @@ def path(value): def comma_list(value): """ Split a comma-delimited list, with optional whitespace around the commas.""" + if value == '': + return [] return _COMMA_SPLIT_RE.split(value) def colon_list(value): """ Split a colon-delimited list. Whitespace is not allowed around the colons. """ + if value == '': + return [] return value.split(':') @@ -38,9 +42,11 @@ def comma_dict(value): for item in items: if '=' in item: key, value = item.split(r'=', 1) - try: - result[key] = bool(value) - except ValueError: + if value in ["true", "yes", "on"]: + result[key] = True + elif value in ["false", "no", "off"]: + result[key] = False + else: try: result[key] = int(value) except ValueError: @@ -107,8 +113,6 @@ def size(value): """ Given a number of bytes in a human-readable format (e.g., '512m', '2g'), get the absolute number of bytes as an integer. """ - if value == -1: - return value mat = _bytes_re.match(value) if not mat: raise ValueError("Not a valid size", value) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 12c9cdaa8..153809a35 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -1,3 +1,4 @@ +import os import sys import atexit import daemon @@ -5,13 +6,12 @@ import logging import time import threading -# pylint: disable=E0611 from lockfile import LockFailed, LockTimeout +# pylint: disable=E0611 try: - from lockfile.pidlockfile import PIDLockFile - from lockfile import Error as PIDFileError + from daemon.pidfile import TimeoutPIDLockFile except ImportError: - from daemon.pidlockfile import PIDLockFile, PIDFileError + from daemon.pidlockfile import TimeoutPIDLockFile # pylint: enable=E0611 import Bcfg2.Logger @@ -30,7 +30,7 @@ class ReportingError(Exception): class ReportingStoreThread(threading.Thread): """Thread for calling the storage backend""" def __init__(self, interaction, storage, group=None, target=None, - name=None, args=(), kwargs=None): + name=None, semaphore=None, args=(), kwargs=None): """Initialize the thread with a reference to the interaction as well as the storage engine to use""" threading.Thread.__init__(self, group, target, name, args, @@ -38,26 +38,37 @@ class ReportingStoreThread(threading.Thread): self.interaction = interaction self.storage = storage self.logger = logging.getLogger('bcfg2-report-collector') + self.semaphore = semaphore def run(self): """Call the database storage procedure (aka import)""" try: - start = time.time() - self.storage.import_interaction(self.interaction) - self.logger.info("Imported interaction for %s in %ss" % - (self.interaction.get('hostname', '<unknown>'), - time.time() - start)) - except: - #TODO requeue? - self.logger.error("Unhandled exception in import thread %s" % - sys.exc_info()[1]) + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', + '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + sys.exc_info()[1]) + finally: + if self.semaphore: + self.semaphore.release() class ReportingCollector(object): """The collecting process for reports""" options = [Bcfg2.Options.Common.reporting_storage, Bcfg2.Options.Common.reporting_transport, - Bcfg2.Options.Common.daemon] + Bcfg2.Options.Common.daemon, + Bcfg2.Options.Option( + '--max-children', dest="children", + cf=('reporting', 'max_children'), type=int, + default=0, + help='Maximum number of children for the reporting collector')] def __init__(self): """Setup the collector. This may be called by the daemon or though @@ -67,6 +78,11 @@ class ReportingCollector(object): self.children = [] self.cleanup_threshold = 25 + self.semaphore = None + if Bcfg2.Options.setup.children > 0: + self.semaphore = threading.Semaphore( + value=Bcfg2.Options.setup.children) + if Bcfg2.Options.setup.debug: level = logging.DEBUG elif Bcfg2.Options.setup.verbose: @@ -113,25 +129,31 @@ class ReportingCollector(object): if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") + self.context.pidfile = TimeoutPIDLockFile( + Bcfg2.Options.setup.daemon, acquire_timeout=5) + # Attempt to ensure lockfile is able to be created and not stale try: - self.context.pidfile = PIDLockFile(Bcfg2.Options.setup.daemon) - self.context.open() + self.context.pidfile.acquire() except LockFailed: self.logger.error("Failed to daemonize: %s" % sys.exc_info()[1]) self.shutdown() return except LockTimeout: - self.logger.error("Failed to daemonize: " - "Failed to acquire lock on %s" % - self.setup['daemon']) - self.shutdown() - return - except PIDFileError: - self.logger.error("Error writing pid file: %s" % - sys.exc_info()[1]) - self.shutdown() - return + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except (OSError, TypeError): # No process with locked PID + self.context.pidfile.break_lock() + else: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + Bcfg2.Options.setup.daemon) + self.shutdown() + return + else: + self.context.pidfile.release() + + self.context.open() self.logger.info("Starting daemon") self.transport.start_monitor(self) @@ -141,7 +163,10 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - store_thread = ReportingStoreThread(interaction, self.storage) + if self.semaphore: + self.semaphore.acquire() + store_thread = ReportingStoreThread(interaction, self.storage, + semaphore=self.semaphore) store_thread.start() self.children.append(store_thread) diff --git a/src/lib/Bcfg2/Reporting/Reports.py b/src/lib/Bcfg2/Reporting/Reports.py index 219d74584..3b9c83433 100755 --- a/src/lib/Bcfg2/Reporting/Reports.py +++ b/src/lib/Bcfg2/Reporting/Reports.py @@ -267,10 +267,11 @@ class CLI(Bcfg2.Options.CommandRegistry): def __init__(self): Bcfg2.Options.CommandRegistry.__init__(self) - Bcfg2.Options.register_commands(self.__class__, globals().values()) + self.register_commands(globals().values()) parser = Bcfg2.Options.get_parser( description="Query the Bcfg2 reporting subsystem", components=[self]) + parser.add_options(self.subcommand_options) parser.parse() def run(self): diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 406216861..96226c424 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -168,7 +168,7 @@ class DjangoORM(StorageBase): # TODO - vcs output act_dict['detail_type'] = PathEntry.DETAIL_UNUSED if path_type == 'directory' and entry.get('prune', 'false') == 'true': - unpruned_elist = [e.get('path') for e in entry.findall('Prune')] + unpruned_elist = [e.get('name') for e in entry.findall('Prune')] if unpruned_elist: act_dict['detail_type'] = PathEntry.DETAIL_PRUNED act_dict['details'] = "\n".join(unpruned_elist) @@ -367,10 +367,11 @@ class DjangoORM(StorageBase): def import_interaction(self, interaction): """Import the data into the backend""" try: - self._import_interaction(interaction) - except: - self.logger.error("Failed to import interaction: %s" % - traceback.format_exc().splitlines()[-1]) + try: + self._import_interaction(interaction) + except: + self.logger.error("Failed to import interaction: %s" % + traceback.format_exc().splitlines()[-1]) finally: self.logger.debug("%s: Closing database connection" % self.__class__.__name__) diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 2d96990b1..ae6f6731b 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -717,9 +717,6 @@ class PathEntry(SuccessEntry): def has_detail(self): return self.detail_type != PathEntry.DETAIL_UNUSED - def is_sensitive(self): - return self.detail_type == PathEntry.DETAIL_SENSITIVE - def is_diff(self): return self.detail_type == PathEntry.DETAIL_DIFF diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index b03d48045..c6e6df020 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -107,7 +107,7 @@ div.entry_list h3 { {{ item.details|syntaxhilight }} </div> {% else %} - {{ item.details }} + {{ item.details|linebreaks }} {% endif %} </div> {% endif %} diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 4a93e77e0..09aebc7fd 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -111,47 +111,58 @@ def filter_navigator(context): try: path = context['request'].META['PATH_INFO'] view, args, kwargs = resolve(path) + except (Resolver404, KeyError): + return dict() - # Strip any page limits and numbers - if 'page_number' in kwargs: - del kwargs['page_number'] - if 'page_limit' in kwargs: - del kwargs['page_limit'] - - # get a query string - qs = context['request'].GET.urlencode() - if qs: - qs = '?' + qs - - filters = [] - for filter in filter_list: - if filter == 'group': - continue - if filter in kwargs: - myargs = kwargs.copy() - del myargs[filter] + # Strip any page limits and numbers + if 'page_number' in kwargs: + del kwargs['page_number'] + if 'page_limit' in kwargs: + del kwargs['page_limit'] + + # get a query string + qs = context['request'].GET.urlencode() + if qs: + qs = '?' + qs + + filters = [] + for filter in filter_list: + if filter == 'group': + continue + if filter in kwargs: + myargs = kwargs.copy() + del myargs[filter] + try: filters.append((filter, reverse(view, args=args, kwargs=myargs) + qs)) - filters.sort(key=lambda x: x[0]) - - myargs = kwargs.copy() - selected = True - if 'group' in myargs: - del myargs['group'] - selected = False - groups = [('---', - reverse(view, args=args, kwargs=myargs) + qs, - selected)] - for group in Group.objects.values('name'): + except NoReverseMatch: + pass + filters.sort(key=lambda x: x[0]) + + myargs = kwargs.copy() + selected = True + if 'group' in myargs: + del myargs['group'] + selected = False + + groups = [] + try: + groups.append(('---', + reverse(view, args=args, kwargs=myargs) + qs, + selected)) + except NoReverseMatch: + pass + + for group in Group.objects.values('name'): + try: myargs['group'] = group['name'] groups.append((group['name'], reverse(view, args=args, kwargs=myargs) + qs, group['name'] == kwargs.get('group', ''))) + except NoReverseMatch: + pass - return {'filters': filters, 'groups': groups} - except (Resolver404, NoReverseMatch, ValueError, KeyError): - pass - return dict() + return {'filters': filters, 'groups': groups} def _subtract_or_na(mdict, x, y): diff --git a/src/lib/Bcfg2/Reporting/utils.py b/src/lib/Bcfg2/Reporting/utils.py index 0d394fcd8..694f38824 100755 --- a/src/lib/Bcfg2/Reporting/utils.py +++ b/src/lib/Bcfg2/Reporting/utils.py @@ -96,12 +96,12 @@ def filteredUrls(pattern, view, kwargs=None, name=None): tail = mtail.group(1) pattern = pattern[:len(pattern) - len(tail)] for filter in ('/state/(?P<state>\w+)', - '/group/(?P<group>[\w\-\.]+)', - '/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)', - '/server/(?P<server>[\w\-\.]+)', - '/server/(?P<server>[\w\-\.]+)/(?P<state>[A-Za-z]+)', - '/server/(?P<server>[\w\-\.]+)/group/(?P<group>[\w\-\.]+)', - '/server/(?P<server>[\w\-\.]+)/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)'): + '/group/(?P<group>[^/]+)', + '/group/(?P<group>[^/]+)/(?P<state>[A-Za-z]+)', + '/server/(?P<server>[^/]+)', + '/server/(?P<server>[^/]+)/(?P<state>[A-Za-z]+)', + '/server/(?P<server>[^/]+)/group/(?P<group>[^/]+)', + '/server/(?P<server>[^/]+)/group/(?P<group>[^/]+)/(?P<state>[A-Za-z]+)'): results += [(pattern + filter + tail, view, kwargs)] return results diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 0807fb2b0..ef7741880 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -439,15 +439,6 @@ class Compare(AdminCmd): print("") -class Help(AdminCmd, Bcfg2.Options.HelpCommand): - """ Get help on a specific subcommand """ - def command_registry(self): - return CLI.commands - - def run(self, setup): - Bcfg2.Options.HelpCommand.run(self, setup) - - class Init(AdminCmd): """Interactively initialize a new repository.""" @@ -1194,16 +1185,20 @@ class Xcmd(_ProxyAdminCmd): class CLI(Bcfg2.Options.CommandRegistry): """ CLI class for bcfg2-admin """ + def __init__(self): Bcfg2.Options.CommandRegistry.__init__(self) - Bcfg2.Options.register_commands(self.__class__, globals().values(), - parent=AdminCmd) + self.register_commands(globals().values(), parent=AdminCmd) parser = Bcfg2.Options.get_parser( description="Manage a running Bcfg2 server", components=[self]) + parser.add_options(self.subcommand_options) parser.parse() def run(self): """ Run bcfg2-admin """ - self.commands[Bcfg2.Options.setup.subcommand].setup() - return self.runcommand() + try: + self.commands[Bcfg2.Options.setup.subcommand].setup() + return self.runcommand() + finally: + self.shutdown() diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 769addf55..e138c57e4 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -1,5 +1,6 @@ """ The core of the builtin Bcfg2 server. """ +import os import sys import time import socket @@ -85,20 +86,30 @@ class BuiltinCore(NetworkCore): def _daemonize(self): """ Open :attr:`context` to drop privileges, write the PID file, and daemonize the server core. """ + # Attempt to ensure lockfile is able to be created and not stale try: - self.context.open() - self.logger.info("%s daemonized" % self.name) - return True + self.context.pidfile.acquire() except LockFailed: err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False except LockTimeout: - err = sys.exc_info()[1] - self.logger.error("Failed to daemonize %s: Failed to acquire lock " - "on %s" % (self.name, - Bcfg2.Options.setup.daemon)) - return False + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except (OSError, TypeError): # No process with locked PID + self.context.pidfile.break_lock() + else: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: Failed to acquire" + "lock on %s" % (self.name, + Bcfg2.Options.setup.daemon)) + return False + else: + self.context.pidfile.release() + + self.context.open() + self.logger.info("%s daemonized" % self.name) + return True def _run(self): """ Create :attr:`server` to start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 892f2832a..bc305e47a 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -84,7 +84,7 @@ def close_db_connection(func): if self._database_available: # pylint: disable=W0212 from django import db self.logger.debug("%s: Closing database connection" % - threading.current_thread().name) + threading.current_thread().getName()) db.close_connection() return rv @@ -783,13 +783,13 @@ class Core(object): for plug in self.plugins_by_type(Threaded): plug.start_threads() + + self.block_for_fam_events() + self._block() except: self.shutdown() raise - self.block_for_fam_events() - self._block() - def _run(self): """ Start up the server; this method should return immediately. This must be overridden by a core diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index d0fd70c5c..8e0dd2efe 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -238,6 +238,8 @@ class FileMonitor(Debuggable): self.handles[event.requestID])) try: self.handles[event.requestID].HandleEvent(event) + except KeyboardInterrupt: + raise except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Error in handling of event %s for %s: %s" % diff --git a/src/lib/Bcfg2/Server/Info.py b/src/lib/Bcfg2/Server/Info.py index a5136f01d..6af561089 100644 --- a/src/lib/Bcfg2/Server/Info.py +++ b/src/lib/Bcfg2/Server/Info.py @@ -123,15 +123,6 @@ class InfoCmd(Bcfg2.Options.Subcommand): # pylint: disable=W0223 list(self.core.metadata.groups.keys())) -class Help(InfoCmd, Bcfg2.Options.HelpCommand): - """ Get help on a specific subcommand """ - def command_registry(self): - return self.core.commands - - def run(self, setup): - Bcfg2.Options.HelpCommand.run(self, setup) - - class Debug(InfoCmd): """ Shell out to a Python interpreter """ interpreters, default_interpreter = load_interpreters() @@ -805,15 +796,12 @@ if HAS_PROFILE: display_trace(prof) -class InfoCore(cmd.Cmd, - Bcfg2.Server.Core.Core, - Bcfg2.Options.CommandRegistry): +class InfoCore(cmd.Cmd, Bcfg2.Server.Core.Core): """Main class for bcfg2-info.""" def __init__(self): cmd.Cmd.__init__(self) Bcfg2.Server.Core.Core.__init__(self) - Bcfg2.Options.CommandRegistry.__init__(self) self.prompt = 'bcfg2-info> ' def get_locals(self): @@ -849,20 +837,20 @@ class InfoCore(cmd.Cmd, pass def shutdown(self): - Bcfg2.Options.CommandRegistry.shutdown(self) Bcfg2.Server.Core.Core.shutdown(self) -class CLI(object): +class CLI(Bcfg2.Options.CommandRegistry): """ The bcfg2-info CLI """ options = [Bcfg2.Options.BooleanOption("-p", "--profile", help="Profile")] def __init__(self): - Bcfg2.Options.register_commands(InfoCore, globals().values(), - parent=InfoCmd) + Bcfg2.Options.CommandRegistry.__init__(self) + self.register_commands(globals().values(), parent=InfoCmd) parser = Bcfg2.Options.get_parser( description="Inspect a running Bcfg2 server", components=[self, InfoCore]) + parser.add_options(self.subcommand_options) parser.parse() if Bcfg2.Options.setup.profile and HAS_PROFILE: @@ -874,11 +862,18 @@ class CLI(object): print("Profiling functionality not available.") self.core = InfoCore() - for command in self.core.commands.values(): + for command in self.commands.values(): command.core = self.core def run(self): """ Run bcfg2-info """ - if Bcfg2.Options.setup.subcommand != 'help': - self.core.run() - return self.core.runcommand() + try: + if Bcfg2.Options.setup.subcommand != 'help': + self.core.run() + return self.runcommand() + finally: + self.shutdown() + + def shutdown(self): + Bcfg2.Options.CommandRegistry.shutdown(self) + self.core.shutdown() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 3d5c68e3f..cfabd8457 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -102,7 +102,8 @@ class AptSource(Source): bdeps[barch][pkgname] = [] brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: - self.essentialpkgs.add(pkgname) + if words[1].strip() == 'yes': + self.essentialpkgs.add(pkgname) elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 24db2963d..67ada2399 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -199,6 +199,9 @@ class Source(Debuggable): # pylint: disable=R0902 #: The "version" attribute from :attr:`xsource` self.version = xsource.get('version', '') + #: The "name" attribute from :attr:`xsource` + self.name = xsource.get('name', None) + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -274,11 +277,11 @@ class Source(Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch) + arch=arch, debsrc=self.debsrc) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch)] + arch=arch, debsrc=self.debsrc)] for setting in usettings: if not self.rawurl: @@ -286,6 +289,7 @@ class Source(Debuggable): # pylint: disable=R0902 else: setting['baseurl'] = self.rawurl setting['url'] = baseurl % setting + setting['name'] = self.get_repo_name(setting) self.url_map.extend(usettings) @property @@ -353,7 +357,7 @@ class Source(Debuggable): # pylint: disable=R0902 if os.path.exists(self.cachefile): try: self.load_state() - except: + except (OSError, cPickle.UnpicklingError): err = sys.exc_info()[1] self.logger.error("Packages: Cachefile %s load failed: %s" % (self.cachefile, err)) @@ -388,8 +392,10 @@ class Source(Debuggable): # pylint: disable=R0902 doing other operations that require repository names. This function tries several approaches: - #. First, if the map contains a ``component`` key, use that as - the name. + #. First, if the source element containts a ``name`` attribute, + use that as the name. + #. If the map contains a ``component`` key, use that as the + name. #. If not, then try to match the repository URL against :attr:`Bcfg2.Server.Plugins.Packages.Source.REPO_RE`. If that succeeds, use the first matched group; additionally, @@ -419,6 +425,9 @@ class Source(Debuggable): # pylint: disable=R0902 :type url_map: dict :returns: string - the name of the repository. """ + if self.name: + return self.name + if url_map['component']: rname = url_map['component'] else: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py index 48304d26e..f26d6ba18 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py @@ -383,10 +383,10 @@ class CLI(Bcfg2.Options.CommandRegistry): def __init__(self): Bcfg2.Options.CommandRegistry.__init__(self) - Bcfg2.Options.register_commands(self.__class__, globals().values(), - parent=HelperSubcommand) + self.register_commands(globals().values(), parent=HelperSubcommand) parser = Bcfg2.Options.get_parser("Bcfg2 yum helper", components=[self]) + parser.add_options(self.subcommand_options) parser.parse() self.logger = logging.getLogger(parser.prog) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index d11ac60fe..cb533f4f1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -33,6 +33,7 @@ class PackagesBackendAction(Bcfg2.Options.ComponentAction): """ ComponentAction to load Packages backends """ bases = ['Bcfg2.Server.Plugins.Packages'] module = True + fail_silently = True class OnDemandDict(MutableMapping): diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index b752650f0..2ca518e53 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -18,12 +18,6 @@ class Svn(Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" options = Bcfg2.Server.Plugin.Version.options + [ Bcfg2.Options.Option( - cf=("svn", "conflict_resolution"), dest="svn_conflict_resolution", - type=lambda v: v.replace("-", "_"), - choices=dir(pysvn.wc_conflict_choice), # pylint: disable=E1101 - default=pysvn.wc_conflict_choice.postpone, # pylint: disable=E1101 - help="SVN conflict resolution method"), - Bcfg2.Options.Option( cf=("svn", "user"), dest="svn_user", help="SVN username"), Bcfg2.Options.Option( cf=("svn", "password"), dest="svn_password", help="SVN password"), @@ -31,6 +25,18 @@ class Svn(Bcfg2.Server.Plugin.Version): cf=("svn", "always_trust"), dest="svn_trust_ssl", help="Always trust SSL certs from SVN server")] + if HAS_SVN: + options.append( + Bcfg2.Options.Option( + cf=("svn", "conflict_resolution"), + dest="svn_conflict_resolution", + type=lambda v: v.replace("-", "_"), + # pylint: disable=E1101 + choices=dir(pysvn.wc_conflict_choice), + default=pysvn.wc_conflict_choice.postpone, + # pylint: enable=E1101 + help="SVN conflict resolution method")) + __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".svn" if HAS_SVN: diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 7150c245a..8d6642a25 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -4,39 +4,19 @@ import sys import logging import Bcfg2.Options import Bcfg2.Server.Plugins -from Bcfg2.Compat import walk_packages -LOGGER = logging.getLogger('Bcfg2.Server.models') +LOGGER = logging.getLogger(__name__) MODELS = [] -def _get_all_plugins(): - rv = [] - for submodule in walk_packages(path=Bcfg2.Server.Plugins.__path__, - prefix="Bcfg2.Server.Plugins."): - module = submodule[1].rsplit('.', 1)[-1] - if submodule[1] == "Bcfg2.Server.Plugins.%s" % module: - # we only include direct children of - # Bcfg2.Server.Plugins -- e.g., all_plugins should - # include Bcfg2.Server.Plugins.Cfg, but not - # Bcfg2.Server.Plugins.Cfg.CfgInfoXML - rv.append(module) - return rv - - -_ALL_PLUGINS = _get_all_plugins() - - class _OptionContainer(object): + """Options for Bcfg2 database models.""" + # we want to provide a different default plugin list -- # namely, _all_ plugins, so that the database is guaranteed to # work, even if /etc/bcfg2.conf isn't set up properly - options = [ - Bcfg2.Options.Option( - cf=('server', 'plugins'), type=Bcfg2.Options.Types.comma_list, - default=_ALL_PLUGINS, dest="models_plugins", - action=Bcfg2.Options.PluginsAction)] + options = [Bcfg2.Options.Common.plugins] @staticmethod def options_parsed_hook(): @@ -58,7 +38,7 @@ def load_models(plugins=None): global MODELS if not plugins: - plugins = Bcfg2.Options.setup.models_plugins + plugins = Bcfg2.Options.setup.plugins if MODELS: # load_models() has been called once, so first unload all of diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py index 0e9e3a141..654e792b8 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/Test_init.py @@ -17,26 +17,26 @@ while path != "/": path = os.path.dirname(path) from common import * -# try to find true -if os.path.exists("/bin/true"): - TRUE = "/bin/true" -elif os.path.exists("/usr/bin/true"): - TRUE = "/usr/bin/true" -else: - TRUE = None - class TestTool(Bcfg2TestCase): test_obj = Tool + # try to find true + if os.path.exists("/bin/true"): + true = "/bin/true" + elif os.path.exists("/usr/bin/true"): + true = "/usr/bin/true" + else: + true = None + def setUp(self): set_setup_default('command_timeout') set_setup_default('interactive', False) + set_setup_default('decision') def get_obj(self, config=None): if config is None: config = lxml.etree.Element("Configuration") - execs = self.test_obj.__execs__ self.test_obj.__execs__ = [] rv = self.test_obj(config) @@ -77,11 +77,11 @@ class TestTool(Bcfg2TestCase): ["/test"] + [e.get("name") for e in important]) t.getSupportedEntries.assert_called_with() - @skipIf(TRUE is None, "/bin/true or equivalent not found") + @skipIf(true is None, "/bin/true or equivalent not found") def test__check_execs(self): t = self.get_obj() if t.__execs__ == []: - t.__execs__.append(TRUE) + t.__execs__.append(self.true) @patch("os.stat") def inner(mock_stat): diff --git a/testsuite/Testsrc/Testlib/TestOptions/One.py b/testsuite/Testsrc/Testlib/TestOptions/One.py new file mode 100644 index 000000000..dac7f4558 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestOptions/One.py @@ -0,0 +1,6 @@ +"""Test module for component loading.""" + + +class One(object): + """Test class for component loading.""" + pass diff --git a/testsuite/Testsrc/Testlib/TestOptions/TestComponents.py b/testsuite/Testsrc/Testlib/TestOptions/TestComponents.py new file mode 100644 index 000000000..61b87de2a --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestOptions/TestComponents.py @@ -0,0 +1,229 @@ +"""test component loading.""" + +import argparse +import os + +from Bcfg2.Options import Option, BooleanOption, PathOption, ComponentAction, \ + get_parser, new_parser, Types, ConfigFileAction, Common + +from testsuite.Testsrc.Testlib.TestOptions import make_config, One, Two, \ + OptionTestCase + + +# create a bunch of fake components for testing component loading options + +class ChildOne(object): + """fake component for testing component loading.""" + options = [Option("--child-one")] + + +class ChildTwo(object): + """fake component for testing component loading.""" + options = [Option("--child-two")] + + +class ChildComponentAction(ComponentAction): + """child component loader action.""" + islist = False + mapping = {"one": ChildOne, + "two": ChildTwo} + + +class ComponentOne(object): + """fake component for testing component loading.""" + options = [BooleanOption("--one")] + + +class ComponentTwo(object): + """fake component for testing component loading.""" + options = [Option("--child", default="one", action=ChildComponentAction)] + + +class ComponentThree(object): + """fake component for testing component loading.""" + options = [BooleanOption("--three")] + + +class ConfigFileComponent(object): + """fake component for testing component loading.""" + options = [Option("--config2", action=ConfigFileAction), + Option(cf=("config", "test"), dest="config2_test", + default="bar")] + + +class PathComponent(object): + """fake component for testing <repository> macros in child components.""" + options = [PathOption(cf=("test", "test_path")), + PathOption(cf=("test", "test_path_default"), + default="<repository>/test/default")] + + +class ParentComponentAction(ComponentAction): + """parent component loader action.""" + mapping = {"one": ComponentOne, + "two": ComponentTwo, + "three": ComponentThree, + "config": ConfigFileComponent, + "path": PathComponent} + + +class TestComponentOptions(OptionTestCase): + """test cases for component loading.""" + + def setUp(self): + OptionTestCase.setUp(self) + self.options = [ + Option("--parent", type=Types.comma_list, + default=["one", "two"], action=ParentComponentAction)] + + self.result = argparse.Namespace() + new_parser() + self.parser = get_parser(components=[self], namespace=self.result, + description="component testing parser") + + @make_config() + def test_loading_components(self, config_file): + """load a single component during option parsing.""" + self.parser.parse(["-C", config_file, "--parent", "one"]) + self.assertEqual(self.result.parent, [ComponentOne]) + + @make_config() + def test_component_option(self, config_file): + """use options from a component loaded during option parsing.""" + self.parser.parse(["--one", "-C", config_file, "--parent", "one"]) + self.assertEqual(self.result.parent, [ComponentOne]) + self.assertTrue(self.result.one) + + @make_config() + def test_multi_component_load(self, config_file): + """load multiple components during option parsing.""" + self.parser.parse(["-C", config_file, "--parent", "one,three"]) + self.assertEqual(self.result.parent, [ComponentOne, ComponentThree]) + + @make_config() + def test_multi_component_options(self, config_file): + """use options from multiple components during option parsing.""" + self.parser.parse(["-C", config_file, "--three", + "--parent", "one,three", "--one"]) + self.assertEqual(self.result.parent, [ComponentOne, ComponentThree]) + self.assertTrue(self.result.one) + self.assertTrue(self.result.three) + + @make_config() + def test_component_default_not_loaded(self, config_file): + """options from default but unused components not available.""" + self.assertRaises( + SystemExit, + self.parser.parse, + ["-C", config_file, "--child", "one", "--parent", "one"]) + + @make_config() + def test_tiered_components(self, config_file): + """load child component.""" + self.parser.parse(["-C", config_file, "--parent", "two", + "--child", "one"]) + self.assertEqual(self.result.parent, [ComponentTwo]) + self.assertEqual(self.result.child, ChildOne) + + @make_config() + def test_options_tiered_components(self, config_file): + """use options from child component.""" + self.parser.parse(["--child-one", "foo", "-C", config_file, "--parent", + "two", "--child", "one"]) + self.assertEqual(self.result.parent, [ComponentTwo]) + self.assertEqual(self.result.child, ChildOne) + self.assertEqual(self.result.child_one, "foo") + + @make_config() + def test_bogus_component(self, config_file): + """error out with bad component name.""" + self.assertRaises(SystemExit, + self.parser.parse, + ["-C", config_file, "--parent", "blargle"]) + + @make_config() + @make_config({"config": {"test": "foo"}}) + def test_config_component(self, config1, config2): + """load component with alternative config file.""" + self.parser.parse(["-C", config1, "--config2", config2, + "--parent", "config"]) + self.assertEqual(self.result.config2, config2) + self.assertEqual(self.result.config2_test, "foo") + + @make_config() + def test_config_component_no_file(self, config_file): + """load component with missing alternative config file.""" + self.parser.parse(["-C", config_file, "--parent", "config"]) + self.assertEqual(self.result.config2, None) + + @make_config({"test": {"test_path": "<repository>/test"}}) + def test_macros_in_component_options(self, config_file): + """fix up <repository> macros in component options.""" + self.parser.add_options([Common.repository]) + self.parser.parse(["-C", config_file, "-Q", "/foo/bar", + "--parent", "path"]) + self.assertEqual(self.result.test_path, "/foo/bar/test") + self.assertEqual(self.result.test_path_default, + "/foo/bar/test/default") + + +class ImportComponentAction(ComponentAction): + """action that imports real classes for testing.""" + islist = False + bases = ["testsuite.Testsrc.Testlib.TestOptions"] + + +class ImportModuleAction(ImportComponentAction): + """action that only imports modules for testing.""" + module = True + + +class TestImportComponentOptions(OptionTestCase): + """test cases for component loading.""" + + def setUp(self): + self.options = [Option("--cls", action=ImportComponentAction), + Option("--module", action=ImportModuleAction)] + + self.result = argparse.Namespace() + new_parser() + self.parser = get_parser(components=[self], namespace=self.result) + + @make_config() + def test_import_component(self, config_file): + """load class components by importing.""" + self.parser.parse(["-C", config_file, "--cls", "One"]) + self.assertEqual(self.result.cls, One.One) + + @make_config() + def test_import_module(self, config_file): + """load module components by importing.""" + self.parser.parse(["-C", config_file, "--module", "One"]) + self.assertEqual(self.result.module, One) + + @make_config() + def test_import_full_path(self, config_file): + """load components by importing the full path.""" + self.parser.parse(["-C", config_file, "--cls", "os.path"]) + self.assertEqual(self.result.cls, os.path) + + @make_config() + def test_import_bogus_class(self, config_file): + """fail to load class component that cannot be imported.""" + self.assertRaises(SystemExit, + self.parser.parse, + ["-C", config_file, "--cls", "Three"]) + + @make_config() + def test_import_bogus_module(self, config_file): + """fail to load module component that cannot be imported.""" + self.assertRaises(SystemExit, + self.parser.parse, + ["-C", config_file, "--module", "Three"]) + + @make_config() + def test_import_bogus_path(self, config_file): + """fail to load component that cannot be imported by full path.""" + self.assertRaises(SystemExit, + self.parser.parse, + ["-C", config_file, "--cls", "Bcfg2.No.Such.Thing"]) diff --git a/testsuite/Testsrc/Testlib/TestOptions/TestConfigFiles.py b/testsuite/Testsrc/Testlib/TestOptions/TestConfigFiles.py new file mode 100644 index 000000000..78acadf1f --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestOptions/TestConfigFiles.py @@ -0,0 +1,50 @@ +"""test reading multiple config files.""" + +import argparse + +import mock + +from Bcfg2.Options import Option, PathOption, ConfigFileAction, get_parser, \ + new_parser +from testsuite.Testsrc.Testlib.TestOptions import make_config, OptionTestCase + + +class TestConfigFiles(OptionTestCase): + def setUp(self): + self.options = [ + PathOption(cf=("test", "config2"), action=ConfigFileAction), + PathOption(cf=("test", "config3"), action=ConfigFileAction), + Option(cf=("test", "foo")), + Option(cf=("test", "bar")), + Option(cf=("test", "baz"))] + self.results = argparse.Namespace() + new_parser() + self.parser = get_parser(components=[self], namespace=self.results) + + @make_config({"test": {"baz": "baz"}}) + def test_config_files(self, config3): + """read multiple config files.""" + # Because make_config() generates temporary files for the + # configuration, we have to work backwards here. first we + # generate config3, then we generate config2 (which includes a + # reference to config3), then we finally generate the main + # config file, which contains a reference to config2. oh how + # I wish we could use context managers here... + + @make_config({"test": {"bar": "bar", "config3": config3}}) + def inner1(config2): + @make_config({"test": {"foo": "foo", "config2": config2}}) + def inner2(config): + self.parser.parse(["-C", config]) + self.assertEqual(self.results.foo, "foo") + self.assertEqual(self.results.bar, "bar") + self.assertEqual(self.results.baz, "baz") + + inner2() + + inner1() + + @mock.patch("os.path.exists", mock.Mock(return_value=False)) + def test_no_config_file(self): + """fail to read config file.""" + self.assertRaises(SystemExit, self.parser.parse, []) diff --git a/testsuite/Testsrc/Testlib/TestOptions/TestOptionGroups.py b/testsuite/Testsrc/Testlib/TestOptions/TestOptionGroups.py new file mode 100644 index 000000000..7611d6202 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestOptions/TestOptionGroups.py @@ -0,0 +1,155 @@ +"""test reading multiple config files.""" + +import argparse +import sys + +from Bcfg2.Options import Option, BooleanOption, Parser, OptionGroup, \ + ExclusiveOptionGroup, WildcardSectionGroup, new_parser, get_parser + +from testsuite.common import Bcfg2TestCase, skipUnless +from testsuite.Testsrc.Testlib.TestOptions import make_config, OptionTestCase + + +class TestOptionGroups(Bcfg2TestCase): + def setUp(self): + self.options = None + + def _test_options(self, options): + """test helper.""" + result = argparse.Namespace() + parser = Parser(components=[self], namespace=result) + parser.parse(options) + return result + + def test_option_group(self): + """basic option group functionality.""" + self.options = [OptionGroup(BooleanOption("--foo"), + BooleanOption("--bar"), + BooleanOption("--baz"), + title="group")] + result = self._test_options(["--foo", "--bar"]) + self.assertTrue(result.foo) + self.assertTrue(result.bar) + self.assertFalse(result.baz) + + def test_exclusive_option_group(self): + """parse options from exclusive option group.""" + self.options = [ + ExclusiveOptionGroup(BooleanOption("--foo"), + BooleanOption("--bar"), + BooleanOption("--baz"))] + result = self._test_options(["--foo"]) + self.assertTrue(result.foo) + self.assertFalse(result.bar) + self.assertFalse(result.baz) + + self.assertRaises(SystemExit, + self._test_options, ["--foo", "--bar"]) + + def test_required_exclusive_option_group(self): + """parse options from required exclusive option group.""" + self.options = [ + ExclusiveOptionGroup(BooleanOption("--foo"), + BooleanOption("--bar"), + BooleanOption("--baz"), + required=True)] + result = self._test_options(["--foo"]) + self.assertTrue(result.foo) + self.assertFalse(result.bar) + self.assertFalse(result.baz) + + self.assertRaises(SystemExit, self._test_options, []) + + +class TestNestedOptionGroups(TestOptionGroups): + def setUp(self): + TestOptionGroups.setUp(self) + self.options = [ + OptionGroup( + BooleanOption("--foo"), + BooleanOption("--bar"), + OptionGroup( + BooleanOption("--baz"), + BooleanOption("--quux"), + ExclusiveOptionGroup( + BooleanOption("--test1"), + BooleanOption("--test2")), + title="inner"), + title="outer")] + + def test_option_group(self): + """nest option groups.""" + result = self._test_options(["--foo", "--baz", "--test1"]) + self.assertTrue(result.foo) + self.assertFalse(result.bar) + self.assertTrue(result.baz) + self.assertFalse(result.quux) + self.assertTrue(result.test1) + self.assertFalse(result.test2) + + @skipUnless(sys.version_info >= (2, 7), + "Nested exclusive option groups do not work in Python 2.6") + def test_nested_exclusive_option_groups(self): + """nest exclusive option groups.""" + self.assertRaises(SystemExit, + self._test_options, ["--test1", "--test2"]) + + +class TestWildcardSectionGroups(OptionTestCase): + config = { + "four:one": { + "foo": "foo one", + "bar": "bar one", + "baz": "baz one" + }, + "four:two": { + "foo": "foo two", + "bar": "bar two" + }, + "five:one": { + "foo": "foo one", + "bar": "bar one" + }, + "five:two": { + "foo": "foo two", + "bar": "bar two" + }, + "five:three": { + "foo": "foo three", + "bar": "bar three" + } + } + + def setUp(self): + self.options = [ + WildcardSectionGroup( + Option(cf=("four:*", "foo")), + Option(cf=("four:*", "bar"))), + WildcardSectionGroup( + Option(cf=("five:*", "foo")), + Option(cf=("five:*", "bar")), + prefix="", + dest="sections")] + self.results = argparse.Namespace() + new_parser() + self.parser = get_parser(components=[self], namespace=self.results) + + @make_config(config) + def test_wildcard_section_groups(self, config_file): + """parse options from wildcard section groups.""" + self.parser.parse(["-C", config_file]) + self.assertEqual(self.results.four_four_one_foo, "foo one") + self.assertEqual(self.results.four_four_one_bar, "bar one") + self.assertEqual(self.results.four_four_two_foo, "foo two") + self.assertEqual(self.results.four_four_two_bar, "bar two") + self.assertItemsEqual(self.results.four_sections, + ["four:one", "four:two"]) + + self.assertEqual(self.results.five_one_foo, "foo one") + self.assertEqual(self.results.five_one_bar, "bar one") + self.assertEqual(self.results.five_two_foo, "foo two") + self.assertEqual(self.results.five_two_bar, "bar two") + self.assertEqual(self.results.five_three_foo, "foo three") + self.assertEqual(self.results.five_three_bar, "bar three") + self.assertItemsEqual(self.results.sections, + ["five:one", "five:two", "five:three"]) diff --git a/testsuite/Testsrc/Testlib/TestOptions/TestOptions.py b/testsuite/Testsrc/Testlib/TestOptions/TestOptions.py new file mode 100644 index 000000000..a2dc8ffe2 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestOptions/TestOptions.py @@ -0,0 +1,500 @@ +"""basic option parsing tests.""" + +import argparse +import os +import tempfile + +import mock + +from Bcfg2.Compat import ConfigParser +from Bcfg2.Options import Option, PathOption, RepositoryMacroOption, \ + BooleanOption, Parser, PositionalArgument, OptionParserException, \ + Common, new_parser, get_parser +from testsuite.Testsrc.Testlib.TestOptions import OptionTestCase, \ + make_config, clean_environment + + +class TestBasicOptions(OptionTestCase): + """test basic option parsing.""" + def setUp(self): + # parsing options can modify the Option objects themselves. + # that's probably bad -- and it's definitely bad if we ever + # want to do real on-the-fly config changes -- but it's easier + # to leave it as is and set the options on each test. + OptionTestCase.setUp(self) + self.options = [ + BooleanOption("--test-true-boolean", env="TEST_TRUE_BOOLEAN", + cf=("test", "true_boolean"), default=True), + BooleanOption("--test-false-boolean", env="TEST_FALSE_BOOLEAN", + cf=("test", "false_boolean"), default=False), + BooleanOption(cf=("test", "true_config_boolean"), + default=True), + BooleanOption(cf=("test", "false_config_boolean"), + default=False), + Option("--test-option", env="TEST_OPTION", cf=("test", "option"), + default="foo"), + PathOption("--test-path-option", env="TEST_PATH_OPTION", + cf=("test", "path"), default="/test")] + + @clean_environment + def _test_options(self, options=None, env=None, config=None): + """helper to test a set of options. + + returns the namespace from parsing the given CLI options with + the given config and environment. + """ + if config is not None: + config = {"test": config} + if options is None: + options = [] + + @make_config(config) + def inner(config_file): + """do the actual tests, since py2.4 lacks context managers.""" + result = argparse.Namespace() + parser = Parser(components=[self], namespace=result) + parser.parse(argv=["-C", config_file] + options) + return result + + if env is not None: + for name, value in env.items(): + os.environ[name] = value + + return inner() + + def test_expand_path(self): + """expand ~ in path option.""" + options = self._test_options(options=["--test-path-option", + "~/test"]) + self.assertEqual(options.test_path_option, + os.path.expanduser("~/test")) + + def test_canonicalize_path(self): + """get absolute path from path option.""" + options = self._test_options(options=["--test-path-option", + "./test"]) + self.assertEqual(options.test_path_option, + os.path.abspath("./test")) + + @make_config() + def test_default_path_canonicalization(self, config_file): + """canonicalize default PathOption values.""" + testdir = os.path.expanduser("~/test") + result = argparse.Namespace() + parser = Parser(namespace=result) + parser.add_options([PathOption("--test1", default="~/test"), + PathOption(cf=("test", "test2"), + default="~/test"), + Common.repository]) + parser.parse(["-C", config_file]) + self.assertEqual(result.test1, testdir) + self.assertEqual(result.test2, testdir) + + def test_default_bool(self): + """use the default value of boolean options.""" + options = self._test_options() + self.assertTrue(options.test_true_boolean) + self.assertFalse(options.test_false_boolean) + self.assertTrue(options.true_config_boolean) + self.assertFalse(options.false_config_boolean) + + def test_default(self): + """use the default value of an option.""" + options = self._test_options() + self.assertEqual(options.test_option, "foo") + + def test_default_path(self): + """use the default value of a path option.""" + options = self._test_options() + self.assertEqual(options.test_path_option, "/test") + + def test_invalid_boolean(self): + """set boolean to invalid values.""" + self.assertRaises(ValueError, + self._test_options, + config={"true_boolean": "you betcha"}) + self.assertRaises(ValueError, + self._test_options, + env={"TEST_TRUE_BOOLEAN": "hell no"}) + + def test_set_boolean_in_config(self): + """set boolean options in config files.""" + set_to_defaults = {"true_boolean": "1", + "false_boolean": "0", + "true_config_boolean": "yes", + "false_config_boolean": "no"} + options = self._test_options(config=set_to_defaults) + self.assertTrue(options.test_true_boolean) + self.assertFalse(options.test_false_boolean) + self.assertTrue(options.true_config_boolean) + self.assertFalse(options.false_config_boolean) + + set_to_other = {"true_boolean": "false", + "false_boolean": "true", + "true_config_boolean": "off", + "false_config_boolean": "on"} + options = self._test_options(config=set_to_other) + self.assertFalse(options.test_true_boolean) + self.assertTrue(options.test_false_boolean) + self.assertFalse(options.true_config_boolean) + self.assertTrue(options.false_config_boolean) + + def test_set_in_config(self): + """set options in config files.""" + options = self._test_options(config={"option": "foo"}) + self.assertEqual(options.test_option, "foo") + + options = self._test_options(config={"option": "bar"}) + self.assertEqual(options.test_option, "bar") + + def test_set_path_in_config(self): + """set path options in config files.""" + options = self._test_options(config={"path": "/test"}) + self.assertEqual(options.test_path_option, "/test") + + options = self._test_options(config={"path": "/foo"}) + self.assertEqual(options.test_path_option, "/foo") + + def test_set_boolean_in_env(self): + """set boolean options in environment.""" + set_to_defaults = {"TEST_TRUE_BOOLEAN": "1", + "TEST_FALSE_BOOLEAN": "0"} + options = self._test_options(env=set_to_defaults) + self.assertTrue(options.test_true_boolean) + self.assertFalse(options.test_false_boolean) + + set_to_other = {"TEST_TRUE_BOOLEAN": "false", + "TEST_FALSE_BOOLEAN": "true"} + options = self._test_options(env=set_to_other) + self.assertFalse(options.test_true_boolean) + self.assertTrue(options.test_false_boolean) + + def test_set_in_env(self): + """set options in environment.""" + options = self._test_options(env={"TEST_OPTION": "foo"}) + self.assertEqual(options.test_option, "foo") + + options = self._test_options(env={"TEST_OPTION": "bar"}) + self.assertEqual(options.test_option, "bar") + + def test_set_path_in_env(self): + """set path options in environment.""" + options = self._test_options(env={"TEST_PATH_OPTION": "/test"}) + self.assertEqual(options.test_path_option, "/test") + + options = self._test_options(env={"TEST_PATH_OPTION": "/foo"}) + self.assertEqual(options.test_path_option, "/foo") + + def test_set_boolean_in_cli(self): + """set boolean options in CLI options.""" + # passing the option yields the reverse of the default, no + # matter the default + options = self._test_options(options=["--test-true-boolean", + "--test-false-boolean"]) + self.assertFalse(options.test_true_boolean) + self.assertTrue(options.test_false_boolean) + + def test_set_in_cli(self): + """set options in CLI options.""" + options = self._test_options(options=["--test-option", "foo"]) + self.assertEqual(options.test_option, "foo") + + options = self._test_options(options=["--test-option", "bar"]) + self.assertEqual(options.test_option, "bar") + + def test_set_path_in_cli(self): + """set path options in CLI options.""" + options = self._test_options(options=["--test-path-option", "/test"]) + self.assertEqual(options.test_path_option, "/test") + + options = self._test_options(options=["--test-path-option", "/foo"]) + self.assertEqual(options.test_path_option, "/foo") + + def test_env_overrides_config_bool(self): + """setting boolean option in the environment overrides config file.""" + config = {"true_boolean": "false", + "false_boolean": "true"} + env = {"TEST_TRUE_BOOLEAN": "yes", + "TEST_FALSE_BOOLEAN": "no"} + options = self._test_options(config=config, env=env) + self.assertTrue(options.test_true_boolean) + self.assertFalse(options.test_false_boolean) + + def test_env_overrides_config(self): + """setting option in the environment overrides config file.""" + options = self._test_options(config={"option": "bar"}, + env={"TEST_OPTION": "baz"}) + self.assertEqual(options.test_option, "baz") + + def test_env_overrides_config_path(self): + """setting path option in the environment overrides config file.""" + options = self._test_options(config={"path": "/foo"}, + env={"TEST_PATH_OPTION": "/bar"}) + self.assertEqual(options.test_path_option, "/bar") + + def test_cli_overrides_config_bool(self): + """setting boolean option in the CLI overrides config file.""" + config = {"true_boolean": "on", + "false_boolean": "off"} + options = ["--test-true-boolean", "--test-false-boolean"] + options = self._test_options(options=options, config=config) + self.assertFalse(options.test_true_boolean) + self.assertTrue(options.test_false_boolean) + + def test_cli_overrides_config(self): + """setting option in the CLI overrides config file.""" + options = self._test_options(options=["--test-option", "baz"], + config={"option": "bar"}) + self.assertEqual(options.test_option, "baz") + + def test_cli_overrides_config_path(self): + """setting path option in the CLI overrides config file.""" + options = self._test_options(options=["--test-path-option", "/bar"], + config={"path": "/foo"}) + self.assertEqual(options.test_path_option, "/bar") + + def test_cli_overrides_env_bool(self): + """setting boolean option in the CLI overrides environment.""" + env = {"TEST_TRUE_BOOLEAN": "0", + "TEST_FALSE_BOOLEAN": "1"} + options = ["--test-true-boolean", "--test-false-boolean"] + options = self._test_options(options=options, env=env) + self.assertFalse(options.test_true_boolean) + self.assertTrue(options.test_false_boolean) + + def test_cli_overrides_env(self): + """setting option in the CLI overrides environment.""" + options = self._test_options(options=["--test-option", "baz"], + env={"TEST_OPTION": "bar"}) + self.assertEqual(options.test_option, "baz") + + def test_cli_overrides_env_path(self): + """setting path option in the CLI overrides environment.""" + options = self._test_options(options=["--test-path-option", "/bar"], + env={"TEST_PATH_OPTION": "/foo"}) + self.assertEqual(options.test_path_option, "/bar") + + def test_cli_overrides_all_bool(self): + """setting boolean option in the CLI overrides everything else.""" + config = {"true_boolean": "no", + "false_boolean": "yes"} + env = {"TEST_TRUE_BOOLEAN": "0", + "TEST_FALSE_BOOLEAN": "1"} + options = ["--test-true-boolean", "--test-false-boolean"] + options = self._test_options(options=options, env=env) + self.assertFalse(options.test_true_boolean) + self.assertTrue(options.test_false_boolean) + + def test_cli_overrides_all(self): + """setting option in the CLI overrides everything else.""" + options = self._test_options(options=["--test-option", "baz"], + env={"TEST_OPTION": "bar"}, + config={"test": "quux"}) + self.assertEqual(options.test_option, "baz") + + def test_cli_overrides_all_path(self): + """setting path option in the CLI overrides everything else.""" + options = self._test_options(options=["--test-path-option", "/bar"], + env={"TEST_PATH_OPTION": "/foo"}, + config={"path": "/baz"}) + self.assertEqual(options.test_path_option, "/bar") + + @make_config() + def _test_dest(self, *args, **kwargs): + """helper to test that ``dest`` is set properly.""" + args = list(args) + expected = args.pop(0) + config_file = args.pop() + + sentinel = object() + kwargs["default"] = sentinel + + result = argparse.Namespace() + parser = Parser(namespace=result) + parser.add_options([Option(*args, **kwargs)]) + parser.parse(["-C", config_file]) + + self.assertTrue(hasattr(result, expected)) + self.assertEqual(getattr(result, expected), sentinel) + + def test_explicit_dest(self): + """set the ``dest`` of an option explicitly.""" + self._test_dest("bar", dest="bar") + + def test_dest_from_env_var(self): + """set the ``dest`` of an option from the env var name.""" + self._test_dest("foo", env="FOO") + + def test_dest_from_cf(self): + """set the ``dest`` of an option from the config option.""" + self._test_dest("foo_bar", cf=("test", "foo-bar")) + + def test_dest_from_cli(self): + """set the ``dest`` of an option from the CLI option.""" + self._test_dest("test_foo", "--test-foo") + + def test_dest_from_all(self): + """set the ``dest`` of an option from the best of multiple sources.""" + self._test_dest("foo_baz", cf=("test", "foo-bar"), env="FOO_BAZ") + self._test_dest("xyzzy", + "--xyzzy", cf=("test", "foo-bar"), env="FOO_BAZ") + self._test_dest("quux", + "--xyzzy", cf=("test", "foo-bar"), env="FOO_BAZ", + dest="quux") + + @make_config() + def test_positional_args(self, config_file): + """get values from positional arguments.""" + result = argparse.Namespace() + parser = Parser(namespace=result) + parser.add_options([PositionalArgument("single")]) + parser.parse(["-C", config_file, "single"]) + self.assertEqual(result.single, "single") + + result = argparse.Namespace() + parser = Parser(namespace=result) + parser.add_options([PositionalArgument("one"), + PositionalArgument("two")]) + parser.parse(["-C", config_file, "one", "two"]) + self.assertEqual(result.one, "one") + self.assertEqual(result.two, "two") + + def test_duplicate_cli_option(self): + """add duplicate CLI option.""" + parser = Parser(components=[self]) + self.assertRaises( + argparse.ArgumentError, + parser.add_options, + [Option("--test-option")]) + + def test_duplicate_env_option(self): + """add duplicate environment option.""" + parser = Parser(components=[self]) + self.assertRaises( + OptionParserException, + parser.add_options, + [Option(env="TEST_OPTION")]) + + def test_duplicate_cf_option(self): + """add duplicate config file option.""" + parser = Parser(components=[self]) + self.assertRaises( + OptionParserException, + parser.add_options, + [Option(cf=("test", "option"))]) + + @make_config({"test": {"test_path": "<repository>/test", + "test_macro": "<repository>"}}) + def test_repository_macro(self, config_file): + """fix up <repository> macros.""" + result = argparse.Namespace() + parser = Parser(namespace=result) + parser.add_options([PathOption("--test1"), + RepositoryMacroOption("--test2"), + PathOption(cf=("test", "test_path")), + PathOption(cf=("test", "test_path_default"), + default="<repository>/test/default"), + RepositoryMacroOption(cf=("test", "test_macro")), + RepositoryMacroOption( + cf=("test", "test_macro_default"), + default="<repository>"), + Common.repository]) + parser.parse(["-C", config_file, "-Q", "/foo/bar", + "--test1", "<repository>/test1", + "--test2", "<repository><repository>"]) + self.assertEqual(result.repository, "/foo/bar") + self.assertEqual(result.test1, "/foo/bar/test1") + self.assertEqual(result.test2, "/foo/bar/foo/bar") + self.assertEqual(result.test_macro, "/foo/bar") + self.assertEqual(result.test_macro_default, "/foo/bar") + self.assertEqual(result.test_path, "/foo/bar/test") + self.assertEqual(result.test_path_default, "/foo/bar/test/default") + + @make_config() + def test_file_like_path_option(self, config_file): + """get file-like object from PathOption.""" + result = argparse.Namespace() + parser = Parser(namespace=result) + parser.add_options([PathOption("--test", type=argparse.FileType('r'))]) + + fd, name = tempfile.mkstemp() + fh = os.fdopen(fd, "w") + fh.write("test") + fh.close() + + try: + parser.parse(["-C", config_file, "--test", name]) + self.assertEqual(result.test.name, name) + self.assertEqual(result.test.read(), "test") + finally: + os.unlink(name) + + @clean_environment + @make_config() + def test_unknown_options(self, config_file): + """error on unknown options.""" + parser = Parser(components=[self]) + self.assertRaises(SystemExit, + parser.parse, + ["-C", config_file, "--not-a-real-option"]) + + @clean_environment + @make_config() + def test_reparse(self, config_file): + """reparse options.""" + result = argparse.Namespace() + parser = Parser(components=[self], namespace=result) + parser.parse(["-C", config_file]) + self.assertFalse(result.test_false_boolean) + + parser.parse(["-C", config_file]) + self.assertFalse(result.test_false_boolean) + + parser.reparse() + self.assertFalse(result.test_false_boolean) + + parser.reparse(["-C", config_file, "--test-false-boolean"]) + self.assertTrue(result.test_false_boolean) + + cfp = ConfigParser.ConfigParser() + cfp.add_section("test") + cfp.set("test", "false_boolean", "on") + parser.parse(["-C", config_file]) + cfp.write(open(config_file, "w")) + self.assertTrue(result.test_false_boolean) + + +class TestParsingHooks(OptionTestCase): + """test option parsing hooks.""" + def setUp(self): + self.options_parsed_hook = mock.Mock() + self.options = [BooleanOption("--test", default=False)] + self.results = argparse.Namespace() + new_parser() + self.parser = get_parser(components=[self], namespace=self.results) + + @make_config() + def test_parsing_hooks(self, config_file): + """option parsing hooks are called.""" + self.parser.parse(["-C", config_file]) + self.options_parsed_hook.assert_called_with() + + +class TestEarlyParsingHooks(OptionTestCase): + """test early option parsing hooks.""" + parse_first = True + + def setUp(self): + self.component_parsed_hook = mock.Mock() + self.options = [BooleanOption("--early-test", default=False)] + self.results = argparse.Namespace() + new_parser() + self.parser = get_parser(components=[self], namespace=self.results) + + @make_config() + def test_parsing_hooks(self, config_file): + """early option parsing hooks are called.""" + self.parser.parse(["-C", config_file, "--early-test"]) + self.assertEqual(self.component_parsed_hook.call_count, 1) + early_opts = self.component_parsed_hook.call_args[0][0] + self.assertTrue(early_opts.early_test) diff --git a/testsuite/Testsrc/Testlib/TestOptions/TestSubcommands.py b/testsuite/Testsrc/Testlib/TestOptions/TestSubcommands.py new file mode 100644 index 000000000..65b4c19c0 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestOptions/TestSubcommands.py @@ -0,0 +1,139 @@ +"""test subcommand option parsing.""" + +import argparse +import sys + +from Bcfg2.Compat import StringIO +from Bcfg2.Options import Option, get_parser, new_parser, Subcommand, \ + Subparser, CommandRegistry +import Bcfg2.Options.Subcommands + +from testsuite.Testsrc.Testlib.TestOptions import make_config, OptionTestCase + + +class MockSubcommand(Subcommand): + """fake subcommand that just records the options it was called with.""" + run_options = None + + def run(self, setup): + self.__class__.run_options = setup + + +class One(MockSubcommand): + """fake subcommand for testing.""" + options = [Option("--test-one")] + + +class Two(MockSubcommand): + """fake subcommand for testing.""" + options = [Option("--test-two")] + + +def local_subclass(cls): + """get a subclass of ``cls`` that adds no functionality. + + This can be used to subclass the various test classes above so + that their options don't get modified by option parsing. + """ + return type("Local%s" % cls.__name__, (cls,), {}) + + +class TestSubcommands(OptionTestCase): + """tests for subcommands and subparsers.""" + + def setUp(self): + self.registry = CommandRegistry() + + self.one = local_subclass(One) + self.two = local_subclass(Two) + + self.registry.register_command(self.one) + self.registry.register_command(self.two) + + self.result = argparse.Namespace() + Bcfg2.Options.Subcommands.master_setup = self.result + + new_parser() + self.parser = get_parser(namespace=self.result, + components=[self]) + self.parser.add_options(self.registry.subcommand_options) + + def test_register_commands(self): + """register subcommands.""" + registry = CommandRegistry() + registry.register_commands(globals().values(), + parent=MockSubcommand) + self.assertItemsEqual(registry.commands.keys(), + ["one", "two", "help"]) + self.assertIsInstance(registry.commands['one'], One) + self.assertIsInstance(registry.commands['two'], Two) + + @make_config() + def test_get_subcommand(self, config_file): + """parse simple subcommands.""" + self.parser.parse(["-C", config_file, "localone"]) + self.assertEqual(self.result.subcommand, "localone") + + def test_subcommand_usage(self): + """sane usage message from subcommands.""" + self.assertEqual( + One().usage(), + "one [--test-one TEST_ONE] - fake subcommand for testing.") + + # subclasses do not inherit the docstring from the parent, so + # this tests a command subclass without a docstring, even + # though that should never happen due to the pylint tests. + self.assertEqual(self.one().usage().strip(), + "localone [--test-one TEST_ONE]") + + def _get_subcommand_output(self, args): + self.parser.parse(args) + old_stdout = sys.stdout + sys.stdout = StringIO() + rv = self.registry.runcommand() + output = [l for l in sys.stdout.getvalue().splitlines() + if not l.startswith("DEBUG: ")] + sys.stdout = old_stdout + return (rv, output) + + @make_config() + def test_help(self, config_file): + """sane help message from subcommand registry.""" + rv, output = self._get_subcommand_output(["-C", config_file, "help"]) + self.assertIn(rv, [0, None]) + + # the help message will look like: + # + # localhelp [<command>] + # localone [--test-one TEST_ONE] + # localtwo [--test-two TEST_TWO] + commands = [] + command_help = { + "help": self.registry.help.usage(), + "localone": self.one().usage(), + "localtwo": self.two().usage()} + for line in output: + command = line.split()[0] + commands.append(command) + if command not in command_help: + self.fail("Got help for unknown command %s: %s" % + (command, line)) + self.assertEqual(line, command_help[command]) + self.assertItemsEqual(commands, command_help.keys()) + + @make_config() + def test_subcommand_help(self, config_file): + """get help message on a single command.""" + rv, output = self._get_subcommand_output( + ["-C", config_file, "help", "localone"]) + self.assertIn(rv, [0, None]) + self.assertEqual(output[0].strip(), + "usage: %s" % self.one().usage().strip()) + + @make_config() + def test_nonexistent_subcommand_help(self, config_file): + """get help message on a nonexistent command.""" + rv, output = self._get_subcommand_output( + ["-C", config_file, "help", "blargle"]) + self.assertNotEqual(rv, 0) + self.assertIn("No such command", output[0]) diff --git a/testsuite/Testsrc/Testlib/TestOptions/TestTypes.py b/testsuite/Testsrc/Testlib/TestOptions/TestTypes.py new file mode 100644 index 000000000..404d67fdc --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestOptions/TestTypes.py @@ -0,0 +1,111 @@ +"""test builtin option types.""" + +import argparse + +from mock import patch + +from Bcfg2.Options import Option, Types, Parser +from testsuite.common import Bcfg2TestCase + + +class TestOptionTypes(Bcfg2TestCase): + """test builtin option types.""" + def setUp(self): + self.options = None + + def _test_options(self, options): + """helper to test option types. + + this expects that self.options is set to a single option named + test. The value of that option is returned. + """ + result = argparse.Namespace() + parser = Parser(components=[self], namespace=result) + parser.parse(options) + return result.test + + def test_comma_list(self): + """parse comma-list values.""" + self.options = [Option("--test", type=Types.comma_list)] + + expected = ["one", "two", "three"] + self.assertItemsEqual(self._test_options(["--test", "one,two,three"]), + expected) + self.assertItemsEqual(self._test_options(["--test", + "one, two, three"]), + expected) + self.assertItemsEqual(self._test_options(["--test", + "one , two ,three"]), + expected) + self.assertItemsEqual(self._test_options(["--test", "one two, three"]), + ["one two", "three"]) + + def test_colon_list(self): + """parse colon-list values.""" + self.options = [Option("--test", type=Types.colon_list)] + self.assertItemsEqual(self._test_options(["--test", "one:two three"]), + ["one", "two three"]) + + def test_comma_dict(self): + """parse comma-dict values.""" + self.options = [Option("--test", type=Types.comma_dict)] + expected = { + "one": True, + "two": 2, + "three": "three", + "four": False} + self.assertDictEqual( + self._test_options(["--test", + "one=yes, two=2 , three=three,four=no"]), + expected) + + self.assertDictEqual( + self._test_options(["--test", "one,two=2,three=three,four=off"]), + expected) + + def test_anchored_regex_list(self): + """parse regex lists.""" + self.options = [Option("--test", type=Types.anchored_regex_list)] + self.assertItemsEqual( + [r.pattern for r in self._test_options(["--test", r'\d+ \s*'])], + [r'^\d+$', r'^\s*$']) + self.assertRaises(SystemExit, + self._test_options, ["--test", '(]']) + + def test_octal(self): + """parse octal options.""" + self.options = [Option("--test", type=Types.octal)] + self.assertEqual(self._test_options(["--test", "0777"]), 511) + self.assertEqual(self._test_options(["--test", "133114255"]), 23894189) + + @patch("pwd.getpwnam") + def test_username(self, mock_getpwnam): + """parse username options.""" + self.options = [Option("--test", type=Types.username)] + mock_getpwnam.return_value = ("test", '********', 1001, 1001, + "Test user", "/home/test", "/bin/bash") + self.assertEqual(self._test_options(["--test", "1001"]), 1001) + self.assertEqual(self._test_options(["--test", "test"]), 1001) + + @patch("grp.getgrnam") + def test_groupname(self, mock_getpwnam): + """parse group name options.""" + self.options = [Option("--test", type=Types.groupname)] + mock_getpwnam.return_value = ("test", '*', 1001, ["test"]) + self.assertEqual(self._test_options(["--test", "1001"]), 1001) + self.assertEqual(self._test_options(["--test", "test"]), 1001) + + def test_timeout(self): + """parse timeout options.""" + self.options = [Option("--test", type=Types.timeout)] + self.assertEqual(self._test_options(["--test", "1.0"]), 1.0) + self.assertEqual(self._test_options(["--test", "1"]), 1.0) + self.assertEqual(self._test_options(["--test", "0"]), None) + + def test_size(self): + """parse human-readable size options.""" + self.options = [Option("--test", type=Types.size)] + self.assertEqual(self._test_options(["--test", "5k"]), 5120) + self.assertEqual(self._test_options(["--test", "5"]), 5) + self.assertRaises(SystemExit, + self._test_options, ["--test", "g5m"]) diff --git a/testsuite/Testsrc/Testlib/TestOptions/TestWildcards.py b/testsuite/Testsrc/Testlib/TestOptions/TestWildcards.py new file mode 100644 index 000000000..da196a912 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestOptions/TestWildcards.py @@ -0,0 +1,47 @@ +"""test wildcard options.""" + +import argparse + +from Bcfg2.Options import Option, Parser +from testsuite.Testsrc.Testlib.TestOptions import OptionTestCase, make_config + + +class TestWildcardOptions(OptionTestCase): + """test parsing wildcard options.""" + config = { + "foo": { + "test1": "test1", + "test2": "test2", + "thing1": "thing1", + "thing2": "thing2", + "foo": "foo" + } + } + + def setUp(self): + # parsing options can modify the Option objects themselves. + # that's probably bad -- and it's definitely bad if we ever + # want to do real on-the-fly config changes -- but it's easier + # to leave it as is and set the options on each test. + self.options = [ + Option(cf=("foo", "*"), dest="all"), + Option(cf=("foo", "test*"), dest="test"), + Option(cf=("foo", "bogus*"), dest="unmatched"), + Option(cf=("bar", "*"), dest="no_section"), + Option(cf=("foo", "foo"))] + + @make_config(config) + def test_wildcard_options(self, config_file): + """parse wildcard options.""" + result = argparse.Namespace() + parser = Parser(components=[self], namespace=result) + parser.parse(argv=["-C", config_file]) + + self.assertDictEqual(result.all, {"test1": "test1", + "test2": "test2", + "thing1": "thing1", + "thing2": "thing2"}) + self.assertDictEqual(result.test, {"test1": "test1", + "test2": "test2"}) + self.assertDictEqual(result.unmatched, {}) + self.assertDictEqual(result.no_section, {}) diff --git a/testsuite/Testsrc/Testlib/TestOptions/Two.py b/testsuite/Testsrc/Testlib/TestOptions/Two.py new file mode 100644 index 000000000..189e0817f --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestOptions/Two.py @@ -0,0 +1,6 @@ +"""Test module for component loading.""" + + +class Two(object): + """Test class for component loading.""" + pass diff --git a/testsuite/Testsrc/Testlib/TestOptions/__init__.py b/testsuite/Testsrc/Testlib/TestOptions/__init__.py new file mode 100644 index 000000000..e92f95e94 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestOptions/__init__.py @@ -0,0 +1,86 @@ +"""helper functions for option testing.""" + +import os +import tempfile + +from Bcfg2.Compat import wraps, ConfigParser +from Bcfg2.Options import Parser, PathOption +from testsuite.common import Bcfg2TestCase + + +class make_config(object): # pylint: disable=invalid-name + """decorator to create a temporary config file from a dict. + + The filename of the temporary config file is added as the last + positional argument to the function call. + """ + def __init__(self, config_data=None): + self.config_data = config_data or {} + + def __call__(self, func): + @wraps(func) + def inner(*args, **kwargs): + """decorated function.""" + cfp = ConfigParser.ConfigParser() + for section, options in self.config_data.items(): + cfp.add_section(section) + for key, val in options.items(): + cfp.set(section, key, val) + fd, name = tempfile.mkstemp() + config_file = os.fdopen(fd, 'w') + cfp.write(config_file) + config_file.close() + + args = list(args) + [name] + try: + rv = func(*args, **kwargs) + finally: + os.unlink(name) + return rv + + return inner + + +def clean_environment(func): + """decorator that unsets any environment variables used by options. + + The list of options is taken from the first argument, which is + presumed to be ``self``. The variables are restored at the end of + the function. + """ + @wraps(func) + def inner(self, *args, **kwargs): + """decorated function.""" + envvars = {} + for opt in self.options: + if opt.env is not None: + envvars[opt.env] = os.environ.get(opt.env) + if opt.env in os.environ: + del os.environ[opt.env] + rv = func(self, *args, **kwargs) + for name, val in envvars.items(): + if val is None and name in os.environ: + del os.environ[name] + elif val is not None: + os.environ[name] = val + return rv + + return inner + + +class OptionTestCase(Bcfg2TestCase): + """test case that doesn't mock out config file reading.""" + + @classmethod + def setUpClass(cls): + # ensure that the option parser actually reads config files + Parser.unit_test = False + Bcfg2TestCase.setUpClass() + + def setUp(self): + Bcfg2TestCase.setUp(self) + PathOption.repository = None + + @classmethod + def tearDownClass(cls): + Parser.unit_test = True diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py index f135a0197..290a7c092 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py @@ -22,6 +22,7 @@ class TestPlugin(TestDebuggable): def setUp(self): TestDebuggable.setUp(self) set_setup_default("filemonitor", MagicMock()) + set_setup_default("repository", datastore) def get_obj(self, core=None): if core is None: diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index 37beaa26c..5a82100d0 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -55,6 +55,7 @@ class TestDatabaseBacked(TestPlugin): def setUp(self): TestPlugin.setUp(self) set_setup_default("%s_db" % self.test_obj.__name__.lower(), False) + set_setup_default("db_engine", None) @skipUnless(HAS_DJANGO, "Django not found") def test__use_db(self): diff --git a/testsuite/Testsrc/Testlib/TestUtils.py b/testsuite/Testsrc/Testlib/TestUtils.py index 349d6cd40..4bed67248 100644 --- a/testsuite/Testsrc/Testlib/TestUtils.py +++ b/testsuite/Testsrc/Testlib/TestUtils.py @@ -1,9 +1,5 @@ import os import sys -import copy -import lxml.etree -import subprocess -from mock import Mock, MagicMock, patch from Bcfg2.Utils import * # add all parent testsuite directories to sys.path to allow (most) diff --git a/testsuite/Testsrc/__init__.py b/testsuite/Testsrc/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/testsuite/Testsrc/__init__.py diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 77b170809..79eff7959 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -35,6 +35,7 @@ contingent_checks = { "lib/Bcfg2/Server/Admin": ["Reports.py", "Syncdb.py"], "sbin": ["bcfg2-reports"]}, ("pyinotify",): {"lib/Bcfg2/Server/FileMonitor": ["Inotify.py"]}, + ("apt",): {"lib/Bcfg2/Client/Tools": ["APT.py"]}, ("yum",): {"lib/Bcfg2/Client/Tools": ["YUM.py"]}, ("genshi",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgGenshiGenerator.py"]}, ("Cheetah",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgCheetahGenerator.py"]}, @@ -68,7 +69,8 @@ error_checks = { # perform no checks at all on the listed files no_checks = { - "lib/Bcfg2/Client/Tools": ["APT.py", "RPM.py", "rpmtools.py"], + "lib/Bcfg2/Client/Tools": ["RPM.py", "rpmtools.py"], + "lib/Bcfg2/Server": ["Snapshots", "Hostbase"], "lib/Bcfg2": ["manage.py"], "lib/Bcfg2/Server/Reports": ["manage.py"], "lib/Bcfg2/Server/Plugins": ["Base.py"], diff --git a/testsuite/common.py b/testsuite/common.py index 5a08f8db5..a86e9c5d9 100644 --- a/testsuite/common.py +++ b/testsuite/common.py @@ -38,7 +38,13 @@ def set_setup_default(option, value=None): if not hasattr(Bcfg2.Options.setup, option): setattr(Bcfg2.Options.setup, option, value) +# these two variables do slightly different things for unit tests; the +# former skips config file reading, while the latter sends option +# debug logging to stdout so it can be captured. These are separate +# because we want to enable config file reading in order to test +# option parsing. Bcfg2.Options.Parser.unit_test = True +Bcfg2.Options.Options.unit_test = True try: import django.conf @@ -119,6 +125,19 @@ class Bcfg2TestCase(TestCase): :func:`assertXMLEqual`, a useful assertion method given all the XML used by Bcfg2. """ + capture_stderr = True + + @classmethod + def setUpClass(cls): + cls._stderr = sys.stderr + if cls.capture_stderr: + sys.stderr = sys.stdout + + @classmethod + def tearDownClass(cls): + if cls.capture_stderr: + sys.stderr = cls._stderr + def assertXMLEqual(self, el1, el2, msg=None): """ Test that the two XML trees given are equal. """ if msg is None: diff --git a/tools/upgrade/1.3/README b/tools/upgrade/1.3/README index 1a919f869..29fd9886b 100644 --- a/tools/upgrade/1.3/README +++ b/tools/upgrade/1.3/README @@ -24,3 +24,7 @@ migrate_probe_groups_to_db.py - Migrate Probe host and group data from XML to DB backend for Metadata and Probe plugins. Does not migrate individual probe return data. Assumes migration to BOTH Metadata and Probe to database backends. + +migrate_sysv_simplename.py + - Migrate any Pkgmgr entries which may have been using the simplename + attribute introduced in 1.3.5 to the simplefile attribute diff --git a/tools/upgrade/1.3/migrate_sysv_simplename.py b/tools/upgrade/1.3/migrate_sysv_simplename.py new file mode 100755 index 000000000..f6599756b --- /dev/null +++ b/tools/upgrade/1.3/migrate_sysv_simplename.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +import os +import sys +import glob +import lxml.etree +import Bcfg2.Options + +def main(): + opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY) + setup = Bcfg2.Options.OptionParser(opts) + setup.parse(sys.argv[1:]) + + files = [] + for plugin in ['Pkgmgr']: + files.extend(glob.glob(os.path.join(setup['repo'], plugin, "*"))) + + for bfile in files: + bdata = lxml.etree.parse(bfile) + changed = False + + if not bdata.xpath("//@type='sysv'"): + print("%s doesn't contain any sysv packages, skipping" % bfile) + continue + + pkglist = bdata.getroot() + if pkglist.tag != "PackageList": + print("%s doesn't look like a PackageList, skipping" % bfile) + continue + + for pkg in bdata.xpath("//Package"): + if "simplename" in pkg.attrib: + pkg.set("simplefile", pkg.get("simplename")) + del pkg.attrib["simplename"] + changed = True + + # if we switched to simplefile, we also need to switch to uri + if changed and "url" in pkglist.attrib: + pkglist.set("uri", pkglist.get("url")) + del pkglist.attrib["url"] + + if changed: + print("Writing %s" % bfile) + try: + open(bfile, "w").write(lxml.etree.tostring(bdata)) + except IOError: + err = sys.exc_info()[1] + print("Could not write %s: %s" % (bfile, err)) + +if __name__ == '__main__': + sys.exit(main()) |