From bd8e639ad56422893e67c74a3b8dae3f27f92276 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 27 Jun 2013 10:25:03 -0400 Subject: Options: wrote completely new option parser --- doc/development/option_parsing.txt | 236 ++++++ src/lib/Bcfg2/Options.py | 1328 --------------------------------- src/lib/Bcfg2/Options/Actions.py | 164 ++++ src/lib/Bcfg2/Options/Common.py | 135 ++++ src/lib/Bcfg2/Options/OptionGroups.py | 209 ++++++ src/lib/Bcfg2/Options/Options.py | 305 ++++++++ src/lib/Bcfg2/Options/Parser.py | 282 +++++++ src/lib/Bcfg2/Options/Subcommands.py | 237 ++++++ src/lib/Bcfg2/Options/Types.py | 87 +++ src/lib/Bcfg2/Options/__init__.py | 10 + 10 files changed, 1665 insertions(+), 1328 deletions(-) create mode 100644 doc/development/option_parsing.txt delete mode 100644 src/lib/Bcfg2/Options.py create mode 100644 src/lib/Bcfg2/Options/Actions.py create mode 100644 src/lib/Bcfg2/Options/Common.py create mode 100644 src/lib/Bcfg2/Options/OptionGroups.py create mode 100644 src/lib/Bcfg2/Options/Options.py create mode 100644 src/lib/Bcfg2/Options/Parser.py create mode 100644 src/lib/Bcfg2/Options/Subcommands.py create mode 100644 src/lib/Bcfg2/Options/Types.py create mode 100644 src/lib/Bcfg2/Options/__init__.py diff --git a/doc/development/option_parsing.txt b/doc/development/option_parsing.txt new file mode 100644 index 000000000..52da8fced --- /dev/null +++ b/doc/development/option_parsing.txt @@ -0,0 +1,236 @@ +.. -*- mode: rst -*- + +.. _development-option-parsing: + +==================== +Bcfg2 Option Parsing +==================== + +Bcfg2 uses an option parsing mechanism based on the Python +:mod:`argparse` module. It does several very useful things that +``argparse`` does not: + +* Collects options from various places, which lets us easily specify + per-plugin options, for example; +* Automatically loads components (such as plugins); +* Synthesizes option values from the command line, config files, and + environment variables; +* Can dynamically create commands with many subcommands (e.g., + bcfg2-info and bcfg2-admin); and +* Supports keeping documentation inline with the option declaration, + which will make it easier to generate man pages. + + +Collecting Options +================== + +One of the more important features of the option parser is its ability +to automatically collect options from loaded components (e.g., Bcfg2 +server plugins). Given the highly pluggable architecture of Bcfg2, +this helps ensure two things: + +#. We do not have to specify all options in all places, or even in + most places. Options are specified alongside the class(es) that use + them. +#. All options needed for a given script to run are guaranteed to be + loaded, without the need to specify all components that script uses + manually. + +For instance, assume a few plugins: + +* The ``Foo`` plugin takes one option, ``--foo`` +* The ``Bar`` plugin takes two options, ``--bar`` and ``--force`` + +The plugins are used by the ``bcfg2-quux`` command, which itself takes +two options: ``--plugins`` (which selects the plugins) and +``--test``. The options would be selected at runtime, so for instance +these would be valid: + +.. code-block:: bash + + bcfg2-quux --plugins Foo --foo --test + bcfg2-quux --plugins Foo,Bar --foo --bar --force + bcfg2-quux --plugins Bar --force + +But this would not: + + bcfg2-quux --plugins Foo --bar + +The help message would reflect the options that are available to the +default set of plugins. (For this reason, allowing component lists to +be set in the config file is very useful; that way, usage messages +reflect the components in the config file.) + +Components (in this example, the plugins) can be classes or modules. +There is no required interface for an option component. They may +*optionally* have: + +* An ``options`` attribute that is a list of + :class:`Bcfg2.Options.Options.Option` objects or option groups. +* A function or static method, ``options_parsed_hook``, that is called + when all options have been parsed. (This will be called again if + :func:`Bcfg2.Options.Parser.Parser.reparse` is called.) + +Options are collected through two primary mechanisms: + +#. The :class:`Bcfg2.Options.Actions.ComponentAction` class. When a + ComponentAction subclass is used as the action of an option, then + options contained in the classes (or modules) given in the option + value will be added to the parser. +#. Modules that are not loaded via a + :class:`Bcfg2.Options.Actions.ComponentAction` option may load + options at runtime. + +Since it is preferred to add components instead of just options, +loading options at runtime is generally best accomplished by creating +a container object whose only purpose is to hold options. For +instance: + +.. code-block:: python + + def foo(): + # do stuff + + class _OptionContainer(object): + options = [ + Bcfg2.Options.BooleanOption("--foo", help="Enable foo")] + + @staticmethod + def options_parsed_hook(): + if Bcfg2.Options.setup.foo: + foo() + + Bcfg2.Options.get_parser().add_component(_OptionContainer) + +The Bcfg2.Options module +======================== + +.. currentmodule:: Bcfg2.Options + +.. autodata:: setup + +Options +------- + +The base :class:`Bcfg2.Options.Option` object represents an option. +Unlike options in :mod:`argparse`, an Option object does not need to +be associated with an option parser; it exists on its own. + +.. autoclass:: Option +.. autoclass:: PathOption +.. autoclass:: BooleanOption +.. autoclass:: PositionalArgument + +The Parser +---------- + +.. autoclass:: Parser +.. autofunction:: get_parser +.. autoexception:: OptionParserException + +Option Groups +------------- + +Options can be grouped in various meaningful ways. This uses a +variety of :mod:`argparse` functionality behind the scenes. + +In all cases, options can be added to groups in-line by simply +specifying them in the object group constructor: + +.. code-block:: python + + options = [ + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.Option(...), + Bcfg2.Options.Option(...), + required=True), + ....] + +Nesting object groups is supported in theory, but barely tested. + +.. autoclass:: OptionGroup +.. autoclass:: ExclusiveOptionGroup +.. autoclass:: Subparser +.. autoclass:: WildcardSectionGroup + +Subcommands +----------- + +This library makes it easier to work with programs that have a large +number of subcommands (e.g., :ref:`bcfg2-info ` and +:ref:`bcfg2-admin `). + +The normal implementation pattern is this: + +#. Define all of your subcommands as children of + :class:`Bcfg2.Options.Subcommand`. +#. Define a :class:`Bcfg2.Options.CommandRegistry` object that will be + used to register all of the commands. Registering a command + collect its options and adds it as a + :class:`Bcfg2.Options.Subparser` option group to the main option + parser. +#. Register your commands with + :func:`Bcfg2.Options.register_commands`, parse options, and run. + +:mod:`Bcfg2.Server.Admin` provides a fairly simple implementation, +where the CLI class is itself the command registry: + +.. code-block:: python + + class CLI(Bcfg2.Options.CommandRegistry): + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, + globals().values(), + parent=AdminCmd) + parser = Bcfg2.Options.get_parser( + description="Manage a running Bcfg2 server", + components=[self]) + parser.parse() + +In this case, commands are collected from amongst all global variables +(the most likely scenario), and they must be children of +:class:`Bcfg2.Server.Admin.AdminCmd`, which itself subclasses +:class:`Bcfg2.Options.Subcommand`. + +Commands are defined by subclassing :class:`Bcfg2.Options.Subcommand`. +At a minimum, the :func:`Bcfg2.Options.Subcommand.run` method must be +overridden, and a docstring written. + +.. autoclass:: Subcommand +.. autoclass:: HelpCommand +.. autoclass:: CommandRegistry +.. autofunction:: register_commands + +Actions +------- + +Several custom argparse `actions +`_ provide +some of the option collection magic of :mod:`Bcfg2.Options`. + +.. autoclass:: ConfigFileAction +.. autoclass:: ComponentAction +.. autoclass:: PluginsAction + +Option Types +------------ + +:mod:`Bcfg2.Options` provides a number of useful types for use as the `type +`_ keyword +argument to +the :class:`Bcfg2.Options.Option` constructor. + +.. autofunction:: Bcfg2.Options.Types.path +.. autofunction:: Bcfg2.Options.Types.comma_list +.. autofunction:: Bcfg2.Options.Types.colon_list +.. autofunction:: Bcfg2.Options.Types.octal +.. autofunction:: Bcfg2.Options.Types.username +.. autofunction:: Bcfg2.Options.Types.groupname +.. autofunction:: Bcfg2.Options.Types.timeout +.. autofunction:: Bcfg2.Options.Types.size + +Common Options +-------------- + +.. autoclass:: Common diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py deleted file mode 100644 index 64408693a..000000000 --- a/src/lib/Bcfg2/Options.py +++ /dev/null @@ -1,1328 +0,0 @@ -"""Option parsing library for utilities.""" - -import copy -import getopt -import inspect -import os -import re -import shlex -import sys -import grp -import pwd -from Bcfg2.Client.Tools import __path__ as toolpath -from Bcfg2.Compat import ConfigParser, walk_packages -from Bcfg2.version import __version__ - - -class OptionFailure(Exception): - """ raised when malformed Option objects are instantiated """ - pass - -DEFAULT_CONFIG_LOCATION = '/etc/bcfg2.conf' -DEFAULT_INSTALL_PREFIX = '/usr' - - -class DefaultConfigParser(ConfigParser.ConfigParser): - """ A config parser that can be used to query options with default - values in the event that the option is not found """ - - def __init__(self, *args, **kwargs): - """Make configuration options case sensitive""" - ConfigParser.ConfigParser.__init__(self, *args, **kwargs) - self.optionxform = str - - def get(self, section, option, **kwargs): - """ convenience method for getting config items """ - default = None - if 'default' in kwargs: - default = kwargs['default'] - del kwargs['default'] - try: - return ConfigParser.ConfigParser.get(self, section, option, - **kwargs) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - if default is not None: - return default - else: - raise - - def getboolean(self, section, option, **kwargs): - """ convenience method for getting boolean config items """ - default = None - if 'default' in kwargs: - default = kwargs['default'] - del kwargs['default'] - try: - return ConfigParser.ConfigParser.getboolean(self, section, - option, **kwargs) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, - ValueError): - if default is not None: - return default - else: - raise - - -class Option(object): - """ a single option, which might be read from the command line, - environment, or config file """ - - # pylint: disable=C0103,R0913 - def __init__(self, desc, default, cmd=None, odesc=False, - env=False, cf=False, cook=False, long_arg=False, - deprecated_cf=None): - self.desc = desc - self.default = default - self.cmd = cmd - self.long = long_arg - if not self.long: - if cmd and (cmd[0] != '-' or len(cmd) != 2): - raise OptionFailure("Poorly formed command %s" % cmd) - elif cmd and not cmd.startswith('--'): - raise OptionFailure("Poorly formed command %s" % cmd) - self.odesc = odesc - self.env = env - self.cf = cf - self.deprecated_cf = deprecated_cf - self.boolean = False - if not odesc and not cook and isinstance(self.default, bool): - self.boolean = True - self.cook = cook - self.value = None - # pylint: enable=C0103,R0913 - - def get_cooked_value(self, value): - """ get the value of this option after performing any option - munging specified in the 'cook' keyword argument to the - constructor """ - if self.boolean: - return True - if self.cook: - return self.cook(value) - else: - return value - - def __str__(self): - rv = ["%s: " % self.__class__.__name__, self.desc] - if self.cmd or self.cf: - rv.append(" (") - if self.cmd: - if self.odesc: - if self.long: - rv.append("%s=%s" % (self.cmd, self.odesc)) - else: - rv.append("%s %s" % (self.cmd, self.odesc)) - else: - rv.append("%s" % self.cmd) - - if self.cf: - if self.cmd: - rv.append("; ") - rv.append("[%s].%s" % self.cf) - if self.cmd or self.cf: - rv.append(")") - if hasattr(self, "value"): - rv.append(": %s" % self.value) - return "".join(rv) - - def buildHelpMessage(self): - """ build the help message for this option """ - vals = [] - if not self.cmd: - return '' - if self.odesc: - if self.long: - vals.append("%s=%s" % (self.cmd, self.odesc)) - else: - vals.append("%s %s" % (self.cmd, self.odesc)) - else: - vals.append(self.cmd) - vals.append(self.desc) - return " %-28s %s\n" % tuple(vals) - - def buildGetopt(self): - """ build a string suitable for describing this short option - to getopt """ - gstr = '' - if self.long: - return gstr - if self.cmd: - gstr = self.cmd[1] - if self.odesc: - gstr += ':' - return gstr - - def buildLongGetopt(self): - """ build a string suitable for describing this long option to - getopt """ - if self.odesc: - return self.cmd[2:] + '=' - else: - return self.cmd[2:] - - def parse(self, opts, rawopts, configparser=None): - """ parse a single option. try parsing the data out of opts - (the results of getopt), rawopts (the raw option string), the - environment, and finally the config parser. either opts or - rawopts should be provided, but not both """ - if self.cmd and opts: - # Processing getopted data - optinfo = [opt[1] for opt in opts if opt[0] == self.cmd] - if optinfo: - if optinfo[0]: - self.value = self.get_cooked_value(optinfo[0]) - else: - self.value = True - return - if self.cmd and self.cmd in rawopts: - if self.odesc: - data = rawopts[rawopts.index(self.cmd) + 1] - else: - data = True - self.value = self.get_cooked_value(data) - return - # No command line option found - if self.env and self.env in os.environ: - self.value = self.get_cooked_value(os.environ[self.env]) - return - if self.cf and configparser: - try: - self.value = self.get_cooked_value(configparser.get(*self.cf)) - return - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - pass - if self.deprecated_cf: - try: - self.value = self.get_cooked_value( - configparser.get(*self.deprecated_cf)) - print("Warning: [%s] %s is deprecated, use [%s] %s instead" - % (self.deprecated_cf[0], self.deprecated_cf[1], - self.cf[0], self.cf[1])) - return - except (ConfigParser.NoSectionError, - ConfigParser.NoOptionError): - pass - - # Default value not cooked - self.value = self.default - - -class OptionSet(dict): - """ a set of Option objects that interfaces with getopt and - DefaultConfigParser to populate a dict of