diff options
Diffstat (limited to 'src/lib/Bcfg2/Options')
| -rw-r--r-- | src/lib/Bcfg2/Options/Actions.py | 166 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Options/Common.py | 133 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Options/OptionGroups.py | 204 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Options/Options.py | 308 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Options/Parser.py | 311 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Options/Subcommands.py | 238 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Options/Types.py | 111 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Options/__init__.py | 10 |
8 files changed, 1481 insertions, 0 deletions
diff --git a/src/lib/Bcfg2/Options/Actions.py b/src/lib/Bcfg2/Options/Actions.py new file mode 100644 index 000000000..8b97f1da8 --- /dev/null +++ b/src/lib/Bcfg2/Options/Actions.py @@ -0,0 +1,166 @@ +""" Custom argparse actions """ + +import sys +import argparse +from Bcfg2.Options.Parser import get_parser + +__all__ = ["ConfigFileAction", "ComponentAction", "PluginsAction"] + + +class ComponentAction(argparse.Action): + """ ComponentAction automatically imports classes and modules + based on the value of the option, and automatically collects + options from the loaded classes and modules. It cannot be used by + itself, but must be subclassed, with either :attr:`mapping` or + :attr:`bases` overridden. See + :class:`Bcfg2.Options.PluginsAction` for an example. + + ComponentActions expect to be given a list of class names. If + :attr:`bases` is overridden, then it will attempt to import those + classes from identically named modules within the given bases. + For instance: + + .. code-block:: python + + class FooComponentAction(Bcfg2.Options.ComponentAction): + bases = ["Bcfg2.Server.Foo"] + + + class FooLoader(object): + options = [ + Bcfg2.Options.Option( + "--foo", + type=Bcfg2.Options.Types.comma_list, + default=["One"], + action=FooComponentAction)] + + If "--foo One,Two,Three" were given on the command line, then + ``FooComponentAction`` would attempt to import + ``Bcfg2.Server.Foo.One.One``, ``Bcfg2.Server.Foo.Two.Two``, and + ``Bcfg2.Server.Foo.Three.Three``. (It would also call + :func:`Bcfg2.Options.Parser.add_component` with each of those + classes as arguments.) + + Note that, although ComponentActions expect lists of components + (by default; this can be overridden by setting :attr:`islist`), + you must still explicitly specify a ``type`` argument to the + :class:`Bcfg2.Options.Option` constructor to split the value into + a list. + + Note also that, unlike other actions, the default value of a + ComponentAction option does not need to be the actual literal + final value. (I.e., you don't have to import + ``Bcfg2.Server.Foo.One.One`` and set it as the default in the + example above; the string "One" suffices.) + """ + + #: A list of parent modules where modules or classes should be + #: imported from. + bases = [] + + #: A mapping of ``<name> => <object>`` that components will be + #: loaded from. This can be used to permit much more complex + #: behavior than just a list of :attr:`bases`. + mapping = dict() + + #: If ``module`` is True, then only the module will be loaded, not + #: a class from the module. For instance, in the example above, + #: ``FooComponentAction`` would attempt instead to import + #: ``Bcfg2.Server.Foo.One``, ``Bcfg2.Server.Foo.Two``, and + #: ``Bcfg2.Server.Foo.Three``. + module = False + + #: By default, ComponentActions expect a list of components to + #: load. If ``islist`` is False, then it will only expect a + #: single component. + islist = True + + #: If ``fail_silently`` is True, then failures to import modules + #: or classes will not be logged. This is useful when the default + #: is to import everything, some of which are expected to fail. + fail_silently = False + + def __init__(self, *args, **kwargs): + if self.mapping: + if 'choices' not in kwargs: + kwargs['choices'] = self.mapping.keys() + self._final = False + argparse.Action.__init__(self, *args, **kwargs) + + def _import(self, module, name): + """ Import the given name from the given module, handling + errors """ + try: + return getattr(__import__(module, fromlist=[name]), name) + except (AttributeError, ImportError): + if not self.fail_silently: + print("Failed to load %s from %s: %s" % + (name, module, sys.exc_info()[1])) + return None + + def _load_component(self, name): + """ Import a single class or module, adding it as a component to + the parser. + + :param name: The name of the class or module to import, without + the base prepended. + :type name: string + :returns: the imported class or module + """ + cls = None + if self.mapping and name in self.mapping: + cls = self.mapping[name] + elif "." in name: + cls = self._import(*name.rsplit(".", 1)) + else: + for base in self.bases: + if self.module: + mod = base + else: + mod = "%s.%s" % (base, name) + cls = self._import(mod, name) + if cls is not None: + break + if cls: + get_parser().add_component(cls) + else: + print("Could not load component %s" % name) + return cls + + def finalize(self, parser, namespace): + """ Finalize a default value by loading the components given + in it. This lets a default be specified with a list of + strings instead of a list of classes. """ + if not self._final: + self.__call__(parser, namespace, self.default) + + def __call__(self, parser, namespace, values, option_string=None): + if values is None: + result = None + else: + if self.islist: + result = [] + for val in values: + cls = self._load_component(val) + if cls is not None: + result.append(cls) + else: + result = self._load_component(values) + self._final = True + setattr(namespace, self.dest, result) + + +class ConfigFileAction(argparse.Action): + """ ConfigFileAction automatically loads and parses a + supplementary config file (e.g., ``bcfg2-web.conf`` or + ``bcfg2-lint.conf``). """ + + def __call__(self, parser, namespace, values, option_string=None): + get_parser().add_config_file(self.dest, values) + setattr(namespace, self.dest, values) + + +class PluginsAction(ComponentAction): + """ :class:`Bcfg2.Options.ComponentAction` subclass for loading + Bcfg2 server plugins. """ + bases = ['Bcfg2.Server.Plugins'] diff --git a/src/lib/Bcfg2/Options/Common.py b/src/lib/Bcfg2/Options/Common.py new file mode 100644 index 000000000..9ba08eb87 --- /dev/null +++ b/src/lib/Bcfg2/Options/Common.py @@ -0,0 +1,133 @@ +""" Common options used in multiple different contexts. """ + +from Bcfg2.Utils import classproperty +from Bcfg2.Options import Types +from Bcfg2.Options.Actions import PluginsAction, ComponentAction +from Bcfg2.Options.Parser import repository as _repository_option +from Bcfg2.Options import Option, PathOption, BooleanOption + +__all__ = ["Common"] + + +class ReportingTransportAction(ComponentAction): + """ :class:`Bcfg2.Options.ComponentAction` that loads a single + reporting transport from :mod:`Bcfg2.Reporting.Transport`. """ + islist = False + bases = ['Bcfg2.Reporting.Transport'] + + +class ReportingStorageAction(ComponentAction): + """ :class:`Bcfg2.Options.ComponentAction` that loads a single + reporting storage driver from :mod:`Bcfg2.Reporting.Storage`. """ + islist = False + bases = ['Bcfg2.Reporting.Storage'] + + +class Common(object): + """ Common options used in multiple different contexts. """ + _plugins = None + _filemonitor = None + _reporting_storage = None + _reporting_transport = None + + @classproperty + def plugins(cls): + """ Load a list of Bcfg2 server plugins """ + if cls._plugins is None: + cls._plugins = Option( + cf=('server', 'plugins'), + type=Types.comma_list, help="Server plugin list", + action=PluginsAction, + default=['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', 'Rules', + 'SSHbase']) + return cls._plugins + + @classproperty + def filemonitor(cls): + """ Load a single Bcfg2 file monitor (from + :attr:`Bcfg2.Server.FileMonitor.available`) """ + if cls._filemonitor is None: + import Bcfg2.Server.FileMonitor + + class FileMonitorAction(ComponentAction): + """ ComponentAction for loading a single FAM backend + class """ + islist = False + mapping = Bcfg2.Server.FileMonitor.available + + cls._filemonitor = Option( + cf=('server', 'filemonitor'), action=FileMonitorAction, + default='default', help='Server file monitoring driver') + return cls._filemonitor + + @classproperty + def reporting_storage(cls): + """ Load a Reporting storage backend """ + if cls._reporting_storage is None: + cls._reporting_storage = Option( + cf=('reporting', 'storage'), dest="reporting_storage", + help='Reporting storage engine', + action=ReportingStorageAction, default='DjangoORM') + return cls._reporting_storage + + @classproperty + def reporting_transport(cls): + """ Load a Reporting transport backend """ + if cls._reporting_transport is None: + cls._reporting_transport = Option( + cf=('reporting', 'transport'), dest="reporting_transport", + help='Reporting transport', + action=ReportingTransportAction, default='DirectStore') + return cls._reporting_transport + + #: Set the path to the Bcfg2 repository + repository = _repository_option + + #: Daemonize process, storing PID + daemon = PathOption( + '-D', '--daemon', help="Daemonize process, storing PID") + + #: Run interactively, prompting the user for each change + interactive = BooleanOption( + "-I", "--interactive", + help='Run interactively, prompting the user for each change') + + #: Log to syslog + syslog = BooleanOption( + cf=('logging', 'syslog'), help="Log to syslog") + + #: Server location + location = Option( + '-S', '--server', cf=('components', 'bcfg2'), + default='https://localhost:6789', metavar='<https://server:port>', + help="Server location") + + #: Communication password + password = Option( + '-x', '--password', cf=('communication', 'password'), + metavar='<password>', help="Communication Password") + + #: Path to SSL key + ssl_key = PathOption( + '--ssl-key', cf=('communication', 'key'), dest="key", + help='Path to SSL key', default="/etc/pki/tls/private/bcfg2.key") + + #: Path to SSL certificate + ssl_cert = PathOption( + cf=('communication', 'certificate'), dest="cert", + help='Path to SSL certificate', default="/etc/pki/tls/certs/bcfg2.crt") + + #: Path to SSL CA certificate + ssl_ca = PathOption( + cf=('communication', 'ca'), help='Path to SSL CA Cert') + + #: Default Path paranoid setting + default_paranoid = Option( + cf=('mdata', 'paranoid'), dest="default_paranoid", default='true', + choices=['true', 'false'], help='Default Path paranoid setting') + + #: Client timeout + client_timeout = Option( + "-t", "--timeout", type=float, default=90.0, dest="client_timeout", + cf=('communication', 'timeout'), + help='Set the client XML-RPC timeout') diff --git a/src/lib/Bcfg2/Options/OptionGroups.py b/src/lib/Bcfg2/Options/OptionGroups.py new file mode 100644 index 000000000..465358fab --- /dev/null +++ b/src/lib/Bcfg2/Options/OptionGroups.py @@ -0,0 +1,204 @@ +""" Option grouping classes """ + +import re +import copy +import fnmatch +from Bcfg2.Options import Option +from itertools import chain + +__all__ = ["OptionGroup", "ExclusiveOptionGroup", "Subparser", + "WildcardSectionGroup"] + + +class OptionContainer(list): + """ Parent class of all option groups """ + + def list_options(self): + """ Get a list of all options contained in this group, + including options contained in option groups in this group, + and so on. """ + return list(chain(*[o.list_options() for o in self])) + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, list.__repr__(self)) + + def add_to_parser(self, parser): + """ Add this option group to a :class:`Bcfg2.Options.Parser` + object. """ + for opt in self: + opt.add_to_parser(parser) + + +class OptionGroup(OptionContainer): + """ Generic option group that is used only to organize options. + This uses :meth:`argparse.ArgumentParser.add_argument_group` + behind the scenes. """ + + def __init__(self, *items, **kwargs): + r""" + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param title: The title of the option group + :type title: string + :param description: A longer description of the option group + :param description: string + """ + OptionContainer.__init__(self, items) + self.title = kwargs.pop('title') + self.description = kwargs.pop('description', None) + + def add_to_parser(self, parser): + group = parser.add_argument_group(self.title, self.description) + OptionContainer.add_to_parser(self, group) + + +class ExclusiveOptionGroup(OptionContainer): + """ Option group that ensures that only one argument in the group + is present. This uses + :meth:`argparse.ArgumentParser.add_mutually_exclusive_group` + behind the scenes.""" + + def __init__(self, *items, **kwargs): + r""" + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param required: Exactly one argument in the group *must* be + specified. + :type required: boolean + """ + OptionContainer.__init__(self, items) + self.required = kwargs.pop('required', False) + + def add_to_parser(self, parser): + group = parser.add_mutually_exclusive_group(required=self.required) + OptionContainer.add_to_parser(self, group) + + +class Subparser(OptionContainer): + """ Option group that adds options in it to a subparser. This + uses a lot of functionality tied to `argparse Sub-commands + <http://docs.python.org/dev/library/argparse.html#sub-commands>`_. + + The subcommand string itself is stored in the + :attr:`Bcfg2.Options.setup` namespace as ``subcommand``. + + This is commonly used with :class:`Bcfg2.Options.Subcommand` + groups. + """ + + _subparsers = dict() + + def __init__(self, *items, **kwargs): + r""" + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param name: The name of the subparser. Required. + :type name: string + :param help: A help message for the subparser + :param help: string + """ + self.name = kwargs.pop('name') + self.help = kwargs.pop('help', None) + OptionContainer.__init__(self, items) + + def __repr__(self): + return "%s %s(%s)" % (self.__class__.__name__, + self.name, + list.__repr__(self)) + + def add_to_parser(self, parser): + if parser not in self._subparsers: + self._subparsers[parser] = parser.add_subparsers(dest='subcommand') + subparser = self._subparsers[parser].add_parser(self.name, + help=self.help) + OptionContainer.add_to_parser(self, subparser) + + +class WildcardSectionGroup(OptionContainer, Option): + """ WildcardSectionGroups contain options that may exist in + several different sections of the config that match a glob. It + works by creating options on the fly to match the sections + described in the glob. For example, consider: + + .. code-block:: python + + options = [ + Bcfg2.Options.WildcardSectionGroup( + Bcfg2.Options.Option(cf=("myplugin:*", "number"), type=int), + Bcfg2.Options.Option(cf=("myplugin:*", "description"))] + + If the config file contained ``[myplugin:foo]`` and + ``[myplugin:bar]`` sections, then this would automagically create + options for each of those. The end result would be: + + .. code-block:: python + + >>> Bcfg2.Options.setup + Namespace(myplugin_bar_description='Bar description', myplugin_bar_number=2, myplugin_foo_description='Foo description', myplugin_foo_number=1, myplugin_sections=['myplugin:foo', 'myplugin:bar']) + + All options must have the same section glob. + + The options are stored in an automatically-generated destination + given by:: + + <prefix><section>_<destination> + + ``<destination>`` is the original `dest + <http://docs.python.org/dev/library/argparse.html#dest>`_ of the + option. ``<section>`` is the section that it's found in. + ``<prefix>`` is automatically generated from the section glob by + replacing all consecutive characters disallowed in Python variable + names into underscores. (This can be overridden with the + constructor.) + + This group stores an additional option, the sections themselves, + in an option given by ``<prefix>sections``. + """ + + #: Regex to automatically get a destination for this option + _dest_re = re.compile(r'(\A(_|[^A-Za-z])+)|((_|[^A-Za-z0-9])+)') + + def __init__(self, *items, **kwargs): + r""" + :param \*args: Child options + :type \*args: Bcfg2.Options.Option + :param prefix: The prefix to use for options generated by this + option group. By default this is generated + automatically from the config glob; see above + for details. + :type prefix: string + :param dest: The destination for the list of known sections + that match the glob. + :param dest: string + """ + OptionContainer.__init__(self, []) + self._section_glob = items[0].cf[0] + # get a default destination + self._prefix = kwargs.get("prefix", + self._dest_re.sub('_', self._section_glob)) + Option.__init__(self, dest=kwargs.get('dest', + self._prefix + "sections")) + self._options = items + + def list_options(self): + return [self] + OptionContainer.list_options(self) + + def from_config(self, cfp): + sections = [] + for section in cfp.sections(): + if fnmatch.fnmatch(section, self._section_glob): + sections.append(section) + newopts = [] + for opt_tmpl in self._options: + option = copy.deepcopy(opt_tmpl) + option.cf = (section, option.cf[1]) + option.dest = self._prefix + section + "_" + option.dest + newopts.append(option) + self.extend(newopts) + for parser in self.parsers: + parser.add_options(newopts) + return sections + + def add_to_parser(self, parser): + Option.add_to_parser(self, parser) + OptionContainer.add_to_parser(self, parser) diff --git a/src/lib/Bcfg2/Options/Options.py b/src/lib/Bcfg2/Options/Options.py new file mode 100644 index 000000000..be7e7c646 --- /dev/null +++ b/src/lib/Bcfg2/Options/Options.py @@ -0,0 +1,308 @@ +""" The base :class:`Bcfg2.Options.Option` object represents an +option. Unlike options in :mod:`argparse`, an Option object does not +need to be associated with an option parser; it exists on its own.""" + +import os +import copy +import fnmatch +import argparse +from Bcfg2.Options import Types +from Bcfg2.Compat import ConfigParser + + +__all__ = ["Option", "BooleanOption", "PathOption", "PositionalArgument"] + +#: A dict that records a mapping of argparse action name (e.g., +#: "store_true") to the argparse Action class for it. See +#: :func:`_get_action_class` +_action_map = dict() # pylint: disable=C0103 + + +def _get_action_class(action_name): + """ Given an argparse action name (e.g., "store_true"), get the + related :class:`argparse.Action` class. The mapping that stores + this information in :mod:`argparse` itself is unfortunately + private, so it's an implementation detail that we shouldn't depend + on. So we just instantiate a dummy parser, add a dummy argument, + and determine the class that way. """ + if (isinstance(action_name, type) and + issubclass(action_name, argparse.Action)): + return action_name + if action_name not in _action_map: + action = argparse.ArgumentParser().add_argument(action_name, + action=action_name) + _action_map[action_name] = action.__class__ + return _action_map[action_name] + + +class Option(object): + """ Representation of an option that can be specified on the + command line, as an environment variable, or in a config + file. Precedence is in that order; that is, an option specified on + the command line takes precendence over an option given by the + environment, which takes precedence over an option specified in + the config file. """ + + #: Keyword arguments that should not be passed on to the + #: :class:`argparse.ArgumentParser` constructor + _local_args = ['cf', 'env', 'man'] + + def __init__(self, *args, **kwargs): + """ See :meth:`argparse.ArgumentParser.add_argument` for a + full list of accepted parameters. + + In addition to supporting all arguments and keyword arguments + from :meth:`argparse.ArgumentParser.add_argument`, several + additional keyword arguments are allowed. + + :param cf: A tuple giving the section and option name that + this argument can be referenced as in the config + file. The option name may contain the wildcard + '*', in which case the value will be a dict of all + options matching the glob. (To use a wildcard in + the section, use a + :class:`Bcfg2.Options.WildcardSectionGroup`.) + :type cf: tuple + :param env: An environment variable that the value of this + option can be taken from. + :type env: string + :param man: A detailed description of the option that will be + used to populate automatically-generated manpages. + :type man: string + """ + #: The options by which this option can be called. + #: (Coincidentally, this is also the list of arguments that + #: will be passed to + #: :meth:`argparse.ArgumentParser.add_argument` when this + #: option is added to a parser.) As a result, ``args`` can be + #: tested to see if this argument can be given on the command + #: line at all, or if it is purely a config file option. + self.args = args + self._kwargs = kwargs + + #: The tuple giving the section and option name for this + #: option in the config file + self.cf = None # pylint: disable=C0103 + + #: The environment variable that this option can take its + #: value from + self.env = None + + #: A detailed description of this option that will be used in + #: man pages. + self.man = None + + #: A list of :class:`Bcfg2.Options.Parser` objects to which + #: this option has been added. (There will be more than one + #: parser if this option is added to a subparser, for + #: instance.) + self.parsers = [] + + #: A dict of :class:`Bcfg2.Options.Parser` -> + #: :class:`argparse.Action` that gives the actions that + #: resulted from adding this option to each parser that it was + #: added to. If this option cannot be specified on the + #: command line (i.e., it only takes its value from the config + #: file), then this will be empty. + self.actions = dict() + + self.type = self._kwargs.get("type") + self.help = self._kwargs.get("help") + self._default = self._kwargs.get("default") + for kwarg in self._local_args: + setattr(self, kwarg, self._kwargs.pop(kwarg, None)) + if self.args: + # cli option + self._dest = None + else: + action_cls = _get_action_class(self._kwargs.get('action', 'store')) + # determine the name of this option. use, in order, the + # 'name' kwarg; the option name; the environment variable + # name. + self._dest = None + if 'dest' in self._kwargs: + self._dest = self._kwargs.pop('dest') + elif self.cf is not None: + self._dest = self.cf[1] + elif self.env is not None: + self._dest = self.env + kwargs = copy.copy(self._kwargs) + kwargs.pop("action", None) + self.actions[None] = action_cls(self._dest, self._dest, **kwargs) + + def __repr__(self): + sources = [] + if self.args: + sources.extend(self.args) + if self.cf: + sources.append("%s.%s" % self.cf) + if self.env: + sources.append("$" + self.env) + spec = ["sources=%s" % sources, "default=%s" % self.default] + spec.append("%d parsers" % (len(self.parsers))) + return 'Option(%s: %s)' % (self.dest, ", ".join(spec)) + + def list_options(self): + """ List options contained in this option. This exists to + provide a consistent interface with + :class:`Bcfg2.Options.OptionGroup` """ + return [self] + + def finalize(self, namespace): + """ Finalize the default value for this option. This is used + with actions (such as :class:`Bcfg2.Options.ComponentAction`) + that allow you to specify a default in a different format than + its final storage format; this can be called after it has been + determined that the default will be used (i.e., the option is + not given on the command line or in the config file) to store + the appropriate default value in the appropriate format.""" + for parser, action in self.actions.items(): + if hasattr(action, "finalize"): + action.finalize(parser, namespace) + + def from_config(self, cfp): + """ Get the value of this option from the given + :class:`ConfigParser.ConfigParser`. If it is not found in the + config file, the default is returned. (If there is no + default, None is returned.) + + :param cfp: The config parser to get the option value from + :type cfp: ConfigParser.ConfigParser + :returns: The default value + """ + if not self.cf: + return None + if '*' in self.cf[1]: + if cfp.has_section(self.cf[0]): + # build a list of known options in this section, and + # exclude them + exclude = set() + for parser in self.parsers: + exclude.update(o.cf[1] + for o in parser.option_list + if o.cf and o.cf[0] == self.cf[0]) + return dict([(o, cfp.get(self.cf[0], o)) + for o in fnmatch.filter(cfp.options(self.cf[0]), + self.cf[1]) + if o not in exclude]) + else: + return dict() + else: + try: + val = cfp.getboolean(*self.cf) + except ValueError: + val = cfp.get(*self.cf) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return None + if self.type: + return self.type(val) + else: + return val + + def default_from_config(self, cfp): + """ Set the default value of this option from the config file + or from the environment. + + :param cfp: The config parser to get the option value from + :type cfp: ConfigParser.ConfigParser + """ + if self.env and self.env in os.environ: + self.default = os.environ[self.env] + else: + val = self.from_config(cfp) + if val is not None: + self.default = val + + def _get_default(self): + """ Getter for the ``default`` property """ + return self._default + + def _set_default(self, value): + """ Setter for the ``default`` property """ + self._default = value + for action in self.actions.values(): + action.default = value + + #: The current default value of this option + default = property(_get_default, _set_default) + + def _get_dest(self): + """ Getter for the ``dest`` property """ + return self._dest + + def _set_dest(self, value): + """ Setter for the ``dest`` property """ + self._dest = value + for action in self.actions.values(): + action.dest = value + + #: The namespace destination of this option (see `dest + #: <http://docs.python.org/dev/library/argparse.html#dest>`_) + dest = property(_get_dest, _set_dest) + + def add_to_parser(self, parser): + """ Add this option to the given parser. + + :param parser: The parser to add the option to. + :type parser: Bcfg2.Options.Parser + :returns: argparse.Action + """ + self.parsers.append(parser) + if self.args: + # cli option + action = parser.add_argument(*self.args, **self._kwargs) + if not self._dest: + self._dest = action.dest + if self._default: + action.default = self._default + self.actions[parser] = action + # else, config file-only option + + +class PathOption(Option): + """ Shortcut for options that expect a path argument. Uses + :meth:`Bcfg2.Options.Types.path` to transform the argument into a + canonical path. + + The type of a path option can also be overridden to return an + option file-like object. For example: + + .. code-block:: python + + options = [ + Bcfg2.Options.PathOption( + "--input", type=argparse.FileType('r'), + help="The input file")] + """ + + def __init__(self, *args, **kwargs): + kwargs.setdefault('type', Types.path) + kwargs.setdefault('metavar', '<path>') + Option.__init__(self, *args, **kwargs) + + +class BooleanOption(Option): + """ Shortcut for boolean options. The default is False, but this + can easily be overridden: + + .. code-block:: python + + options = [ + Bcfg2.Options.PathOption( + "--dwim", default=True, help="Do What I Mean")] + """ + def __init__(self, *args, **kwargs): + if 'default' in kwargs and kwargs['default']: + kwargs.setdefault('action', 'store_false') + else: + kwargs.setdefault('action', 'store_true') + kwargs.setdefault('default', False) + Option.__init__(self, *args, **kwargs) + + +class PositionalArgument(Option): + """ Shortcut for positional arguments. """ + def __init__(self, *args, **kwargs): + if 'metavar' not in kwargs: + kwargs['metavar'] = '<%s>' % args[0] + Option.__init__(self, *args, **kwargs) diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py new file mode 100644 index 000000000..b672aebb9 --- /dev/null +++ b/src/lib/Bcfg2/Options/Parser.py @@ -0,0 +1,311 @@ +""" The option parser """ + +import os +import sys +import argparse +from Bcfg2.version import __version__ +from Bcfg2.Compat import ConfigParser +from Bcfg2.Options import Option, PathOption, BooleanOption + +__all__ = ["setup", "OptionParserException", "Parser", "get_parser"] + + +#: The repository option. This is specified here (and imported into +#: :module:`Bcfg2.Options.Common`) rather than vice-versa due to +#: circular imports. +repository = PathOption( # pylint: disable=C0103 + '-Q', '--repository', cf=('server', 'repository'), + default='var/lib/bcfg2', help="Server repository path") + + +#: A module-level :class:`argparse.Namespace` object that stores all +#: configuration for Bcfg2. +setup = argparse.Namespace(version=__version__, # pylint: disable=C0103 + name="Bcfg2", + uri='http://trac.mcs.anl.gov/projects/bcfg2') + + +class OptionParserException(Exception): + """ Base exception raised for generic option parser errors """ + + +class Parser(argparse.ArgumentParser): + """ The Bcfg2 option parser. Most interfaces should not need to + instantiate a parser, but should instead use + :func:`Bcfg2.Options.get_parser` to get the parser that already + exists.""" + + #: Option for specifying the path to the Bcfg2 config file + configfile = PathOption('-C', '--config', + help="Path to configuration file", + default="/etc/bcfg2.conf") + + #: Builtin options that apply to all commands + options = [configfile, + BooleanOption('--version', help="Print the version and exit"), + Option('-E', '--encoding', metavar='<encoding>', + default='UTF-8', help="Encoding of config files", + cf=('components', 'encoding'))] + + #: Flag used in unit tests to disable actual config file reads + unit_test = False + + def __init__(self, **kwargs): + """ See :class:`argparse.ArgumentParser` for a full list of + accepted parameters. + + In addition to supporting all arguments and keyword arguments + from :class:`argparse.ArgumentParser`, several additional + keyword arguments are allowed. + + :param components: A list of components to add to the parser. + :type components: list + :param namespace: The namespace to store options in. Default + is :attr:`Bcfg2.Options.setup`. + :type namespace: argparse.Namespace + :param add_base_options: Whether or not to add the options in + :attr:`Bcfg2.Options.Parser.options` + to the parser. Setting this to False + is default for subparsers. Default is + True. + :type add_base_options: bool + """ + self._cfp = ConfigParser.ConfigParser() + components = kwargs.pop('components', []) + + #: The namespace options will be stored in. + self.namespace = kwargs.pop('namespace', setup) + if self.namespace is None: + self.namespace = setup + add_base_options = kwargs.pop('add_base_options', True) + + #: Flag to indicate that this is the pre-parsing 'early' run + #: for important options like database settings that must be + #: loaded before other components can be. + self._early = kwargs.pop('early', False) + + if 'add_help' not in kwargs: + kwargs['add_help'] = add_base_options + argparse.ArgumentParser.__init__(self, **kwargs) + + #: Whether or not parsing has completed on all current options. + self.parsed = False + + #: The argument list that was parsed. + self.argv = None + + #: Components that have been added to the parser + self.components = [] + + #: Options that have been added to the parser + self.option_list = [] + self._defaults_set = [] + self._config_files = [] + if add_base_options: + self.add_component(self) + if components: + for component in components: + self.add_component(component) + + def add_options(self, options): + """ Add an explicit list of options to the parser. When + possible, prefer :func:`Bcfg2.Options.Parser.add_component` to + add a whole component instead.""" + self.parsed = False + for option in options: + if option not in self.option_list: + self.option_list.extend(option.list_options()) + option.add_to_parser(self) + + def add_component(self, component): + """ Add a component (and all of its options) to the + parser. """ + if component not in self.components: + self.components.append(component) + if hasattr(component, "options"): + self.add_options(getattr(component, "options")) + + def _set_defaults(self): + """ Set defaults from the config file for all options that can + come from the config file, but haven't yet had their default + set """ + for opt in self.option_list: + if opt not in self._defaults_set: + opt.default_from_config(self._cfp) + self._defaults_set.append(opt) + + def _parse_config_options(self): + """ populate the namespace with default values for any options + that aren't already in the namespace (i.e., options without + CLI arguments) """ + for opt in self.option_list[:]: + if not opt.args and opt.dest not in self.namespace: + value = opt.default + if value: + for parser, action in opt.actions.items(): + if parser is None: + action(self, self.namespace, value) + else: + action(parser, parser.namespace, value) + else: + setattr(self.namespace, opt.dest, value) + + def _finalize(self): + """ Finalize the value of any options that require that + additional post-processing step. (Mostly + :class:`Bcfg2.Options.Actions.ComponentAction` subclasses.) + """ + for opt in self.option_list[:]: + opt.finalize(self.namespace) + + def _reset_namespace(self): + """ Delete all options from the namespace except for a few + predefined values and config file options. """ + self.parsed = False + for attr in dir(self.namespace): + if (not attr.startswith("_") and + attr not in ['uri', 'version', 'name'] and + attr not in self._config_files): + delattr(self.namespace, attr) + + def add_config_file(self, dest, cfile, reparse=True): + """ Add a config file, which triggers a full reparse of all + options. """ + if dest not in self._config_files: + self._reset_namespace() + self._cfp.read([cfile]) + self._defaults_set = [] + self._set_defaults() + if reparse: + self._parse_config_options() + self._config_files.append(dest) + + def reparse(self, argv=None): + """ Reparse options after they have already been parsed. + + :param argv: The argument list to parse. By default, + :attr:`Bcfg2.Options.Parser.argv` is reused. + (I.e., the argument list that was initially + parsed.) :type argv: list + """ + self._reset_namespace() + self.parse(argv or self.argv) + + def parse(self, argv=None): + """ Parse options. + + :param argv: The argument list to parse. By default, + ``sys.argv[1:]`` is used. This is stored in + :attr:`Bcfg2.Options.Parser.argv` for reuse by + :func:`Bcfg2.Options.Parser.reparse`. + :type argv: list + """ + if argv is None: + argv = sys.argv[1:] + if self.parsed and self.argv == argv: + return self.namespace + self.argv = argv + + # phase 1: get and read config file + bootstrap_parser = argparse.ArgumentParser(add_help=False) + self.configfile.add_to_parser(bootstrap_parser) + bootstrap = bootstrap_parser.parse_known_args(args=self.argv)[0] + + # check whether the specified bcfg2.conf exists + if not self.unit_test and not os.path.exists(bootstrap.config): + self.error("Could not read %s" % bootstrap.config) + self.add_config_file(self.configfile.dest, bootstrap.config, + reparse=False) + + # phase 2: re-parse command line for early options; currently, + # that's database options + if not self._early: + early_opts = argparse.Namespace() + early_parser = Parser(add_help=False, namespace=early_opts, + early=True) + # add the repo option so we can resolve <repository> + # macros + early_parser.add_options([repository]) + early_components = [] + for component in self.components: + if getattr(component, "parse_first", False): + early_components.append(component) + early_parser.add_component(component) + early_parser.parse(self.argv) + for component in early_components: + if hasattr(component, "component_parsed_hook"): + getattr(component, "component_parsed_hook")(early_opts) + + # phase 3: re-parse command line, loading additional + # components, until all components have been loaded. On each + # iteration, set defaults from config file/environment + # variables + while not self.parsed: + self.parsed = True + self._set_defaults() + self.parse_known_args(args=self.argv, namespace=self.namespace) + self._parse_config_options() + self._finalize() + self._parse_config_options() + + # phase 4: fix up <repository> macros + repo = getattr(self.namespace, "repository", repository.default) + for attr in dir(self.namespace): + value = getattr(self.namespace, attr) + if not attr.startswith("_") and hasattr(value, "replace"): + setattr(self.namespace, attr, + value.replace("<repository>", repo, 1)) + + # phase 5: call post-parsing hooks + if not self._early: + for component in self.components: + if hasattr(component, "options_parsed_hook"): + getattr(component, "options_parsed_hook")() + + return self.namespace + + +#: A module-level :class:`Bcfg2.Options.Parser` object that is used +#: for all parsing +_parser = Parser() # pylint: disable=C0103 + + +def get_parser(description=None, components=None, namespace=None): + """ Get an existing :class:`Bcfg2.Options.Parser` object. (One is + created at the module level when :mod:`Bcfg2.Options` is + imported.) If no arguments are given, then the existing parser is + simply fetched. + + If arguments are given, then one of two things happens: + + * If this is the first ``get_parser`` call with arguments, then + the values given are set accordingly in the parser, and it is + returned. + * If this is not the first such call, then + :class:`Bcfg2.Options.OptionParserException` is raised. + + That is, a ``get_parser`` call with options is considered to + initialize the parser that already exists, and that can only + happen once. + + :param description: Set the parser description + :type description: string + :param components: Load the given components in the parser + :type components: list + :param namespace: Use the given namespace instead of + :attr:`Bcfg2.Options.setup` + :type namespace: argparse.Namespace + :returns: Bcfg2.Options.Parser object + """ + if Parser.unit_test: + return Parser(description=description, components=components, + namespace=namespace) + elif (description or components or namespace): + if description: + _parser.description = description + if components is not None: + for component in components: + _parser.add_component(component) + if namespace: + _parser.namespace = namespace + return _parser diff --git a/src/lib/Bcfg2/Options/Subcommands.py b/src/lib/Bcfg2/Options/Subcommands.py new file mode 100644 index 000000000..660bd5077 --- /dev/null +++ b/src/lib/Bcfg2/Options/Subcommands.py @@ -0,0 +1,238 @@ +""" Classes to make it easier to create commands with large numbers of +subcommands (e.g., bcfg2-admin, bcfg2-info). """ + +import re +import cmd +import sys +import copy +import shlex +import logging +from Bcfg2.Compat import StringIO +from Bcfg2.Options import PositionalArgument +from Bcfg2.Options.OptionGroups import Subparser +from Bcfg2.Options.Parser import Parser, setup as master_setup + +__all__ = ["Subcommand", "HelpCommand", "CommandRegistry", "register_commands"] + + +class Subcommand(object): + """ Base class for subcommands. This must be subclassed to create + commands. + + Specifically, you must override + :func:`Bcfg2.Options.Subcommand.run`. You may want to override: + + * The docstring, which will be used as the short help. + * :attr:`Bcfg2.Options.Subcommand.options` + * :attr:`Bcfg2.Options.Subcommand.help` + * :attr:`Bcfg2.Options.Subcommand.interactive` + * + * :func:`Bcfg2.Options.Subcommand.shutdown` + + You should not need to override + :func:`Bcfg2.Options.Subcommand.__call__` or + :func:`Bcfg2.Options.Subcommand.usage`. + + A ``Subcommand`` subclass constructor must not take any arguments. + """ + + #: Options this command takes + options = [] + + #: Longer help message + help = None + + #: Whether or not to expose this command in an interactive + #: :class:`cmd.Cmd` shell, if one is used. (``bcfg2-info`` uses + #: one, ``bcfg2-admin`` does not.) + interactive = True + + _ws_re = re.compile(r'\s+', flags=re.MULTILINE) + + def __init__(self): + self.core = None + description = "%s: %s" % (self.__class__.__name__.lower(), + self.__class__.__doc__) + + #: The :class:`Bcfg2.Options.Parser` that will be used to + #: parse options if this subcommand is called from an + #: interactive :class:`cmd.Cmd` shell. + self.parser = Parser( + prog=self.__class__.__name__.lower(), + description=description, + components=[self], + add_base_options=False, + epilog=self.help) + self._usage = None + + #: A :class:`logging.Logger` that can be used to produce + #: logging output for this command. + self.logger = logging.getLogger(self.__class__.__name__.lower()) + + def __call__(self, args=None): + """ Perform option parsing and other tasks necessary to + support running ``Subcommand`` objects as part of a + :class:`cmd.Cmd` shell. You should not need to override + ``__call__``. + + :param args: Arguments given in the interactive shell + :type args: list of strings + :returns: The return value of :func:`Bcfg2.Options.Subcommand.run` + """ + if args is not None: + self.parser.namespace = copy.copy(master_setup) + alist = shlex.split(args) + try: + setup = self.parser.parse(alist) + except SystemExit: + return sys.exc_info()[1].code + return self.run(setup) + else: + return self.run(master_setup) + + def usage(self): + """ Get the short usage message. """ + if self._usage is None: + sio = StringIO() + self.parser.print_usage(file=sio) + usage = self._ws_re.sub(' ', sio.getvalue()).strip()[7:] + doc = self._ws_re.sub(' ', getattr(self, "__doc__")).strip() + if doc is None: + self._usage = usage + else: + self._usage = "%s - %s" % (usage, doc) + return self._usage + + def run(self, setup): + """ Run the command. + + :param setup: A namespace giving the options for this command. + This must be used instead of + :attr:`Bcfg2.Options.setup` because this command + may have been called from an interactive + :class:`cmd.Cmd` shell, and thus has its own + option parser and its own (private) namespace. + ``setup`` is guaranteed to contain all of the + options in the global + :attr:`Bcfg2.Options.setup` namespace, in + addition to any local options given to this + command from the interactive shell. + :type setup: argparse.Namespace + """ + raise NotImplementedError + + def shutdown(self): + """ Perform any necessary shtudown tasks for this command This + is called to when the program exits (*not* when this command + is finished executing). """ + pass + + +class HelpCommand(Subcommand): + """ Get help on a specific subcommand. This must be subclassed to + create the actual help command by overriding + :func:`Bcfg2.Options.HelpCommand.command_registry` and giving the + command access to a :class:`Bcfg2.Options.CommandRegistry`. """ + options = [PositionalArgument("command", nargs='?')] + + # the interactive shell has its own help + interactive = False + + def command_registry(self): + """ Return a :class:`Bcfg2.Options.CommandRegistry` class. + All commands registered with the class will be included in the + help message. """ + raise NotImplementedError + + def run(self, setup): + commands = self.command_registry() + if setup.command: + try: + commands[setup.command].parser.print_help() + return 0 + except KeyError: + print("No such command: %s" % setup.command) + for command in sorted(commands.keys()): + print(commands[command].usage()) + + +class CommandRegistry(object): + """ A ``CommandRegistry`` is used to register subcommands and + provides a single interface to run them. It's also used by + :class:`Bcfg2.Options.HelpCommand` to produce help messages for + all available commands. """ + + #: A dict of registered commands. Keys are the class names, + #: lowercased (i.e., the command names), and values are instances + #: of the command objects. + commands = dict() + + options = [] + + def runcommand(self): + """ Run the single command named in + ``Bcfg2.Options.setup.subcommand``, which is where + :class:`Bcfg2.Options.Subparser` groups store the + subcommand. """ + try: + return self.commands[master_setup.subcommand].run(master_setup) + finally: + self.shutdown() + + def shutdown(self): + """ Perform shutdown tasks. This calls the ``shutdown`` + method of all registered subcommands. """ + self.commands[master_setup.subcommand].shutdown() + + @classmethod + def register_command(cls, cmdcls): + """ Register a single command. + + :param cmdcls: The command class to register + :type cmdcls: type + :returns: An instance of ``cmdcls`` + """ + cmd_obj = cmdcls() + name = cmdcls.__name__.lower() + cls.commands[name] = cmd_obj + # py2.5 can't mix *magic and non-magical keyword args, thus + # the **dict(...) + cls.options.append( + Subparser(*cmdcls.options, **dict(name=name, help=cmdcls.__doc__))) + if issubclass(cls, cmd.Cmd) and cmdcls.interactive: + setattr(cls, "do_%s" % name, cmd_obj) + setattr(cls, "help_%s" % name, cmd_obj.parser.print_help) + return cmd_obj + + +def register_commands(registry, candidates, parent=Subcommand): + """ Register all subcommands in ``candidates`` against the + :class:`Bcfg2.Options.CommandRegistry` subclass given in + ``registry``. A command is registered if and only if: + + * It is a subclass of the given ``parent`` (by default, + :class:`Bcfg2.Options.Subcommand`); + * It is not the parent class itself; and + * Its name does not start with an underscore. + + :param registry: The :class:`Bcfg2.Options.CommandRegistry` + subclass against which commands will be + registered. + :type registry: Bcfg2.Options.CommandRegistry + :param candidates: A list of objects that will be considered for + registration. Only objects that meet the + criteria listed above will be registered. + :type candidates: list + :param parent: Specify a parent class other than + :class:`Bcfg2.Options.Subcommand` that all + registered commands must subclass. + :type parent: type + """ + for attr in candidates: + try: + if (issubclass(attr, parent) and + attr != parent and + not attr.__name__.startswith("_")): + registry.register_command(attr) + except TypeError: + pass diff --git a/src/lib/Bcfg2/Options/Types.py b/src/lib/Bcfg2/Options/Types.py new file mode 100644 index 000000000..2f0fd7d52 --- /dev/null +++ b/src/lib/Bcfg2/Options/Types.py @@ -0,0 +1,111 @@ +""" :mod:`Bcfg2.Options` provides a number of useful types for use +with the :class:`Bcfg2.Options.Option` constructor. """ + +import os +import re +import pwd +import grp + +_COMMA_SPLIT_RE = re.compile(r'\s*,\s*') + + +def path(value): + """ A generic path. ``~`` will be expanded with + :func:`os.path.expanduser` and the absolute resulting path will be + used. This does *not* ensure that the path exists. """ + return os.path.abspath(os.path.expanduser(value)) + + +def comma_list(value): + """ Split a comma-delimited list, with optional whitespace around + the commas.""" + return _COMMA_SPLIT_RE.split(value) + + +def colon_list(value): + """ Split a colon-delimited list. Whitespace is not allowed + around the colons. """ + return value.split(':') + + +def comma_dict(value): + """ Split an option string on commas, optionally surrounded by + whitespace, and split the resulting items again on equals signs, + returning a dict """ + result = dict() + if value: + items = comma_list(value) + for item in items: + if '=' in item: + key, value = item.split(r'=', 1) + try: + result[key] = bool(value) + except ValueError: + try: + result[key] = int(value) + except ValueError: + result[key] = value + else: + result[item] = True + return result + + +def octal(value): + """ Given an octal string, get an integer representation. """ + return int(value, 8) + + +def username(value): + """ Given a username or numeric UID, get a numeric UID. The user + must exist.""" + try: + return int(value) + except ValueError: + return int(pwd.getpwnam(value)[2]) + + +def groupname(value): + """ Given a group name or numeric GID, get a numeric GID. The + user must exist.""" + try: + return int(value) + except ValueError: + return int(grp.getgrnam(value)[2]) + + +def timeout(value): + """ Convert the value into a float or None. """ + if value is None: + return value + rv = float(value) # pass ValueError up the stack + if rv <= 0: + return None + return rv + + +# pylint: disable=C0103 +_bytes_multipliers = dict(k=1, + m=2, + g=3, + t=4) +_suffixes = "".join(_bytes_multipliers.keys()).lower() +_suffixes += _suffixes.upper() +_bytes_re = re.compile(r'(?P<value>\d+)(?P<multiplier>[%s])?' % _suffixes) +# pylint: enable=C0103 + + +def size(value): + """ Given a number of bytes in a human-readable format (e.g., + '512m', '2g'), get the absolute number of bytes as an integer. + """ + if value == -1: + return value + mat = _bytes_re.match(value) + if not mat: + raise ValueError("Not a valid size", value) + rvalue = int(mat.group("value")) + mult = mat.group("multiplier") + if mult: + return rvalue * (1024 ** _bytes_multipliers[mult.lower()]) + else: + return rvalue diff --git a/src/lib/Bcfg2/Options/__init__.py b/src/lib/Bcfg2/Options/__init__.py new file mode 100644 index 000000000..96465ec56 --- /dev/null +++ b/src/lib/Bcfg2/Options/__init__.py @@ -0,0 +1,10 @@ +""" Bcfg2 options parsing. """ + +# pylint: disable=W0611,W0401 +from Bcfg2.Options import Types +from Bcfg2.Options.Options import * +from Bcfg2.Options.Common import * +from Bcfg2.Options.Parser import * +from Bcfg2.Options.Actions import * +from Bcfg2.Options.Subcommands import * +from Bcfg2.Options.OptionGroups import * |