diff options
Diffstat (limited to 'src/lib')
33 files changed, 763 insertions, 503 deletions
diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index cf4e7c7ea..739ba013f 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -5,6 +5,7 @@ import warnings warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) import os +import sys import apt.cache import Bcfg2.Options import Bcfg2.Client.Tools @@ -12,7 +13,7 @@ import Bcfg2.Client.Tools class APT(Bcfg2.Client.Tools.Tool): """The Debian toolset implements package and service operations - and inherits the rest from Tools.Tool. """ + and inherits the rest from Tools.Tool.""" options = Bcfg2.Client.Tools.Tool.options + [ Bcfg2.Options.PathOption( @@ -79,33 +80,30 @@ class APT(Bcfg2.Client.Tools.Tool): try: self.pkg_cache = apt.cache.Cache() except SystemError: - e = sys.exc_info()[1] - self.logger.info("Failed to initialize APT cache: %s" % e) + err = sys.exc_info()[1] + self.logger.info("Failed to initialize APT cache: %s" % err) raise Bcfg2.Client.Tools.ToolInstantiationError - self.pkg_cache.update() + try: + self.pkg_cache.update() + except apt.cache.FetchFailedException: + err = sys.exc_info()[1] + self.logger.info("Failed to update APT cache: %s" % err) self.pkg_cache = apt.cache.Cache() - if 'req_reinstall_pkgs' in dir(self.pkg_cache): - self._newapi = True - else: - self._newapi = False def FindExtra(self): """Find extra packages.""" packages = [entry.get('name') for entry in self.getSupportedEntries()] - if self._newapi: - extras = [(p.name, p.installed.version) for p in self.pkg_cache - if p.is_installed and p.name not in packages] - else: - extras = [(p.name, p.installedVersion) for p in self.pkg_cache - if p.isInstalled and p.name not in packages] + extras = [(p.name, p.installed.version) for p in self.pkg_cache + if p.is_installed and p.name not in packages] return [Bcfg2.Client.XML.Element('Package', name=name, type='deb', version=version) for (name, version) in extras] def VerifyDebsums(self, entry, modlist): + """Verify the package contents with debsum information.""" output = \ self.cmd.run("%s -as %s" % - (self.debsums, entry.get('name'))).stdout.splitlines() + (self.debsums, entry.get('name'))).stderr.splitlines() if len(output) == 1 and "no md5sums for" in output[0]: self.logger.info("Package %s has no md5sums. Cannot verify" % entry.get('name')) @@ -127,11 +125,11 @@ class APT(Bcfg2.Client.Tools.Tool): # these files should not exist continue elif "is not installed" in item or "missing file" in item: - self.logger.error("Package %s is not fully installed" % - entry.get('name')) + self.logger.error("Package %s is not fully installed" + % entry.get('name')) else: - self.logger.error("Got Unsupported pattern %s from debsums" % - item) + self.logger.error("Got Unsupported pattern %s from debsums" + % item) files.append(item) files = list(set(files) - set(self.ignores)) # We check if there is file in the checksum to do @@ -142,67 +140,53 @@ class APT(Bcfg2.Client.Tools.Tool): bad = [filename for filename in files if filename not in modlist] if bad: self.logger.debug("It is suggested that you either manage " - "these files, revert the changes, or ignore " - "false failures:") - self.logger.info("Package %s failed validation. Bad files " - "are:" % entry.get('name')) + "these files, revert the changes, or " + "ignore false failures:") + self.logger.info("Package %s failed validation. Bad files are:" + % entry.get('name')) self.logger.info(bad) - entry.set('qtext', - "Reinstall Package %s-%s to fix failing files? " - "(y/N) " % (entry.get('name'), entry.get('version'))) + entry.set( + 'qtext', + "Reinstall Package %s-%s to fix failing files? (y/N) " + % (entry.get('name'), entry.get('version'))) return False return True def VerifyPackage(self, entry, modlist, checksums=True): """Verify package for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (entry.attrib['name'])) return False pkgname = entry.get('name') - if self.pkg_cache.has_key(pkgname): # nopep8 - if self._newapi: - is_installed = self.pkg_cache[pkgname].is_installed - else: - is_installed = self.pkg_cache[pkgname].isInstalled - if not self.pkg_cache.has_key(pkgname) or not is_installed: # nopep8 + if pkgname not in self.pkg_cache or \ + not self.pkg_cache[pkgname].is_installed: self.logger.info("Package %s not installed" % (entry.get('name'))) entry.set('current_exists', 'false') return False pkg = self.pkg_cache[pkgname] - if self._newapi: - installed_version = pkg.installed.version - candidate_version = pkg.candidate.version - else: - installed_version = pkg.installedVersion - candidate_version = pkg.candidateVersion + installed_version = pkg.installed.version if entry.get('version') == 'auto': - if self._newapi: - is_upgradable = \ - self.pkg_cache._depcache.is_upgradable(pkg._pkg) - else: - is_upgradable = \ - self.pkg_cache._depcache.IsUpgradable(pkg._pkg) - if is_upgradable: - desiredVersion = candidate_version + if pkg.is_upgradable: + desired_version = pkg.candidate.version else: - desiredVersion = installed_version + desired_version = installed_version elif entry.get('version') == 'any': - desiredVersion = installed_version + desired_version = installed_version else: - desiredVersion = entry.get('version') - if desiredVersion != installed_version: + desired_version = entry.get('version') + if desired_version != installed_version: entry.set('current_version', installed_version) entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % (entry.get('name'), entry.get('current_version'), - desiredVersion)) + desired_version)) return False else: # version matches - if (not Bcfg2.Options.setup.quick and - entry.get('verify', 'true') == 'true' - and checksums): + if not Bcfg2.Options.setup.quick \ + and entry.get('verify', 'true') == 'true' \ + and checksums: pkgsums = self.VerifyDebsums(entry, modlist) return pkgsums return True @@ -215,21 +199,8 @@ class APT(Bcfg2.Client.Tools.Tool): self.logger.info('Removing packages:') self.logger.info(pkgnames) for pkg in pkgnames.split(" "): - try: - if self._newapi: - self.pkg_cache[pkg].mark_delete(purge=True) - else: - self.pkg_cache[pkg].markDelete(purge=True) - except: - if self._newapi: - self.pkg_cache[pkg].mark_delete() - else: - self.pkg_cache[pkg].markDelete() - try: - self.pkg_cache.commit() - except SystemExit: - # thank you python-apt 0.6 - pass + self.pkg_cache[pkg].mark_delete(purge=True) + self.pkg_cache.commit() self.pkg_cache = apt.cache.Cache() self.modified += packages self.extra = self.FindExtra() @@ -240,40 +211,28 @@ class APT(Bcfg2.Client.Tools.Tool): ipkgs = [] bad_pkgs = [] for pkg in packages: - if not self.pkg_cache.has_key(pkg.get('name')): # nopep8 - self.logger.error("APT has no information about package %s" % - (pkg.get('name'))) + pkgname = pkg.get('name') + if pkgname not in self.pkg_cache: + self.logger.error("APT has no information about package %s" + % pkgname) continue if pkg.get('version') in ['auto', 'any']: - if self._newapi: - try: - cversion = \ - self.pkg_cache[pkg.get('name')].candidate.version - ipkgs.append("%s=%s" % (pkg.get('name'), cversion)) - except AttributeError: - self.logger.error("Failed to find %s in apt package " - "cache" % pkg.get('name')) - continue - else: - cversion = self.pkg_cache[pkg.get('name')].candidateVersion - ipkgs.append("%s=%s" % (pkg.get('name'), cversion)) - continue - if self._newapi: - avail_vers = [ - x.ver_str for x in - self.pkg_cache[pkg.get('name')]._pkg.version_list] - else: - avail_vers = [ - x.VerStr for x in - self.pkg_cache[pkg.get('name')]._pkg.VersionList] + try: + ipkgs.append("%s=%s" % ( + pkgname, + self.pkg_cache[pkgname].candidate.version)) + except AttributeError: + self.logger.error("Failed to find %s in apt package " + "cache" % pkgname) + continue + avail_vers = self.pkg_cache[pkgname].versions.keys() if pkg.get('version') in avail_vers: - ipkgs.append("%s=%s" % (pkg.get('name'), pkg.get('version'))) + ipkgs.append("%s=%s" % (pkgname, pkg.get('version'))) continue else: - self.logger.error("Package %s: desired version %s not in %s" % - (pkg.get('name'), pkg.get('version'), - avail_vers)) - bad_pkgs.append(pkg.get('name')) + self.logger.error("Package %s: desired version %s not in %s" + % (pkgname, pkg.get('version'), avail_vers)) + bad_pkgs.append(pkgname) if bad_pkgs: self.logger.error("Cannot find correct versions of packages:") self.logger.error(bad_pkgs) @@ -290,6 +249,6 @@ class APT(Bcfg2.Client.Tools.Tool): self.modified.append(package) return states - def VerifyPath(self, entry, _): + def VerifyPath(self, entry, _): # pylint: disable=W0613 """Do nothing here since we only verify Path type=ignore.""" return True diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index dedc50d89..ca0502b75 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -2,7 +2,6 @@ import Bcfg2.Client.Tools from Bcfg2.Utils import safe_input -from Bcfg2.Client import matches_white_list, passes_black_list class Action(Bcfg2.Client.Tools.Tool): @@ -11,23 +10,6 @@ class Action(Bcfg2.Client.Tools.Tool): __handles__ = [('Action', None)] __req__ = {'Action': ['name', 'timing', 'when', 'command', 'status']} - def _action_allowed(self, action): - """ Return true if the given action is allowed to be run by - the whitelist or blacklist """ - if (Bcfg2.Options.setup.decision == 'whitelist' and - not matches_white_list(action, - Bcfg2.Options.setup.decision_list)): - self.logger.info("In whitelist mode: suppressing Action: %s" % - action.get('name')) - return False - if (Bcfg2.Options.setup.decision == 'blacklist' and - not passes_black_list(action, - Bcfg2.Options.setup.decision_list)): - self.logger.info("In blacklist mode: suppressing Action: %s" % - action.get('name')) - return False - return True - def RunAction(self, entry): """This method handles command execution and status return.""" shell = False @@ -76,7 +58,7 @@ class Action(Bcfg2.Client.Tools.Tool): states = dict() for action in bundle.findall("Action"): if action.get('timing') in ['post', 'both']: - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) return states @@ -87,7 +69,7 @@ class Action(Bcfg2.Client.Tools.Tool): for action in bundle.findall("Action"): if (action.get('timing') in ['post', 'both'] and action.get('when') != 'modified'): - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) return states diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index a7fcb6709..7200b0fc2 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -160,7 +160,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group") + if entry.get("name") in g[3] and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 5698f237a..332638de4 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,6 +4,8 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML +from Bcfg2.Compat import urlretrieve + # pylint: disable=C0103 noask = ''' @@ -37,6 +39,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): # noaskfile needs to live beyond __init__ otherwise file is removed self.noaskfile = tempfile.NamedTemporaryFile() self.noaskname = self.noaskfile.name + # for any pkg files downloaded + self.tmpfiles = [] try: self.noaskfile.write(noask) # flush admin file contents to disk @@ -45,6 +49,41 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.pkgtool[1]) except: # pylint: disable=W0702 self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1]) + self.origpkgtool = self.pkgtool + + def pkgmogrify(self, packages): + """ Take a list of pkg objects, check for a 'simplefile' attribute. + If present, insert a _sysv_pkg_path attribute to the package and + download the datastream format SYSV package to a temporary file. + """ + for pkg in packages: + if pkg.get('simplefile'): + tmpfile = tempfile.NamedTemporaryFile() + self.tmpfiles.append(tmpfile) + self.logger.info("Downloading %s to %s" % (pkg.get('url'), + tmpfile.name)) + urlretrieve(pkg.get('url'), tmpfile.name) + pkg.set('_sysv_pkg_path', tmpfile.name) + + def _get_package_command(self, packages): + """Override the default _get_package_command, replacing the attribute + 'url' if '_sysv_pkg_path' if necessary in the returned command + string + """ + if hasattr(self, 'origpkgtool'): + if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): + self.pkgtool = (self.pkgtool[0], ('%s %s', + ['_sysv_pkg_path', 'name'])) + else: + self.pkgtool = self.origpkgtool + + pkgcmd = super(SYSV, self)._get_package_command(packages) + self.logger.debug("Calling install command: %s" % pkgcmd) + return pkgcmd + + def Install(self, packages): + self.pkgmogrify(packages) + super(SYSV, self).Install(packages) def RefreshPackages(self): """Refresh memory hashes of packages.""" @@ -80,8 +119,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Package %s not installed" % entry.get("name")) else: - if (Bcfg2.Options.setup.quick or - entry.attrib.get('verify', 'true') == 'false'): + if Bcfg2.Options.setup.quick or \ + entry.attrib.get('verify', 'true') == 'false': return True rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name')) if rv.success: diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 027d91c71..3b60c8285 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,15 +13,25 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} + def get_svc_name(self, service): + """Append .service to name if name doesn't specify a unit type.""" + svc = service.get('name') + if svc.endswith(('.service', '.socket', '.device', '.mount', + '.automount', '.swap', '.target', '.path', + '.timer', '.snapshot', '.slice', '.scope')): + return svc + else: + return '%s.service' % svc + def get_svc_command(self, service, action): - return "/bin/systemctl %s %s.service" % (action, service.get('name')) + return "/bin/systemctl %s %s" % (action, self.get_svc_name(service)) def VerifyService(self, entry, _): """Verify Service status for entry.""" if entry.get('status') == 'ignore': return True - cmd = "/bin/systemctl status %s.service " % (entry.get('name')) + cmd = "/bin/systemctl status %s" % (self.get_svc_name(entry)) rv = self.cmd.run(cmd) if 'Loaded: error' in rv.stdout: diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 21fc05b0d..a8a80974a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -11,6 +11,7 @@ import yum.callbacks import yum.Errors import yum.misc import rpmUtils.arch +import rpmUtils.miscutils import Bcfg2.Client.XML import Bcfg2.Client.Tools import Bcfg2.Options @@ -148,12 +149,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): dest="yum_verify_flags", type=Bcfg2.Options.Types.comma_list, help="YUM verify flags"), Bcfg2.Options.Option( - cf=('YUM', 'disabled_plugins'), default=[], - type=Bcfg2.Options.Types.comma_list, dest="yum_disabled_plugins", + cf=('YUM', 'disabled_plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, dest="yum_disabled_plugins", help="YUM disabled plugins"), Bcfg2.Options.Option( - cf=('YUM', 'enabled_plugins'), default=[], - type=Bcfg2.Options.Types.comma_list, dest="yum_enabled_plugins", + cf=('YUM', 'enabled_plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, dest="yum_enabled_plugins", help="YUM enabled plugins")] pkgtype = 'yum' @@ -660,7 +661,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra.get('release', 'any')) entry.set('current_version', "%s:%s-%s" % current_evr) entry.set('version', "%s:%s-%s" % wanted_evr) - if yum.compareEVR(current_evr, wanted_evr) == 1: + if rpmUtils.miscutils.compareEVR(current_evr, wanted_evr) == 1: entry.set("package_fail_action", "downgrade") else: entry.set("package_fail_action", "update") @@ -976,8 +977,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra2string(build_yname(pkg.get('name'), inst))) continue status = self.instance_status[inst] - if (not status.get('installed', False) and - Bcfg2.Options.setup.yum_install_missing): + if not status.get('installed', False) and \ + Bcfg2.Options.setup.yum_install_missing: queue_pkg(pkg, inst, install_pkgs) elif (status.get('version_fail', False) and Bcfg2.Options.setup.yum_fix_version): diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index cd294db98..ae7fa3aed 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -129,6 +129,23 @@ class Tool(object): raise ToolInstantiationError("%s: %s not executable" % (self.name, filename)) + def _install_allowed(self, entry): + """ Return true if the given entry is allowed to be installed by + the whitelist or blacklist """ + if (Bcfg2.Options.setup.decision == 'whitelist' and + not Bcfg2.Client.matches_white_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In whitelist mode: suppressing Action: %s" % + entry.get('name')) + return False + if (Bcfg2.Options.setup.decision == 'blacklist' and + not Bcfg2.Client.passes_black_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In blacklist mode: suppressing Action: %s" % + entry.get('name')) + return False + return True + def BundleUpdated(self, bundle): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. @@ -587,7 +604,8 @@ class SvcTool(Tool): return for entry in bundle: - if not self.handlesEntry(entry): + if (not self.handlesEntry(entry) + or not self._install_allowed(entry)): continue estatus = entry.get('status') diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 049236e03..b8a75a0c5 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -20,6 +20,7 @@ except ImportError: # urllib imports try: from urllib import quote_plus + from urllib import urlretrieve from urlparse import urljoin, urlparse from urllib2 import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ @@ -27,7 +28,8 @@ try: except ImportError: from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, urlopen + HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ + urlopen, urlretrieve from urllib.error import HTTPError, URLError try: diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index b817ecb94..f5b5d16aa 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -212,7 +212,7 @@ class _OptionContainer(object): Bcfg2.Options.Option( cf=('database', 'engine'), default='sqlite3', help='Database engine', dest='db_engine'), - Bcfg2.Options.Option( + Bcfg2.Options.RepositoryMacroOption( cf=('database', 'name'), default='<repository>/etc/bcfg2.sqlite', help="Database name", dest="db_name"), Bcfg2.Options.Option( diff --git a/src/lib/Bcfg2/Options/Actions.py b/src/lib/Bcfg2/Options/Actions.py index 8b941f2bb..854e6039d 100644 --- a/src/lib/Bcfg2/Options/Actions.py +++ b/src/lib/Bcfg2/Options/Actions.py @@ -2,7 +2,8 @@ import sys import argparse -from Bcfg2.Options.Parser import get_parser +from Bcfg2.Options.Parser import get_parser, OptionParserException +from Bcfg2.Options.Options import _debug __all__ = ["ConfigFileAction", "ComponentAction", "PluginsAction"] @@ -101,7 +102,7 @@ class ComponentAction(FinalizableAction): fail_silently = False def __init__(self, *args, **kwargs): - if self.mapping: + if self.mapping and not self.islist: if 'choices' not in kwargs: kwargs['choices'] = self.mapping.keys() FinalizableAction.__init__(self, *args, **kwargs) @@ -112,9 +113,12 @@ class ComponentAction(FinalizableAction): try: return getattr(__import__(module, fromlist=[name]), name) except (AttributeError, ImportError): + msg = "Failed to load %s from %s: %s" % (name, module, + sys.exc_info()[1]) if not self.fail_silently: - print("Failed to load %s from %s: %s" % - (name, module, sys.exc_info()[1])) + print(msg) + else: + _debug(msg) return None def _load_component(self, name): @@ -143,7 +147,7 @@ class ComponentAction(FinalizableAction): if cls: get_parser().add_component(cls) elif not self.fail_silently: - print("Could not load component %s" % name) + raise OptionParserException("Could not load component %s" % name) return cls def __call__(self, parser, namespace, values, option_string=None): @@ -168,7 +172,10 @@ class ConfigFileAction(FinalizableAction): ``bcfg2-lint.conf``). """ def __call__(self, parser, namespace, values, option_string=None): - parser.add_config_file(self.dest, values, reparse=False) + if values: + parser.add_config_file(self.dest, values, reparse=False) + else: + _debug("No config file passed for %s" % self) FinalizableAction.__call__(self, parser, namespace, values, option_string=option_string) @@ -177,3 +184,4 @@ class PluginsAction(ComponentAction): """ :class:`Bcfg2.Options.ComponentAction` subclass for loading Bcfg2 server plugins. """ bases = ['Bcfg2.Server.Plugins'] + fail_silently = True diff --git a/src/lib/Bcfg2/Options/OptionGroups.py b/src/lib/Bcfg2/Options/OptionGroups.py index 465358fab..49340ab36 100644 --- a/src/lib/Bcfg2/Options/OptionGroups.py +++ b/src/lib/Bcfg2/Options/OptionGroups.py @@ -10,7 +10,7 @@ __all__ = ["OptionGroup", "ExclusiveOptionGroup", "Subparser", "WildcardSectionGroup"] -class OptionContainer(list): +class _OptionContainer(list): """ Parent class of all option groups """ def list_options(self): @@ -29,7 +29,7 @@ class OptionContainer(list): opt.add_to_parser(parser) -class OptionGroup(OptionContainer): +class OptionGroup(_OptionContainer): """ Generic option group that is used only to organize options. This uses :meth:`argparse.ArgumentParser.add_argument_group` behind the scenes. """ @@ -43,16 +43,16 @@ class OptionGroup(OptionContainer): :param description: A longer description of the option group :param description: string """ - OptionContainer.__init__(self, items) + _OptionContainer.__init__(self, items) self.title = kwargs.pop('title') self.description = kwargs.pop('description', None) def add_to_parser(self, parser): group = parser.add_argument_group(self.title, self.description) - OptionContainer.add_to_parser(self, group) + _OptionContainer.add_to_parser(self, group) -class ExclusiveOptionGroup(OptionContainer): +class ExclusiveOptionGroup(_OptionContainer): """ Option group that ensures that only one argument in the group is present. This uses :meth:`argparse.ArgumentParser.add_mutually_exclusive_group` @@ -66,15 +66,15 @@ class ExclusiveOptionGroup(OptionContainer): specified. :type required: boolean """ - OptionContainer.__init__(self, items) + _OptionContainer.__init__(self, items) self.required = kwargs.pop('required', False) def add_to_parser(self, parser): - group = parser.add_mutually_exclusive_group(required=self.required) - OptionContainer.add_to_parser(self, group) + _OptionContainer.add_to_parser( + self, parser.add_mutually_exclusive_group(required=self.required)) -class Subparser(OptionContainer): +class Subparser(_OptionContainer): """ Option group that adds options in it to a subparser. This uses a lot of functionality tied to `argparse Sub-commands <http://docs.python.org/dev/library/argparse.html#sub-commands>`_. @@ -99,7 +99,7 @@ class Subparser(OptionContainer): """ self.name = kwargs.pop('name') self.help = kwargs.pop('help', None) - OptionContainer.__init__(self, items) + _OptionContainer.__init__(self, items) def __repr__(self): return "%s %s(%s)" % (self.__class__.__name__, @@ -111,11 +111,11 @@ class Subparser(OptionContainer): self._subparsers[parser] = parser.add_subparsers(dest='subcommand') subparser = self._subparsers[parser].add_parser(self.name, help=self.help) - OptionContainer.add_to_parser(self, subparser) + _OptionContainer.add_to_parser(self, subparser) -class WildcardSectionGroup(OptionContainer, Option): - """ WildcardSectionGroups contain options that may exist in +class WildcardSectionGroup(_OptionContainer, Option): + """WildcardSectionGroups contain options that may exist in several different sections of the config that match a glob. It works by creating options on the fly to match the sections described in the glob. For example, consider: @@ -134,7 +134,7 @@ class WildcardSectionGroup(OptionContainer, Option): .. code-block:: python >>> Bcfg2.Options.setup - Namespace(myplugin_bar_description='Bar description', myplugin_bar_number=2, myplugin_foo_description='Foo description', myplugin_foo_number=1, myplugin_sections=['myplugin:foo', 'myplugin:bar']) + Namespace(myplugin_bar_description='Bar description', myplugin_myplugin_bar_number=2, myplugin_myplugin_foo_description='Foo description', myplugin_myplugin_foo_number=1, myplugin_sections=['myplugin:foo', 'myplugin:bar']) All options must have the same section glob. @@ -146,10 +146,10 @@ class WildcardSectionGroup(OptionContainer, Option): ``<destination>`` is the original `dest <http://docs.python.org/dev/library/argparse.html#dest>`_ of the option. ``<section>`` is the section that it's found in. - ``<prefix>`` is automatically generated from the section glob by - replacing all consecutive characters disallowed in Python variable - names into underscores. (This can be overridden with the - constructor.) + ``<prefix>`` is automatically generated from the section glob. + (This can be overridden with the constructor.) Both ``<section>`` + and ``<prefix>`` have had all consecutive characters disallowed in + Python variable names replaced with underscores. This group stores an additional option, the sections themselves, in an option given by ``<prefix>sections``. @@ -171,17 +171,17 @@ class WildcardSectionGroup(OptionContainer, Option): that match the glob. :param dest: string """ - OptionContainer.__init__(self, []) + _OptionContainer.__init__(self, []) self._section_glob = items[0].cf[0] # get a default destination self._prefix = kwargs.get("prefix", self._dest_re.sub('_', self._section_glob)) Option.__init__(self, dest=kwargs.get('dest', self._prefix + "sections")) - self._options = items + self.option_templates = items def list_options(self): - return [self] + OptionContainer.list_options(self) + return [self] + _OptionContainer.list_options(self) def from_config(self, cfp): sections = [] @@ -189,10 +189,12 @@ class WildcardSectionGroup(OptionContainer, Option): if fnmatch.fnmatch(section, self._section_glob): sections.append(section) newopts = [] - for opt_tmpl in self._options: + for opt_tmpl in self.option_templates: option = copy.deepcopy(opt_tmpl) option.cf = (section, option.cf[1]) - option.dest = self._prefix + section + "_" + option.dest + option.dest = "%s%s_%s" % (self._prefix, + self._dest_re.sub('_', section), + option.dest) newopts.append(option) self.extend(newopts) for parser in self.parsers: @@ -201,4 +203,17 @@ class WildcardSectionGroup(OptionContainer, Option): def add_to_parser(self, parser): Option.add_to_parser(self, parser) - OptionContainer.add_to_parser(self, parser) + _OptionContainer.add_to_parser(self, parser) + + def __eq__(self, other): + return (_OptionContainer.__eq__(self, other) and + self.option_templates == getattr(other, "option_templates", + None)) + + def __repr__(self): + if len(self) == 0: + return "%s(%s)" % (self.__class__.__name__, + ", ".join(".".join(o.cf) + for o in self.option_templates)) + else: + return _OptionContainer.__repr__(self) diff --git a/src/lib/Bcfg2/Options/Options.py b/src/lib/Bcfg2/Options/Options.py index 3874f810d..752e01b4e 100644 --- a/src/lib/Bcfg2/Options/Options.py +++ b/src/lib/Bcfg2/Options/Options.py @@ -1,17 +1,23 @@ -""" The base :class:`Bcfg2.Options.Option` object represents an -option. Unlike options in :mod:`argparse`, an Option object does not -need to be associated with an option parser; it exists on its own.""" +"""Base :class:`Bcfg2.Options.Option` object to represent an option. -import os +Unlike options in :mod:`argparse`, an Option object does not need to +be associated with an option parser; it exists on its own. +""" + +import argparse import copy import fnmatch -import argparse +import os +import sys + from Bcfg2.Options import Types from Bcfg2.Compat import ConfigParser -__all__ = ["Option", "BooleanOption", "PathOption", "PositionalArgument", - "_debug"] +__all__ = ["Option", "BooleanOption", "RepositoryMacroOption", "PathOption", + "PositionalArgument", "_debug"] + +unit_test = False # pylint: disable=C0103 def _debug(msg): @@ -19,8 +25,11 @@ def _debug(msg): they're options, after all -- so option parsing verbosity is enabled by changing this to True. The verbosity here is primarily of use to developers. """ - if os.environ.get('BCFG2_OPTIONS_DEBUG', '0') == '1': - print(msg) + if unit_test: + print("DEBUG: %s" % msg) + elif os.environ.get('BCFG2_OPTIONS_DEBUG', '0').lower() in ["true", "yes", + "on", "1"]: + sys.stderr.write("%s\n" % msg) #: A dict that records a mapping of argparse action name (e.g., @@ -37,7 +46,7 @@ def _get_action_class(action_name): on. So we just instantiate a dummy parser, add a dummy argument, and determine the class that way. """ if (isinstance(action_name, type) and - issubclass(action_name, argparse.Action)): + issubclass(action_name, argparse.Action)): return action_name if action_name not in _action_map: action = argparse.ArgumentParser().add_argument(action_name, @@ -133,10 +142,11 @@ class Option(object): self._dest = None if 'dest' in self._kwargs: self._dest = self._kwargs.pop('dest') - elif self.cf is not None: - self._dest = self.cf[1] elif self.env is not None: self._dest = self.env + elif self.cf is not None: + self._dest = self.cf[1] + self._dest = self._dest.lower().replace("-", "_") kwargs = copy.copy(self._kwargs) kwargs.pop("action", None) self.actions[None] = action_cls(self._dest, self._dest, **kwargs) @@ -149,9 +159,10 @@ class Option(object): sources.append("%s.%s" % self.cf) if self.env: sources.append("$" + self.env) - spec = ["sources=%s" % sources, "default=%s" % self.default] - spec.append("%d parsers" % (len(self.parsers))) - return 'Option(%s: %s)' % (self.dest, ", ".join(spec)) + spec = ["sources=%s" % sources, "default=%s" % self.default, + "%d parsers" % len(self.parsers)] + return '%s(%s: %s)' % (self.__class__.__name__, + self.dest, ", ".join(spec)) def list_options(self): """ List options contained in this option. This exists to @@ -175,6 +186,17 @@ class Option(object): _debug("Finalizing %s" % self) action.finalize(parser, namespace) + @property + def _type_func(self): + """get a function for converting a value to the option type. + + this always returns a callable, even when ``type`` is None. + """ + if self.type: + return self.type + else: + return lambda x: x + def from_config(self, cfp): """ Get the value of this option from the given :class:`ConfigParser.ConfigParser`. If it is not found in the @@ -201,21 +223,33 @@ class Option(object): self.cf[1]) if o not in exclude]) else: - rv = dict() + rv = {} else: - if self.type: - rtype = self.type - else: - rtype = lambda x: x try: - rv = rtype(cfp.getboolean(*self.cf)) - except ValueError: - rv = rtype(cfp.get(*self.cf)) + rv = self._type_func(self.get_config_value(cfp)) except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): rv = None - _debug("Setting %s from config file(s): %s" % (self, rv)) + _debug("Getting value of %s from config file(s): %s" % (self, rv)) return rv + def get_config_value(self, cfp): + """fetch a value from the config file. + + This is passed the config parser. Its result is passed to the + type function for this option. It can be overridden to, e.g., + handle boolean options. + """ + return cfp.get(*self.cf) + + def get_environ_value(self, value): + """fetch a value from the environment. + + This is passed the raw value from the environment variable, + and its result is passed to the type function for this + option. It can be overridden to, e.g., handle boolean options. + """ + return value + def default_from_config(self, cfp): """ Set the default value of this option from the config file or from the environment. @@ -224,7 +258,8 @@ class Option(object): :type cfp: ConfigParser.ConfigParser """ if self.env and self.env in os.environ: - self.default = os.environ[self.env] + self.default = self._type_func( + self.get_environ_value(os.environ[self.env])) _debug("Setting the default of %s from environment: %s" % (self, self.default)) else: @@ -257,6 +292,13 @@ class Option(object): for action in self.actions.values(): action.dest = value + def early_parsing_hook(self, early_opts): # pylint: disable=C0111 + """Hook called at the end of early option parsing. + + This can be used to save option values for macro fixup. + """ + pass + #: The namespace destination of this option (see `dest #: <http://docs.python.org/dev/library/argparse.html#dest>`_) dest = property(_get_dest, _set_dest) @@ -284,13 +326,65 @@ class Option(object): (self, parser)) -class PathOption(Option): - """ Shortcut for options that expect a path argument. Uses - :meth:`Bcfg2.Options.Types.path` to transform the argument into a - canonical path. +class RepositoryMacroOption(Option): + """Option that does translation of ``<repository>`` macros. - The type of a path option can also be overridden to return an - option file-like object. For example: + Macro translation is done on the fly instead of just fixing up all + values at the end of parsing because macro expansion needs to be + done before path canonicalization for + :class:`Bcfg2.Options.Options.PathOption`. + """ + repository = None + + def __init__(self, *args, **kwargs): + self._original_type = kwargs.pop('type', lambda x: x) + kwargs['type'] = self._type + kwargs.setdefault('metavar', '<path>') + Option.__init__(self, *args, **kwargs) + + def early_parsing_hook(self, early_opts): + if hasattr(early_opts, "repository"): + if self.__class__.repository is None: + _debug("Setting repository to %s for %s" % + (early_opts.repository, self.__class__.__name__)) + self.__class__.repository = early_opts.repository + else: + _debug("Repository is already set for %s" % self.__class__) + + def _get_default(self): + """ Getter for the ``default`` property """ + if not hasattr(self._default, "replace"): + return self._default + else: + return self._type(self._default) + + default = property(_get_default, Option._set_default) + + def transform_value(self, value): + """transform the value after macro expansion. + + this can be overridden to further transform the value set by + the user *after* macros are expanded, but before the user's + ``type`` function is applied. principally exists for + PathOption to canonicalize the path. + """ + return value + + def _type(self, value): + """Type function that fixes up <repository> macros.""" + if self.__class__.repository is None: + return value + else: + return self._original_type(self.transform_value( + value.replace("<repository>", self.__class__.repository))) + + +class PathOption(RepositoryMacroOption): + """Shortcut for options that expect a path argument. + + Uses :meth:`Bcfg2.Options.Types.path` to transform the argument + into a canonical path. The type of a path option can also be + overridden to return a file-like object. For example: .. code-block:: python @@ -298,30 +392,41 @@ class PathOption(Option): Bcfg2.Options.PathOption( "--input", type=argparse.FileType('r'), help="The input file")] - """ - def __init__(self, *args, **kwargs): - kwargs.setdefault('type', Types.path) - kwargs.setdefault('metavar', '<path>') - Option.__init__(self, *args, **kwargs) + PathOptions also do translation of ``<repository>`` macros. + """ + def transform_value(self, value): + return Types.path(value) class _BooleanOptionAction(argparse.Action): - """ BooleanOptionAction sets a boolean value in the following ways: + """BooleanOptionAction sets a boolean value. + - if None is passed, store the default - if the option_string is not None, then the option was passed on the command line, thus store the opposite of the default (this is the argparse store_true and store_false behavior) - if a boolean value is passed, use that + Makes a copy of the initial default, because otherwise the default + can be changed by config file settings or environment + variables. For instance, if a boolean option that defaults to True + was set to False in the config file, specifying the option on the + CLI would then set it back to True. + Defined here instead of :mod:`Bcfg2.Options.Actions` because otherwise - there is a circular import Options -> Actions -> Parser -> Options """ + there is a circular import Options -> Actions -> Parser -> Options. + """ + + def __init__(self, *args, **kwargs): + argparse.Action.__init__(self, *args, **kwargs) + self.original = self.default def __call__(self, parser, namespace, values, option_string=None): if values is None: setattr(namespace, self.dest, self.default) elif option_string is not None: - setattr(namespace, self.dest, not self.default) + setattr(namespace, self.dest, not self.original) else: setattr(namespace, self.dest, bool(values)) @@ -340,9 +445,25 @@ class BooleanOption(Option): kwargs.setdefault('action', _BooleanOptionAction) kwargs.setdefault('nargs', 0) kwargs.setdefault('default', False) - Option.__init__(self, *args, **kwargs) + def get_environ_value(self, value): + if value.lower() in ["false", "no", "off", "0"]: + return False + elif value.lower() in ["true", "yes", "on", "1"]: + return True + else: + raise ValueError("Invalid boolean value %s" % value) + + def get_config_value(self, cfp): + """fetch a value from the config file. + + This is passed the config parser. Its result is passed to the + type function for this option. It can be overridden to, e.g., + handle boolean options. + """ + return cfp.getboolean(*self.cf) + class PositionalArgument(Option): """ Shortcut for positional arguments. """ diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py index 677a69e4c..c846e8093 100644 --- a/src/lib/Bcfg2/Options/Parser.py +++ b/src/lib/Bcfg2/Options/Parser.py @@ -1,13 +1,15 @@ -""" The option parser """ +"""The option parser.""" +import argparse import os import sys -import argparse + from Bcfg2.version import __version__ from Bcfg2.Compat import ConfigParser from Bcfg2.Options import Option, PathOption, BooleanOption, _debug -__all__ = ["setup", "OptionParserException", "Parser", "get_parser"] +__all__ = ["setup", "OptionParserException", "Parser", "get_parser", + "new_parser"] #: The repository option. This is specified here (and imported into @@ -108,13 +110,28 @@ class Parser(argparse.ArgumentParser): for component in components: self.add_component(component) + def _check_duplicate_cf(self, option): + """Check for a duplicate config file option.""" + def add_options(self, options): """ Add an explicit list of options to the parser. When possible, prefer :func:`Bcfg2.Options.Parser.add_component` to add a whole component instead.""" + _debug("Adding options: %s" % options) self.parsed = False for option in options: if option not in self.option_list: + # check for duplicates + if (hasattr(option, "env") and option.env and + option.env in [o.env for o in self.option_list]): + raise OptionParserException( + "Duplicate environment variable option: %s" % + option.env) + if (hasattr(option, "cf") and option.cf and + option.cf in [o.cf for o in self.option_list]): + raise OptionParserException( + "Duplicate config file option: %s" % (option.cf,)) + self.option_list.extend(option.list_options()) option.add_to_parser(self) @@ -151,6 +168,8 @@ class Parser(argparse.ArgumentParser): (opt, value)) action(self, self.namespace, value) else: + _debug("Setting config file-only option %s to %s" % + (opt, value)) setattr(self.namespace, opt.dest, value) def _finalize(self): @@ -169,11 +188,58 @@ class Parser(argparse.ArgumentParser): _debug("Resetting namespace") for attr in dir(self.namespace): if (not attr.startswith("_") and - attr not in ['uri', 'version', 'name'] and - attr not in self._config_files): + attr not in ['uri', 'version', 'name'] and + attr not in self._config_files): _debug("Deleting %s" % attr) delattr(self.namespace, attr) + def _parse_early_options(self): + """Parse early options. + + Early options are options that need to be parsed before other + options for some reason. These fall into two basic cases: + + 1. Database options, which need to be parsed so that Django + modules can be imported, since Django configuration is all + done at import-time; + 2. The repository (``-Q``) option, so that ``<repository>`` + macros in other options can be resolved. + """ + _debug("Option parsing phase 2: Parse early options") + early_opts = argparse.Namespace() + early_parser = Parser(add_help=False, namespace=early_opts, + early=True) + + # add the repo option so we can resolve <repository> + # macros + early_parser.add_options([repository]) + + early_components = [] + for component in self.components: + if getattr(component, "parse_first", False): + early_components.append(component) + early_parser.add_component(component) + early_parser.parse(self.argv) + + _debug("Fixing up <repository> macros in early options") + for attr_name in dir(early_opts): + if not attr_name.startswith("_"): + attr = getattr(early_opts, attr_name) + if hasattr(attr, "replace"): + setattr(early_opts, attr_name, + attr.replace("<repository>", + early_opts.repository)) + + _debug("Early parsing complete, calling hooks") + for component in early_components: + if hasattr(component, "component_parsed_hook"): + _debug("Calling component_parsed_hook on %s" % component) + getattr(component, "component_parsed_hook")(early_opts) + _debug("Calling early parsing hooks; early options: %s" % + early_opts) + for option in self.option_list: + option.early_parsing_hook(early_opts) + def add_config_file(self, dest, cfile, reparse=True): """ Add a config file, which triggers a full reparse of all options. """ @@ -190,10 +256,11 @@ class Parser(argparse.ArgumentParser): def reparse(self, argv=None): """ Reparse options after they have already been parsed. - :param argv: The argument list to parse. By default, + :param argv: The argument list to parse. By default, :attr:`Bcfg2.Options.Parser.argv` is reused. (I.e., the argument list that was initially - parsed.) :type argv: list + parsed.) + :type argv: list """ _debug("Reparsing all options") self._reset_namespace() @@ -210,7 +277,7 @@ class Parser(argparse.ArgumentParser): """ _debug("Parsing options") if argv is None: - argv = sys.argv[1:] + argv = sys.argv[1:] # pragma: nocover if self.parsed and self.argv == argv: _debug("Returning already parsed namespace") return self.namespace @@ -231,25 +298,10 @@ class Parser(argparse.ArgumentParser): # phase 2: re-parse command line for early options; currently, # that's database options - _debug("Option parsing phase 2: Parse early options") if not self._early: - early_opts = argparse.Namespace() - early_parser = Parser(add_help=False, namespace=early_opts, - early=True) - # add the repo option so we can resolve <repository> - # macros - early_parser.add_options([repository]) - early_components = [] - for component in self.components: - if getattr(component, "parse_first", False): - early_components.append(component) - early_parser.add_component(component) - early_parser.parse(self.argv) - _debug("Early parsing complete, calling hooks") - for component in early_components: - if hasattr(component, "component_parsed_hook"): - _debug("Calling component_parsed_hook on %s" % component) - getattr(component, "component_parsed_hook")(early_opts) + self._parse_early_options() + else: + _debug("Skipping parsing phase 2 in early mode") # phase 3: re-parse command line, loading additional # components, until all components have been loaded. On each @@ -273,31 +325,34 @@ class Parser(argparse.ArgumentParser): # _parse_config_options is called, all config file options will get set # to their hardcoded defaults. This process defines the options in the # namespace and _parse_config_options will never look at them again. - self._set_defaults_from_config() - self._parse_config_options() + # + # we have to do the parsing in two loops: first, we squeeze as + # much data out of the config file as we can to ensure that + # all config file settings are read before we use any default + # values. then we can start looking at the command line. while not self.parsed: self.parsed = True self._set_defaults_from_config() - self.parse_known_args(args=self.argv, namespace=self.namespace) + self._parse_config_options() + self.parsed = False + remaining = [] + while not self.parsed: + self.parsed = True + _debug("Parsing known arguments") + try: + _, remaining = self.parse_known_args(args=self.argv, + namespace=self.namespace) + except OptionParserException: + self.error(sys.exc_info()[1]) + self._set_defaults_from_config() self._parse_config_options() self._finalize() + if len(remaining) and not self._early: + self.error("Unknown options: %s" % " ".join(remaining)) - # phase 4: fix up <repository> macros - _debug("Option parsing phase 4: Fix up macros") - repo = getattr(self.namespace, "repository", repository.default) - for attr in dir(self.namespace): - value = getattr(self.namespace, attr) - if (not attr.startswith("_") and - hasattr(value, "replace") and - "<repository>" in value): - setattr(self.namespace, attr, - value.replace("<repository>", repo, 1)) - _debug("Fixing up macros in %s: %s -> %s" % - (attr, value, getattr(self.namespace, attr))) - - # phase 5: call post-parsing hooks - _debug("Option parsing phase 5: Call hooks") + # phase 4: call post-parsing hooks if not self._early: + _debug("Option parsing phase 4: Call hooks") for component in self.components: if hasattr(component, "options_parsed_hook"): _debug("Calling post-parsing hook on %s" % component) @@ -311,23 +366,23 @@ class Parser(argparse.ArgumentParser): _parser = Parser() # pylint: disable=C0103 +def new_parser(): + """Create a new :class:`Bcfg2.Options.Parser` object. + + The new object can be retrieved with + :func:`Bcfg2.Options.get_parser`. This is useful for unit + testing. + """ + global _parser + _parser = Parser() + + def get_parser(description=None, components=None, namespace=None): - """ Get an existing :class:`Bcfg2.Options.Parser` object. (One is - created at the module level when :mod:`Bcfg2.Options` is - imported.) If no arguments are given, then the existing parser is - simply fetched. - - If arguments are given, then one of two things happens: - - * If this is the first ``get_parser`` call with arguments, then - the values given are set accordingly in the parser, and it is - returned. - * If this is not the first such call, then - :class:`Bcfg2.Options.OptionParserException` is raised. - - That is, a ``get_parser`` call with options is considered to - initialize the parser that already exists, and that can only - happen once. + """Get an existing :class:`Bcfg2.Options.Parser` object. + + A Parser is created at the module level when :mod:`Bcfg2.Options` + is imported. If any arguments are given, then the existing parser + is modified before being returned. :param description: Set the parser description :type description: string diff --git a/src/lib/Bcfg2/Options/Subcommands.py b/src/lib/Bcfg2/Options/Subcommands.py index 660bd5077..8972bde00 100644 --- a/src/lib/Bcfg2/Options/Subcommands.py +++ b/src/lib/Bcfg2/Options/Subcommands.py @@ -7,12 +7,13 @@ import sys import copy import shlex import logging + from Bcfg2.Compat import StringIO -from Bcfg2.Options import PositionalArgument +from Bcfg2.Options import PositionalArgument, _debug from Bcfg2.Options.OptionGroups import Subparser from Bcfg2.Options.Parser import Parser, setup as master_setup -__all__ = ["Subcommand", "HelpCommand", "CommandRegistry", "register_commands"] +__all__ = ["Subcommand", "CommandRegistry"] class Subcommand(object): @@ -96,8 +97,8 @@ class Subcommand(object): sio = StringIO() self.parser.print_usage(file=sio) usage = self._ws_re.sub(' ', sio.getvalue()).strip()[7:] - doc = self._ws_re.sub(' ', getattr(self, "__doc__")).strip() - if doc is None: + doc = self._ws_re.sub(' ', getattr(self, "__doc__") or "").strip() + if not doc: self._usage = usage else: self._usage = "%s - %s" % (usage, doc) @@ -119,120 +120,130 @@ class Subcommand(object): command from the interactive shell. :type setup: argparse.Namespace """ - raise NotImplementedError + raise NotImplementedError # pragma: nocover def shutdown(self): - """ Perform any necessary shtudown tasks for this command This + """ Perform any necessary shutdown tasks for this command This is called to when the program exits (*not* when this command is finished executing). """ - pass + pass # pragma: nocover -class HelpCommand(Subcommand): - """ Get help on a specific subcommand. This must be subclassed to - create the actual help command by overriding - :func:`Bcfg2.Options.HelpCommand.command_registry` and giving the - command access to a :class:`Bcfg2.Options.CommandRegistry`. """ +class Help(Subcommand): + """List subcommands and usage, or get help on a specific subcommand.""" options = [PositionalArgument("command", nargs='?')] # the interactive shell has its own help interactive = False - def command_registry(self): - """ Return a :class:`Bcfg2.Options.CommandRegistry` class. - All commands registered with the class will be included in the - help message. """ - raise NotImplementedError + def __init__(self, registry): + Subcommand.__init__(self) + self._registry = registry def run(self, setup): - commands = self.command_registry() + commands = self._registry.commands if setup.command: try: commands[setup.command].parser.print_help() return 0 except KeyError: print("No such command: %s" % setup.command) + return 1 for command in sorted(commands.keys()): print(commands[command].usage()) class CommandRegistry(object): - """ A ``CommandRegistry`` is used to register subcommands and - provides a single interface to run them. It's also used by - :class:`Bcfg2.Options.HelpCommand` to produce help messages for - all available commands. """ + """A ``CommandRegistry`` is used to register subcommands and provides + a single interface to run them. It's also used by + :class:`Bcfg2.Options.Subcommands.Help` to produce help messages + for all available commands. + """ + + def __init__(self): + #: A dict of registered commands. Keys are the class names, + #: lowercased (i.e., the command names), and values are instances + #: of the command objects. + self.commands = dict() - #: A dict of registered commands. Keys are the class names, - #: lowercased (i.e., the command names), and values are instances - #: of the command objects. - commands = dict() + #: A list of options that should be added to the option parser + #: in order to handle registered subcommands. + self.subcommand_options = [] - options = [] + #: the help command + self.help = Help(self) + self.register_command(self.help) def runcommand(self): """ Run the single command named in ``Bcfg2.Options.setup.subcommand``, which is where :class:`Bcfg2.Options.Subparser` groups store the subcommand. """ + _debug("Running subcommand %s" % master_setup.subcommand) try: return self.commands[master_setup.subcommand].run(master_setup) finally: self.shutdown() def shutdown(self): - """ Perform shutdown tasks. This calls the ``shutdown`` - method of all registered subcommands. """ + """Perform shutdown tasks. + + This calls the ``shutdown`` method of the subcommand that was + run. + """ + _debug("Shutting down subcommand %s" % master_setup.subcommand) self.commands[master_setup.subcommand].shutdown() - @classmethod - def register_command(cls, cmdcls): + def register_command(self, cls_or_obj): """ Register a single command. - :param cmdcls: The command class to register - :type cmdcls: type + :param cls_or_obj: The command class or object to register + :type cls_or_obj: type or Subcommand :returns: An instance of ``cmdcls`` """ - cmd_obj = cmdcls() + if isinstance(cls_or_obj, type): + cmdcls = cls_or_obj + cmd_obj = cmdcls() + else: + cmd_obj = cls_or_obj + cmdcls = cmd_obj.__class__ name = cmdcls.__name__.lower() - cls.commands[name] = cmd_obj + self.commands[name] = cmd_obj # py2.5 can't mix *magic and non-magical keyword args, thus # the **dict(...) - cls.options.append( + self.subcommand_options.append( Subparser(*cmdcls.options, **dict(name=name, help=cmdcls.__doc__))) - if issubclass(cls, cmd.Cmd) and cmdcls.interactive: - setattr(cls, "do_%s" % name, cmd_obj) - setattr(cls, "help_%s" % name, cmd_obj.parser.print_help) + if issubclass(self.__class__, cmd.Cmd) and cmdcls.interactive: + setattr(self, "do_%s" % name, cmd_obj) + setattr(self, "help_%s" % name, cmd_obj.parser.print_help) return cmd_obj - -def register_commands(registry, candidates, parent=Subcommand): - """ Register all subcommands in ``candidates`` against the - :class:`Bcfg2.Options.CommandRegistry` subclass given in - ``registry``. A command is registered if and only if: - - * It is a subclass of the given ``parent`` (by default, - :class:`Bcfg2.Options.Subcommand`); - * It is not the parent class itself; and - * Its name does not start with an underscore. - - :param registry: The :class:`Bcfg2.Options.CommandRegistry` - subclass against which commands will be - registered. - :type registry: Bcfg2.Options.CommandRegistry - :param candidates: A list of objects that will be considered for - registration. Only objects that meet the - criteria listed above will be registered. - :type candidates: list - :param parent: Specify a parent class other than - :class:`Bcfg2.Options.Subcommand` that all - registered commands must subclass. - :type parent: type - """ - for attr in candidates: - try: - if (issubclass(attr, parent) and - attr != parent and - not attr.__name__.startswith("_")): - registry.register_command(attr) - except TypeError: - pass + def register_commands(self, candidates, parent=Subcommand): + """ Register all subcommands in ``candidates`` against the + :class:`Bcfg2.Options.CommandRegistry` subclass given in + ``registry``. A command is registered if and only if: + + * It is a subclass of the given ``parent`` (by default, + :class:`Bcfg2.Options.Subcommand`); + * It is not the parent class itself; and + * Its name does not start with an underscore. + + :param registry: The :class:`Bcfg2.Options.CommandRegistry` + subclass against which commands will be + registered. + :type registry: Bcfg2.Options.CommandRegistry + :param candidates: A list of objects that will be considered for + registration. Only objects that meet the + criteria listed above will be registered. + :type candidates: list + :param parent: Specify a parent class other than + :class:`Bcfg2.Options.Subcommand` that all + registered commands must subclass. + :type parent: type + """ + for attr in candidates: + if (isinstance(attr, type) and + issubclass(attr, parent) and + attr != parent and + not attr.__name__.startswith("_")): + self.register_command(attr) diff --git a/src/lib/Bcfg2/Options/Types.py b/src/lib/Bcfg2/Options/Types.py index d11e54fba..ac099e135 100644 --- a/src/lib/Bcfg2/Options/Types.py +++ b/src/lib/Bcfg2/Options/Types.py @@ -19,12 +19,16 @@ def path(value): def comma_list(value): """ Split a comma-delimited list, with optional whitespace around the commas.""" + if value == '': + return [] return _COMMA_SPLIT_RE.split(value) def colon_list(value): """ Split a colon-delimited list. Whitespace is not allowed around the colons. """ + if value == '': + return [] return value.split(':') @@ -38,9 +42,11 @@ def comma_dict(value): for item in items: if '=' in item: key, value = item.split(r'=', 1) - try: - result[key] = bool(value) - except ValueError: + if value in ["true", "yes", "on"]: + result[key] = True + elif value in ["false", "no", "off"]: + result[key] = False + else: try: result[key] = int(value) except ValueError: @@ -107,8 +113,6 @@ def size(value): """ Given a number of bytes in a human-readable format (e.g., '512m', '2g'), get the absolute number of bytes as an integer. """ - if value == -1: - return value mat = _bytes_re.match(value) if not mat: raise ValueError("Not a valid size", value) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 12c9cdaa8..153809a35 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -1,3 +1,4 @@ +import os import sys import atexit import daemon @@ -5,13 +6,12 @@ import logging import time import threading -# pylint: disable=E0611 from lockfile import LockFailed, LockTimeout +# pylint: disable=E0611 try: - from lockfile.pidlockfile import PIDLockFile - from lockfile import Error as PIDFileError + from daemon.pidfile import TimeoutPIDLockFile except ImportError: - from daemon.pidlockfile import PIDLockFile, PIDFileError + from daemon.pidlockfile import TimeoutPIDLockFile # pylint: enable=E0611 import Bcfg2.Logger @@ -30,7 +30,7 @@ class ReportingError(Exception): class ReportingStoreThread(threading.Thread): """Thread for calling the storage backend""" def __init__(self, interaction, storage, group=None, target=None, - name=None, args=(), kwargs=None): + name=None, semaphore=None, args=(), kwargs=None): """Initialize the thread with a reference to the interaction as well as the storage engine to use""" threading.Thread.__init__(self, group, target, name, args, @@ -38,26 +38,37 @@ class ReportingStoreThread(threading.Thread): self.interaction = interaction self.storage = storage self.logger = logging.getLogger('bcfg2-report-collector') + self.semaphore = semaphore def run(self): """Call the database storage procedure (aka import)""" try: - start = time.time() - self.storage.import_interaction(self.interaction) - self.logger.info("Imported interaction for %s in %ss" % - (self.interaction.get('hostname', '<unknown>'), - time.time() - start)) - except: - #TODO requeue? - self.logger.error("Unhandled exception in import thread %s" % - sys.exc_info()[1]) + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', + '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + sys.exc_info()[1]) + finally: + if self.semaphore: + self.semaphore.release() class ReportingCollector(object): """The collecting process for reports""" options = [Bcfg2.Options.Common.reporting_storage, Bcfg2.Options.Common.reporting_transport, - Bcfg2.Options.Common.daemon] + Bcfg2.Options.Common.daemon, + Bcfg2.Options.Option( + '--max-children', dest="children", + cf=('reporting', 'max_children'), type=int, + default=0, + help='Maximum number of children for the reporting collector')] def __init__(self): """Setup the collector. This may be called by the daemon or though @@ -67,6 +78,11 @@ class ReportingCollector(object): self.children = [] self.cleanup_threshold = 25 + self.semaphore = None + if Bcfg2.Options.setup.children > 0: + self.semaphore = threading.Semaphore( + value=Bcfg2.Options.setup.children) + if Bcfg2.Options.setup.debug: level = logging.DEBUG elif Bcfg2.Options.setup.verbose: @@ -113,25 +129,31 @@ class ReportingCollector(object): if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") + self.context.pidfile = TimeoutPIDLockFile( + Bcfg2.Options.setup.daemon, acquire_timeout=5) + # Attempt to ensure lockfile is able to be created and not stale try: - self.context.pidfile = PIDLockFile(Bcfg2.Options.setup.daemon) - self.context.open() + self.context.pidfile.acquire() except LockFailed: self.logger.error("Failed to daemonize: %s" % sys.exc_info()[1]) self.shutdown() return except LockTimeout: - self.logger.error("Failed to daemonize: " - "Failed to acquire lock on %s" % - self.setup['daemon']) - self.shutdown() - return - except PIDFileError: - self.logger.error("Error writing pid file: %s" % - sys.exc_info()[1]) - self.shutdown() - return + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except (OSError, TypeError): # No process with locked PID + self.context.pidfile.break_lock() + else: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + Bcfg2.Options.setup.daemon) + self.shutdown() + return + else: + self.context.pidfile.release() + + self.context.open() self.logger.info("Starting daemon") self.transport.start_monitor(self) @@ -141,7 +163,10 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - store_thread = ReportingStoreThread(interaction, self.storage) + if self.semaphore: + self.semaphore.acquire() + store_thread = ReportingStoreThread(interaction, self.storage, + semaphore=self.semaphore) store_thread.start() self.children.append(store_thread) diff --git a/src/lib/Bcfg2/Reporting/Reports.py b/src/lib/Bcfg2/Reporting/Reports.py index 219d74584..3b9c83433 100755 --- a/src/lib/Bcfg2/Reporting/Reports.py +++ b/src/lib/Bcfg2/Reporting/Reports.py @@ -267,10 +267,11 @@ class CLI(Bcfg2.Options.CommandRegistry): def __init__(self): Bcfg2.Options.CommandRegistry.__init__(self) - Bcfg2.Options.register_commands(self.__class__, globals().values()) + self.register_commands(globals().values()) parser = Bcfg2.Options.get_parser( description="Query the Bcfg2 reporting subsystem", components=[self]) + parser.add_options(self.subcommand_options) parser.parse() def run(self): diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 406216861..96226c424 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -168,7 +168,7 @@ class DjangoORM(StorageBase): # TODO - vcs output act_dict['detail_type'] = PathEntry.DETAIL_UNUSED if path_type == 'directory' and entry.get('prune', 'false') == 'true': - unpruned_elist = [e.get('path') for e in entry.findall('Prune')] + unpruned_elist = [e.get('name') for e in entry.findall('Prune')] if unpruned_elist: act_dict['detail_type'] = PathEntry.DETAIL_PRUNED act_dict['details'] = "\n".join(unpruned_elist) @@ -367,10 +367,11 @@ class DjangoORM(StorageBase): def import_interaction(self, interaction): """Import the data into the backend""" try: - self._import_interaction(interaction) - except: - self.logger.error("Failed to import interaction: %s" % - traceback.format_exc().splitlines()[-1]) + try: + self._import_interaction(interaction) + except: + self.logger.error("Failed to import interaction: %s" % + traceback.format_exc().splitlines()[-1]) finally: self.logger.debug("%s: Closing database connection" % self.__class__.__name__) diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 2d96990b1..ae6f6731b 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -717,9 +717,6 @@ class PathEntry(SuccessEntry): def has_detail(self): return self.detail_type != PathEntry.DETAIL_UNUSED - def is_sensitive(self): - return self.detail_type == PathEntry.DETAIL_SENSITIVE - def is_diff(self): return self.detail_type == PathEntry.DETAIL_DIFF diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index b03d48045..c6e6df020 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -107,7 +107,7 @@ div.entry_list h3 { {{ item.details|syntaxhilight }} </div> {% else %} - {{ item.details }} + {{ item.details|linebreaks }} {% endif %} </div> {% endif %} diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 4a93e77e0..09aebc7fd 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -111,47 +111,58 @@ def filter_navigator(context): try: path = context['request'].META['PATH_INFO'] view, args, kwargs = resolve(path) + except (Resolver404, KeyError): + return dict() - # Strip any page limits and numbers - if 'page_number' in kwargs: - del kwargs['page_number'] - if 'page_limit' in kwargs: - del kwargs['page_limit'] - - # get a query string - qs = context['request'].GET.urlencode() - if qs: - qs = '?' + qs - - filters = [] - for filter in filter_list: - if filter == 'group': - continue - if filter in kwargs: - myargs = kwargs.copy() - del myargs[filter] + # Strip any page limits and numbers + if 'page_number' in kwargs: + del kwargs['page_number'] + if 'page_limit' in kwargs: + del kwargs['page_limit'] + + # get a query string + qs = context['request'].GET.urlencode() + if qs: + qs = '?' + qs + + filters = [] + for filter in filter_list: + if filter == 'group': + continue + if filter in kwargs: + myargs = kwargs.copy() + del myargs[filter] + try: filters.append((filter, reverse(view, args=args, kwargs=myargs) + qs)) - filters.sort(key=lambda x: x[0]) - - myargs = kwargs.copy() - selected = True - if 'group' in myargs: - del myargs['group'] - selected = False - groups = [('---', - reverse(view, args=args, kwargs=myargs) + qs, - selected)] - for group in Group.objects.values('name'): + except NoReverseMatch: + pass + filters.sort(key=lambda x: x[0]) + + myargs = kwargs.copy() + selected = True + if 'group' in myargs: + del myargs['group'] + selected = False + + groups = [] + try: + groups.append(('---', + reverse(view, args=args, kwargs=myargs) + qs, + selected)) + except NoReverseMatch: + pass + + for group in Group.objects.values('name'): + try: myargs['group'] = group['name'] groups.append((group['name'], reverse(view, args=args, kwargs=myargs) + qs, group['name'] == kwargs.get('group', ''))) + except NoReverseMatch: + pass - return {'filters': filters, 'groups': groups} - except (Resolver404, NoReverseMatch, ValueError, KeyError): - pass - return dict() + return {'filters': filters, 'groups': groups} def _subtract_or_na(mdict, x, y): diff --git a/src/lib/Bcfg2/Reporting/utils.py b/src/lib/Bcfg2/Reporting/utils.py index 0d394fcd8..694f38824 100755 --- a/src/lib/Bcfg2/Reporting/utils.py +++ b/src/lib/Bcfg2/Reporting/utils.py @@ -96,12 +96,12 @@ def filteredUrls(pattern, view, kwargs=None, name=None): tail = mtail.group(1) pattern = pattern[:len(pattern) - len(tail)] for filter in ('/state/(?P<state>\w+)', - '/group/(?P<group>[\w\-\.]+)', - '/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)', - '/server/(?P<server>[\w\-\.]+)', - '/server/(?P<server>[\w\-\.]+)/(?P<state>[A-Za-z]+)', - '/server/(?P<server>[\w\-\.]+)/group/(?P<group>[\w\-\.]+)', - '/server/(?P<server>[\w\-\.]+)/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)'): + '/group/(?P<group>[^/]+)', + '/group/(?P<group>[^/]+)/(?P<state>[A-Za-z]+)', + '/server/(?P<server>[^/]+)', + '/server/(?P<server>[^/]+)/(?P<state>[A-Za-z]+)', + '/server/(?P<server>[^/]+)/group/(?P<group>[^/]+)', + '/server/(?P<server>[^/]+)/group/(?P<group>[^/]+)/(?P<state>[A-Za-z]+)'): results += [(pattern + filter + tail, view, kwargs)] return results diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 0807fb2b0..ef7741880 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -439,15 +439,6 @@ class Compare(AdminCmd): print("") -class Help(AdminCmd, Bcfg2.Options.HelpCommand): - """ Get help on a specific subcommand """ - def command_registry(self): - return CLI.commands - - def run(self, setup): - Bcfg2.Options.HelpCommand.run(self, setup) - - class Init(AdminCmd): """Interactively initialize a new repository.""" @@ -1194,16 +1185,20 @@ class Xcmd(_ProxyAdminCmd): class CLI(Bcfg2.Options.CommandRegistry): """ CLI class for bcfg2-admin """ + def __init__(self): Bcfg2.Options.CommandRegistry.__init__(self) - Bcfg2.Options.register_commands(self.__class__, globals().values(), - parent=AdminCmd) + self.register_commands(globals().values(), parent=AdminCmd) parser = Bcfg2.Options.get_parser( description="Manage a running Bcfg2 server", components=[self]) + parser.add_options(self.subcommand_options) parser.parse() def run(self): """ Run bcfg2-admin """ - self.commands[Bcfg2.Options.setup.subcommand].setup() - return self.runcommand() + try: + self.commands[Bcfg2.Options.setup.subcommand].setup() + return self.runcommand() + finally: + self.shutdown() diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 769addf55..e138c57e4 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -1,5 +1,6 @@ """ The core of the builtin Bcfg2 server. """ +import os import sys import time import socket @@ -85,20 +86,30 @@ class BuiltinCore(NetworkCore): def _daemonize(self): """ Open :attr:`context` to drop privileges, write the PID file, and daemonize the server core. """ + # Attempt to ensure lockfile is able to be created and not stale try: - self.context.open() - self.logger.info("%s daemonized" % self.name) - return True + self.context.pidfile.acquire() except LockFailed: err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False except LockTimeout: - err = sys.exc_info()[1] - self.logger.error("Failed to daemonize %s: Failed to acquire lock " - "on %s" % (self.name, - Bcfg2.Options.setup.daemon)) - return False + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except (OSError, TypeError): # No process with locked PID + self.context.pidfile.break_lock() + else: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: Failed to acquire" + "lock on %s" % (self.name, + Bcfg2.Options.setup.daemon)) + return False + else: + self.context.pidfile.release() + + self.context.open() + self.logger.info("%s daemonized" % self.name) + return True def _run(self): """ Create :attr:`server` to start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 892f2832a..bc305e47a 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -84,7 +84,7 @@ def close_db_connection(func): if self._database_available: # pylint: disable=W0212 from django import db self.logger.debug("%s: Closing database connection" % - threading.current_thread().name) + threading.current_thread().getName()) db.close_connection() return rv @@ -783,13 +783,13 @@ class Core(object): for plug in self.plugins_by_type(Threaded): plug.start_threads() + + self.block_for_fam_events() + self._block() except: self.shutdown() raise - self.block_for_fam_events() - self._block() - def _run(self): """ Start up the server; this method should return immediately. This must be overridden by a core diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index d0fd70c5c..8e0dd2efe 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -238,6 +238,8 @@ class FileMonitor(Debuggable): self.handles[event.requestID])) try: self.handles[event.requestID].HandleEvent(event) + except KeyboardInterrupt: + raise except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Error in handling of event %s for %s: %s" % diff --git a/src/lib/Bcfg2/Server/Info.py b/src/lib/Bcfg2/Server/Info.py index a5136f01d..6af561089 100644 --- a/src/lib/Bcfg2/Server/Info.py +++ b/src/lib/Bcfg2/Server/Info.py @@ -123,15 +123,6 @@ class InfoCmd(Bcfg2.Options.Subcommand): # pylint: disable=W0223 list(self.core.metadata.groups.keys())) -class Help(InfoCmd, Bcfg2.Options.HelpCommand): - """ Get help on a specific subcommand """ - def command_registry(self): - return self.core.commands - - def run(self, setup): - Bcfg2.Options.HelpCommand.run(self, setup) - - class Debug(InfoCmd): """ Shell out to a Python interpreter """ interpreters, default_interpreter = load_interpreters() @@ -805,15 +796,12 @@ if HAS_PROFILE: display_trace(prof) -class InfoCore(cmd.Cmd, - Bcfg2.Server.Core.Core, - Bcfg2.Options.CommandRegistry): +class InfoCore(cmd.Cmd, Bcfg2.Server.Core.Core): """Main class for bcfg2-info.""" def __init__(self): cmd.Cmd.__init__(self) Bcfg2.Server.Core.Core.__init__(self) - Bcfg2.Options.CommandRegistry.__init__(self) self.prompt = 'bcfg2-info> ' def get_locals(self): @@ -849,20 +837,20 @@ class InfoCore(cmd.Cmd, pass def shutdown(self): - Bcfg2.Options.CommandRegistry.shutdown(self) Bcfg2.Server.Core.Core.shutdown(self) -class CLI(object): +class CLI(Bcfg2.Options.CommandRegistry): """ The bcfg2-info CLI """ options = [Bcfg2.Options.BooleanOption("-p", "--profile", help="Profile")] def __init__(self): - Bcfg2.Options.register_commands(InfoCore, globals().values(), - parent=InfoCmd) + Bcfg2.Options.CommandRegistry.__init__(self) + self.register_commands(globals().values(), parent=InfoCmd) parser = Bcfg2.Options.get_parser( description="Inspect a running Bcfg2 server", components=[self, InfoCore]) + parser.add_options(self.subcommand_options) parser.parse() if Bcfg2.Options.setup.profile and HAS_PROFILE: @@ -874,11 +862,18 @@ class CLI(object): print("Profiling functionality not available.") self.core = InfoCore() - for command in self.core.commands.values(): + for command in self.commands.values(): command.core = self.core def run(self): """ Run bcfg2-info """ - if Bcfg2.Options.setup.subcommand != 'help': - self.core.run() - return self.core.runcommand() + try: + if Bcfg2.Options.setup.subcommand != 'help': + self.core.run() + return self.runcommand() + finally: + self.shutdown() + + def shutdown(self): + Bcfg2.Options.CommandRegistry.shutdown(self) + self.core.shutdown() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 3d5c68e3f..cfabd8457 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -102,7 +102,8 @@ class AptSource(Source): bdeps[barch][pkgname] = [] brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: - self.essentialpkgs.add(pkgname) + if words[1].strip() == 'yes': + self.essentialpkgs.add(pkgname) elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 24db2963d..67ada2399 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -199,6 +199,9 @@ class Source(Debuggable): # pylint: disable=R0902 #: The "version" attribute from :attr:`xsource` self.version = xsource.get('version', '') + #: The "name" attribute from :attr:`xsource` + self.name = xsource.get('name', None) + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -274,11 +277,11 @@ class Source(Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch) + arch=arch, debsrc=self.debsrc) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch)] + arch=arch, debsrc=self.debsrc)] for setting in usettings: if not self.rawurl: @@ -286,6 +289,7 @@ class Source(Debuggable): # pylint: disable=R0902 else: setting['baseurl'] = self.rawurl setting['url'] = baseurl % setting + setting['name'] = self.get_repo_name(setting) self.url_map.extend(usettings) @property @@ -353,7 +357,7 @@ class Source(Debuggable): # pylint: disable=R0902 if os.path.exists(self.cachefile): try: self.load_state() - except: + except (OSError, cPickle.UnpicklingError): err = sys.exc_info()[1] self.logger.error("Packages: Cachefile %s load failed: %s" % (self.cachefile, err)) @@ -388,8 +392,10 @@ class Source(Debuggable): # pylint: disable=R0902 doing other operations that require repository names. This function tries several approaches: - #. First, if the map contains a ``component`` key, use that as - the name. + #. First, if the source element containts a ``name`` attribute, + use that as the name. + #. If the map contains a ``component`` key, use that as the + name. #. If not, then try to match the repository URL against :attr:`Bcfg2.Server.Plugins.Packages.Source.REPO_RE`. If that succeeds, use the first matched group; additionally, @@ -419,6 +425,9 @@ class Source(Debuggable): # pylint: disable=R0902 :type url_map: dict :returns: string - the name of the repository. """ + if self.name: + return self.name + if url_map['component']: rname = url_map['component'] else: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py index 48304d26e..f26d6ba18 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py @@ -383,10 +383,10 @@ class CLI(Bcfg2.Options.CommandRegistry): def __init__(self): Bcfg2.Options.CommandRegistry.__init__(self) - Bcfg2.Options.register_commands(self.__class__, globals().values(), - parent=HelperSubcommand) + self.register_commands(globals().values(), parent=HelperSubcommand) parser = Bcfg2.Options.get_parser("Bcfg2 yum helper", components=[self]) + parser.add_options(self.subcommand_options) parser.parse() self.logger = logging.getLogger(parser.prog) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index d11ac60fe..cb533f4f1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -33,6 +33,7 @@ class PackagesBackendAction(Bcfg2.Options.ComponentAction): """ ComponentAction to load Packages backends """ bases = ['Bcfg2.Server.Plugins.Packages'] module = True + fail_silently = True class OnDemandDict(MutableMapping): diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index b752650f0..2ca518e53 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -18,12 +18,6 @@ class Svn(Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" options = Bcfg2.Server.Plugin.Version.options + [ Bcfg2.Options.Option( - cf=("svn", "conflict_resolution"), dest="svn_conflict_resolution", - type=lambda v: v.replace("-", "_"), - choices=dir(pysvn.wc_conflict_choice), # pylint: disable=E1101 - default=pysvn.wc_conflict_choice.postpone, # pylint: disable=E1101 - help="SVN conflict resolution method"), - Bcfg2.Options.Option( cf=("svn", "user"), dest="svn_user", help="SVN username"), Bcfg2.Options.Option( cf=("svn", "password"), dest="svn_password", help="SVN password"), @@ -31,6 +25,18 @@ class Svn(Bcfg2.Server.Plugin.Version): cf=("svn", "always_trust"), dest="svn_trust_ssl", help="Always trust SSL certs from SVN server")] + if HAS_SVN: + options.append( + Bcfg2.Options.Option( + cf=("svn", "conflict_resolution"), + dest="svn_conflict_resolution", + type=lambda v: v.replace("-", "_"), + # pylint: disable=E1101 + choices=dir(pysvn.wc_conflict_choice), + default=pysvn.wc_conflict_choice.postpone, + # pylint: enable=E1101 + help="SVN conflict resolution method")) + __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".svn" if HAS_SVN: diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 7150c245a..8d6642a25 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -4,39 +4,19 @@ import sys import logging import Bcfg2.Options import Bcfg2.Server.Plugins -from Bcfg2.Compat import walk_packages -LOGGER = logging.getLogger('Bcfg2.Server.models') +LOGGER = logging.getLogger(__name__) MODELS = [] -def _get_all_plugins(): - rv = [] - for submodule in walk_packages(path=Bcfg2.Server.Plugins.__path__, - prefix="Bcfg2.Server.Plugins."): - module = submodule[1].rsplit('.', 1)[-1] - if submodule[1] == "Bcfg2.Server.Plugins.%s" % module: - # we only include direct children of - # Bcfg2.Server.Plugins -- e.g., all_plugins should - # include Bcfg2.Server.Plugins.Cfg, but not - # Bcfg2.Server.Plugins.Cfg.CfgInfoXML - rv.append(module) - return rv - - -_ALL_PLUGINS = _get_all_plugins() - - class _OptionContainer(object): + """Options for Bcfg2 database models.""" + # we want to provide a different default plugin list -- # namely, _all_ plugins, so that the database is guaranteed to # work, even if /etc/bcfg2.conf isn't set up properly - options = [ - Bcfg2.Options.Option( - cf=('server', 'plugins'), type=Bcfg2.Options.Types.comma_list, - default=_ALL_PLUGINS, dest="models_plugins", - action=Bcfg2.Options.PluginsAction)] + options = [Bcfg2.Options.Common.plugins] @staticmethod def options_parsed_hook(): @@ -58,7 +38,7 @@ def load_models(plugins=None): global MODELS if not plugins: - plugins = Bcfg2.Options.setup.models_plugins + plugins = Bcfg2.Options.setup.plugins if MODELS: # load_models() has been called once, so first unload all of |