diff options
Diffstat (limited to 'src')
44 files changed, 1282 insertions, 1084 deletions
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 482320e0d..d7a70e202 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -54,6 +54,10 @@ class POSIXFile(POSIXTool): def verify(self, entry, modlist): ondisk = self._exists(entry) tempdata, is_binary = self._get_data(entry) + if isinstance(tempdata, str) and str != unicode: + tempdatasize = len(tempdata) + else: + tempdatasize = len(tempdata.encode(Bcfg2.Options.setup.encoding)) different = False content = None @@ -62,7 +66,7 @@ class POSIXFile(POSIXTool): # they're clearly different different = True content = "" - elif len(tempdata) != ondisk[stat.ST_SIZE]: + elif tempdatasize != ondisk[stat.ST_SIZE]: # next, see if the size of the target file is different # from the size of the desired content different = True @@ -73,6 +77,9 @@ class POSIXFile(POSIXTool): # for everything else try: content = open(entry.get('name')).read() + except UnicodeDecodeError: + content = open(entry.get('name'), + encoding=Bcfg2.Options.setup.encoding).read() except IOError: self.logger.error("POSIX: Failed to read %s: %s" % (entry.get("name"), sys.exc_info()[1])) @@ -90,7 +97,7 @@ class POSIXFile(POSIXTool): def _write_tmpfile(self, entry): """ Write the file data to a temp file """ - filedata, _ = self._get_data(entry) + filedata = self._get_data(entry)[0] # get a temp file to write to that is in the same directory as # the existing file in order to preserve any permissions # protections on that directory, and also to avoid issues with @@ -106,7 +113,11 @@ class POSIXFile(POSIXTool): (os.path.dirname(entry.get('name')), err)) return False try: - os.fdopen(newfd, 'w').write(filedata) + if isinstance(filedata, str) and str != unicode: + os.fdopen(newfd, 'w').write(filedata) + else: + os.fdopen(newfd, 'wb').write( + filedata.encode(Bcfg2.Options.setup.encoding)) except (OSError, IOError): err = sys.exc_info()[1] self.logger.error("POSIX: Failed to open temp file %s for writing " diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index c9164cb88..bd2f8f87e 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -525,7 +525,8 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if entry.get("secontext") == "__default__": try: wanted_secontext = \ - selinux.matchpathcon(path, 0)[1].split(":")[2] + selinux.matchpathcon( + path, ondisk[stat.ST_MODE])[1].split(":")[2] except OSError: errors.append("%s has no default SELinux context" % entry.get("name")) diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index aab2459f2..4a808aa60 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -590,14 +590,15 @@ class SvcTool(Tool): if not self.handlesEntry(entry): continue + estatus = entry.get('status') restart = entry.get("restart", "true").lower() - if (restart == "false" or + if (restart == "false" or estatus == 'ignore' or (restart == "interactive" and not Bcfg2.Options.setup.interactive)): continue success = False - if entry.get('status') == 'on': + if estatus == 'on': if Bcfg2.Options.setup.service_mode == 'build': success = self.stop_service(entry) elif entry.get('name') not in self.restarted: diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 3cba93fff..433fb570a 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -12,9 +12,9 @@ import argparse import tempfile import Bcfg2.Logger import Bcfg2.Options -import XML # pylint: disable=W0403 -import Proxy # pylint: disable=W0403 -import Tools # pylint: disable=W0403 +from Bcfg2.Client import XML +from Bcfg2.Client import Proxy +from Bcfg2.Client import Tools from Bcfg2.Utils import locked, Executor, safe_input from Bcfg2.version import __version__ # pylint: disable=W0622 @@ -88,7 +88,6 @@ class Client(object): options = Proxy.ComponentProxy.options + [ Bcfg2.Options.Common.syslog, - Bcfg2.Options.Common.location, Bcfg2.Options.Common.interactive, Bcfg2.Options.BooleanOption( "-q", "--quick", help="Disable some checksum verification"), @@ -194,7 +193,11 @@ class Client(object): ret = XML.Element("probe-data", name=name, source=probe.get('source')) try: scripthandle, scriptname = tempfile.mkstemp() - script = os.fdopen(scripthandle, 'w') + if sys.hexversion >= 0x03000000: + script = os.fdopen(scripthandle, 'w', + encoding=Bcfg2.Options.setup.encoding) + else: + script = os.fdopen(scripthandle, 'w') try: script.write("#!%s\n" % (probe.attrib.get('interpreter', '/bin/sh'))) @@ -666,13 +669,15 @@ class Client(object): # first process prereq actions for bundle in bundles[:]: if bundle.tag == 'Bundle': - bmodified = any(item in self.whitelist for item in bundle) + bmodified = any((item in self.whitelist or + item in self.modified) for item in bundle) else: bmodified = False actions = [a for a in bundle.findall('./Action') if (a.get('timing') in ['pre', 'both'] and (bmodified or a.get('when') == 'always'))] - # now we process all "always actions" + # now we process all "pre" and "both" actions that are either + # always or the bundle has been modified if Bcfg2.Options.setup.interactive: self.promptFilter(iprompt, actions) self.DispatchInstallCalls(actions) diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index f9fd42d33..0f7995e0f 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -236,7 +236,6 @@ def setup_logging(): logging.root.setLevel(logging.DEBUG) logging.root.debug("Configured logging: %s" % "; ".join(params)) - print("Configured logging: %s" % "; ".join(params)) logging.already_setup = True diff --git a/src/lib/Bcfg2/Options/Actions.py b/src/lib/Bcfg2/Options/Actions.py index 637a09577..8b97f1da8 100644 --- a/src/lib/Bcfg2/Options/Actions.py +++ b/src/lib/Bcfg2/Options/Actions.py @@ -2,7 +2,7 @@ import sys import argparse -from Parser import get_parser # pylint: disable=W0403 +from Bcfg2.Options.Parser import get_parser __all__ = ["ConfigFileAction", "ComponentAction", "PluginsAction"] diff --git a/src/lib/Bcfg2/Options/Common.py b/src/lib/Bcfg2/Options/Common.py index eb4af5bb6..9ba08eb87 100644 --- a/src/lib/Bcfg2/Options/Common.py +++ b/src/lib/Bcfg2/Options/Common.py @@ -1,12 +1,10 @@ """ Common options used in multiple different contexts. """ from Bcfg2.Utils import classproperty -# pylint: disable=W0403 -import Types -from Actions import PluginsAction, ComponentAction -from Parser import repository as _repository_option -from Options import Option, PathOption, BooleanOption -# pylint: enable=W0403 +from Bcfg2.Options import Types +from Bcfg2.Options.Actions import PluginsAction, ComponentAction +from Bcfg2.Options.Parser import repository as _repository_option +from Bcfg2.Options import Option, PathOption, BooleanOption __all__ = ["Common"] diff --git a/src/lib/Bcfg2/Options/OptionGroups.py b/src/lib/Bcfg2/Options/OptionGroups.py index 70cb5d0dd..465358fab 100644 --- a/src/lib/Bcfg2/Options/OptionGroups.py +++ b/src/lib/Bcfg2/Options/OptionGroups.py @@ -3,7 +3,7 @@ import re import copy import fnmatch -from Options import Option # pylint: disable=W0403 +from Bcfg2.Options import Option from itertools import chain __all__ = ["OptionGroup", "ExclusiveOptionGroup", "Subparser", diff --git a/src/lib/Bcfg2/Options/Options.py b/src/lib/Bcfg2/Options/Options.py index d60c536cf..be7e7c646 100644 --- a/src/lib/Bcfg2/Options/Options.py +++ b/src/lib/Bcfg2/Options/Options.py @@ -4,9 +4,9 @@ need to be associated with an option parser; it exists on its own.""" import os import copy -import Types # pylint: disable=W0403 import fnmatch import argparse +from Bcfg2.Options import Types from Bcfg2.Compat import ConfigParser diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py index dd7874d35..bede85a1f 100644 --- a/src/lib/Bcfg2/Options/Parser.py +++ b/src/lib/Bcfg2/Options/Parser.py @@ -5,7 +5,7 @@ import sys import argparse from Bcfg2.version import __version__ from Bcfg2.Compat import ConfigParser -from Options import Option, PathOption, BooleanOption # pylint: disable=W0403 +from Bcfg2.Options import Option, PathOption, BooleanOption __all__ = ["setup", "OptionParserException", "Parser", "get_parser"] @@ -201,8 +201,7 @@ class Parser(argparse.ArgumentParser): # check whether the specified bcfg2.conf exists if not os.path.exists(bootstrap.config): - print("Could not read %s" % bootstrap.config) - return 1 + self.error("Could not read %s" % bootstrap.config) self.add_config_file(self.configfile.dest, bootstrap.config) # phase 2: re-parse command line, loading additional @@ -212,7 +211,7 @@ class Parser(argparse.ArgumentParser): while not self.parsed: self.parsed = True self._set_defaults() - self.parse_known_args(namespace=self.namespace) + self.parse_known_args(args=self.argv, namespace=self.namespace) self._parse_config_options() self._finalize() self._parse_config_options() diff --git a/src/lib/Bcfg2/Options/Subcommands.py b/src/lib/Bcfg2/Options/Subcommands.py index 7d7a3f928..660bd5077 100644 --- a/src/lib/Bcfg2/Options/Subcommands.py +++ b/src/lib/Bcfg2/Options/Subcommands.py @@ -8,11 +8,9 @@ import copy import shlex import logging from Bcfg2.Compat import StringIO -# pylint: disable=W0403 -from OptionGroups import Subparser -from Options import PositionalArgument -from Parser import Parser, setup as master_setup -# pylint: enable=W0403 +from Bcfg2.Options import PositionalArgument +from Bcfg2.Options.OptionGroups import Subparser +from Bcfg2.Options.Parser import Parser, setup as master_setup __all__ = ["Subcommand", "HelpCommand", "CommandRegistry", "register_commands"] diff --git a/src/lib/Bcfg2/Options/__init__.py b/src/lib/Bcfg2/Options/__init__.py index 546068f1f..96465ec56 100644 --- a/src/lib/Bcfg2/Options/__init__.py +++ b/src/lib/Bcfg2/Options/__init__.py @@ -1,10 +1,10 @@ """ Bcfg2 options parsing. """ -# pylint: disable=W0611,W0401,W0403 -import Types -from Common import * -from Parser import * -from Actions import * -from Options import * -from Subcommands import * -from OptionGroups import * +# pylint: disable=W0611,W0401 +from Bcfg2.Options import Types +from Bcfg2.Options.Options import * +from Bcfg2.Options.Common import * +from Bcfg2.Options.Parser import * +from Bcfg2.Options.Actions import * +from Bcfg2.Options.Subcommands import * +from Bcfg2.Options.OptionGroups import * diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index a1e6025e3..a93f1b0ae 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -44,15 +44,11 @@ class ReportingCollector(object): else: level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-report-collector', - to_console=logging.INFO, - to_syslog=Bcfg2.Options.setup.syslog, - to_file=Bcfg2.Options.setup.logging, - level=level) + Bcfg2.Logger.setup_logging() self.logger = logging.getLogger('bcfg2-report-collector') try: - self.transport = Bcfg2.Options.setup.transport() + self.transport = Bcfg2.Options.setup.reporting_transport() self.storage = Bcfg2.Options.setup.reporting_storage() except TransportError: self.logger.error("Failed to load transport: %s" % @@ -82,7 +78,7 @@ class ReportingCollector(object): """Startup the processing and go!""" self.terminate = threading.Event() atexit.register(self.shutdown) - self.context = daemon.DaemonContext() + self.context = daemon.DaemonContext(detach_process=True) if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 9505682a7..69da9c571 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -378,7 +378,7 @@ class DjangoORM(StorageBase): def validate(self): """Validate backend storage. Should be called once when loaded""" - settings.read_config(repo=Bcfg2.Options.setup.repository) + settings.read_config() # verify our database schema try: diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py index d901ded56..189967cb0 100644 --- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -10,7 +10,6 @@ import select import time import traceback import Bcfg2.Options -import Bcfg2.CommonOptions import Bcfg2.Server.FileMonitor from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError from Bcfg2.Reporting.Transport.base import TransportBase, TransportError diff --git a/src/lib/Bcfg2/Reporting/Transport/base.py b/src/lib/Bcfg2/Reporting/Transport/base.py index 9fbf8c9d5..9a0a4262f 100644 --- a/src/lib/Bcfg2/Reporting/Transport/base.py +++ b/src/lib/Bcfg2/Reporting/Transport/base.py @@ -4,6 +4,7 @@ The base for all server -> collector Transports import os import sys +import Bcfg2.Options from Bcfg2.Logger import Debuggable diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 24cb46bd9..59059c240 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -59,7 +59,7 @@ class ccolors: # pylint: disable=C0103,W0232 def gen_password(length): """Generates a random alphanumeric password with length characters.""" - chars = string.letters + string.digits + chars = string.ascii_letters + string.digits return "".join(random.choice(chars) for i in range(length)) @@ -1091,7 +1091,7 @@ class Viz(_ServerAdminCmd): help="Show a key for different digraph shapes"), Bcfg2.Options.Option( "-c", "--only-client", metavar="<hostname>", - help="Show only the groups, bundles for the named client"), + help="Only show groups and bundles for the named client"), Bcfg2.Options.PathOption( "-o", "--outfile", help="Write viz output to an output file")] diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 179a6aa9f..0023e9313 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -113,7 +113,6 @@ class BuiltinCore(NetworkCore): keyfile=Bcfg2.Options.setup.key, certfile=Bcfg2.Options.setup.cert, register=False, - timeout=1, ca=Bcfg2.Options.setup.ca) except: # pylint: disable=W0702 err = sys.exc_info()[1] diff --git a/src/lib/Bcfg2/Server/Cache.py b/src/lib/Bcfg2/Server/Cache.py index 842098eda..d05eb0bf6 100644 --- a/src/lib/Bcfg2/Server/Cache.py +++ b/src/lib/Bcfg2/Server/Cache.py @@ -1,14 +1,180 @@ -""" An implementation of a simple memory-backed cache. Right now this -doesn't provide many features, but more (time-based expiration, etc.) -can be added as necessary. """ +""" ``Bcfg2.Server.Cache`` is an implementation of a simple +memory-backed cache. Right now this doesn't provide many features, but +more (time-based expiration, etc.) can be added as necessary. +The normal workflow is to get a Cache object, which is simply a dict +interface to the unified cache that automatically uses a certain tag +set. For instance: -class Cache(dict): - """ an implementation of a simple memory-backed cache """ +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + groupcache['foo.example.com'] = ['group1', 'group2'] + +This would create a Cache object that automatically tags its entries +with ``frozenset(["Probes", "probegroups"])``, and store the list +``['group1', 'group1']`` with the *additional* tag +``foo.example.com``. So the unified backend cache would then contain +a single entry: + +.. code-block:: python + + {frozenset(["Probes", "probegroups", "foo.example.com"]): + ['group1', 'group2']} + +In addition to the dict interface, Cache objects (returned from +:func:`Bcfg2.Server.Cache.Cache`) have one additional method, +``expire()``, which is mostly identical to +:func:`Bcfg2.Server.Cache.expire`, except that it is specific to the +tag set of the cache object. E.g., to expire all ``foo.example.com`` +records for a given cache, you could do: + +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + groupcache.expire("foo.example.com") + +This is mostly functionally identical to: + +.. code-block:: python + + Bcfg2.Server.Cache.expire("Probes", "probegroups", "foo.example.com") + +It's not completely identical, though; the first example will expire, +at most, exactly one item from the cache. The second example will +expire all items that are tagged with a superset of the given tags. +To illustrate the difference, consider the following two examples: + +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes") + groupcache.expire("probegroups") + + Bcfg2.Server.Cache.expire("Probes", "probegroups") + +The former will not expire any data, because there is no single datum +tagged with ``"Probes", "probegroups"``. The latter will expire *all* +items tagged with ``"Probes", "probegroups"`` -- i.e., the entire +cache. In this case, the latter call is equivalent to: + +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + groupcache.expire() + +""" + +from Bcfg2.Compat import MutableMapping + + +class _Cache(MutableMapping): + """ The object returned by :func:`Bcfg2.Server.Cache.Cache` that + presents a dict-like interface to the portion of the unified cache + that uses the specified tags. """ + def __init__(self, registry, tags): + self._registry = registry + self._tags = tags + + def __getitem__(self, key): + return self._registry[self._tags | set([key])] + + def __setitem__(self, key, value): + self._registry[self._tags | set([key])] = value + + def __delitem__(self, key): + del self._registry[self._tags | set([key])] + + def __iter__(self): + for item in self._registry.iterate(*self._tags): + yield list(item.difference(self._tags))[0] + + def keys(self): + """ List cache keys """ + return list(iter(self)) + + def __len__(self): + return len(list(iter(self))) def expire(self, key=None): """ expire all items, or a specific item, from the cache """ if key is None: - self.clear() - elif key in self: - del self[key] + expire(*self._tags) + else: + tags = self._tags | set([key]) + # py 2.5 doesn't support mixing *args and explicit keyword + # args + kwargs = dict(exact=True) + expire(*tags, **kwargs) + + def __repr__(self): + return repr(dict(self)) + + def __str__(self): + return str(dict(self)) + + +class _CacheRegistry(dict): + """ The grand unified cache backend which contains all cache + items. """ + + def iterate(self, *tags): + """ Iterate over all items that match the given tags *and* + have exactly one additional tag. This is used to get items + for :class:`Bcfg2.Server.Cache._Cache` objects that have been + instantiated via :func:`Bcfg2.Server.Cache.Cache`. """ + tags = frozenset(tags) + for key in self.keys(): + if key.issuperset(tags) and len(key.difference(tags)) == 1: + yield key + + def iter_all(self, *tags): + """ Iterate over all items that match the given tags, + regardless of how many additional tags they have (or don't + have). This is used to expire all cache data that matches a + set of tags. """ + tags = frozenset(tags) + for key in list(self.keys()): + if key.issuperset(tags): + yield key + + +_cache = _CacheRegistry() # pylint: disable=C0103 +_hooks = [] # pylint: disable=C0103 + + +def Cache(*tags): # pylint: disable=C0103 + """ A dict interface to the cache data tagged with the given + tags. """ + return _Cache(_cache, frozenset(tags)) + + +def expire(*tags, **kwargs): + """ Expire all items, a set of items, or one specific item from + the cache. If ``exact`` is set to True, then if the given tag set + doesn't match exactly one item in the cache, nothing will be + expired. """ + exact = kwargs.pop("exact", False) + count = 0 + if not tags: + count = len(_cache) + _cache.clear() + elif exact: + if frozenset(tags) in _cache: + count = 1 + del _cache[frozenset(tags)] + else: + for match in _cache.iter_all(*tags): + count += 1 + del _cache[match] + + for hook in _hooks: + hook(tags, exact, count) + + +def add_expire_hook(func): + """ Add a hook that will be called when an item is expired from + the cache. The callable passed in must take three options: the + first will be the tag set that was expired; the second will be the + state of the ``exact`` flag (True or False); and the third will be + the number of items that were expired from the cache. """ + _hooks.append(func) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 501a78bc0..69d61580f 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -23,6 +23,7 @@ from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin import track_statistics +from Bcfg2.Server.Plugins.Metadata import MetadataGroup try: import psyco @@ -78,9 +79,23 @@ class NoExposedMethod (Exception): method exposed with the given name. """ -# pylint: disable=W0702 +class DefaultACL(Plugin, ClientACLs): + """ Default ACL 'plugin' that provides security by default. This + is only loaded if no other ClientACLs plugin is enabled. """ + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.ClientACLs.__init__(self) + + def check_acl_ip(self, address, rmi): + return (("." not in rmi and + not rmi.endswith("_debug") and + rmi != 'get_statistics') or + address[0] == "127.0.0.1") + + # in core we frequently want to catch all exceptions, regardless of # type, so disable the pylint rule that catches that. +# pylint: disable=W0702 class Core(object): """ The server core is the container for all Bcfg2 server logic @@ -186,6 +201,10 @@ class Core(object): # load plugins Bcfg2.settings.read_config() + # mapping of group name => plugin name to record where groups + # that are created by Connector plugins came from + self._dynamic_groups = dict() + #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ @@ -207,7 +226,7 @@ class Core(object): #: A :class:`Bcfg2.Server.Cache.Cache` object for caching client #: metadata - self.metadata_cache = Cache() + self.metadata_cache = Cache("Metadata") #: Whether or not it's possible to use the Django database #: backend for plugins that have that capability @@ -227,20 +246,6 @@ class Core(object): self.logger.error("Updating database %s failed: %s" % (Bcfg2.Options.setup.db_name, err)) - def expire_caches_by_type(self, base_cls, key=None): - """ Expire caches for all - :class:`Bcfg2.Server.Plugin.interfaces.Caching` plugins that - are instances of ``base_cls``. - - :param base_cls: The base plugin interface class to match (see - :mod:`Bcfg2.Server.Plugin.interfaces`) - :type base_cls: type - :param key: The cache key to expire - """ - for plugin in self.plugins_by_type(base_cls): - if isinstance(plugin, Bcfg2.Server.Plugin.Caching): - plugin.expire_cache(key) - def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -296,7 +301,7 @@ class Core(object): continue self.logger.info("File monitor thread terminated") - @Bcfg2.Server.Statistics.track_statistics() + @track_statistics() def _update_vcs_revision(self): """ Update the revision of the current configuration on-disk from the VCS plugin """ @@ -358,14 +363,16 @@ class Core(object): "failed to instantiate Core") raise CoreInitError("No Metadata Plugin") + # ensure that an ACL plugin is loaded + if not self.plugins_by_type(Bcfg2.Server.Plugin.ClientACLs): + self.init_plugin(DefaultACL) + def init_plugin(self, plugin): """ Import and instantiate a single plugin. The plugin is stored to :attr:`plugins`. - :param plugin: The name of the plugin. This is just the name - of the plugin, in the appropriate case. I.e., - ``Cfg``, not ``Bcfg2.Server.Plugins.Cfg``. - :type plugin: string + :param plugin: The plugin class to load. + :type plugin: type :returns: None """ self.logger.debug("Loading plugin %s" % plugin.name) @@ -683,7 +690,7 @@ class Core(object): if event.code2str() == 'deleted': return Bcfg2.Options.get_parser().reparse() - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + self.metadata_cache.expire() def block_for_fam_events(self, handle_events=False): """ Block until all fam events have been handleed, optionally @@ -848,8 +855,35 @@ class Core(object): (client_name, sys.exc_info()[1])) connectors = self.plugins_by_type(Connector) for conn in connectors: - grps = conn.get_additional_groups(imd) - self.metadata.merge_additional_groups(imd, grps) + groups = conn.get_additional_groups(imd) + groupnames = [] + for group in groups: + if isinstance(group, MetadataGroup): + groupname = group.name + if groupname in self._dynamic_groups: + if self._dynamic_groups[groupname] == conn.name: + self.metadata.groups[groupname] = group + else: + self.logger.warning( + "Refusing to clobber dynamic group %s " + "defined by %s" % + (self._dynamic_groups[groupname], + groupname)) + elif groupname in self.metadata.groups: + # not recorded as a dynamic group, but + # present in metadata.groups -- i.e., a + # static group + self.logger.warning( + "Refusing to clobber predefined group %s" % + groupname) + else: + self.metadata.groups[groupname] = group + self._dynamic_groups[groupname] = conn.name + groupnames.append(groupname) + else: + groupnames.append(group) + + self.metadata.merge_additional_groups(imd, groupnames) for conn in connectors: data = conn.get_additional_data(imd) self.metadata.merge_additional_data(imd, conn.name, data) @@ -1070,7 +1104,7 @@ class Core(object): # that's created for RecvProbeData doesn't get cached. # I.e., the next metadata object that's built, after probe # data is processed, is cached. - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + self.metadata_cache.expire(client) try: xpdata = lxml.etree.XML(probedata.encode('utf-8'), parser=Bcfg2.Server.XMLParser) diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py index e64a6627f..dd3f46b07 100755 --- a/src/lib/Bcfg2/Server/Encryption.py +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -215,7 +215,7 @@ class CryptoTool(object): """ get the passphrase for the current file """ if not Bcfg2.Options.setup.passphrases: raise PassphraseError("No passphrases available in %s" % - Bcfg2.Options.setup.configfile) + Bcfg2.Options.setup.config) pname = None if Bcfg2.Options.setup.passphrase: @@ -229,7 +229,7 @@ class CryptoTool(object): return (pname, passphrase) except KeyError: raise PassphraseError("Could not find passphrase %s in %s" % - (pname, Bcfg2.Options.setup.configfile)) + (pname, Bcfg2.Options.setup.config)) else: if len(Bcfg2.Options.setup.passphrases) == 1: pname, passphrase = Bcfg2.Options.setup.passphrases.items()[0] @@ -285,7 +285,7 @@ class CfgEncryptor(Encryptor): if self.passphrase is None: raise PassphraseError("Multiple passphrases found in %s, " "specify one on the command line with -p" % - Bcfg2.Options.setup.configfile) + Bcfg2.Options.setup.config) def encrypt(self): return ssl_encrypt(self.data, self.passphrase) @@ -367,19 +367,19 @@ class PropertiesCryptoMixin(object): """ Get the passphrase to use to encrypt or decrypt a given element """ pname = element.get("encrypted") - if pname in self.passphrases: - passphrase = self.passphrases[pname] + if pname in Bcfg2.Options.setup.passphrases: + passphrase = Bcfg2.Options.setup.passphrases[pname] elif self.passphrase: if pname: self.logger.warning("Passphrase %s not found in %s, " "using passphrase given on command line" % - (pname, Bcfg2.Options.setup.configfile)) + (pname, Bcfg2.Options.setup.config)) passphrase = self.passphrase pname = self.pname else: raise PassphraseError("Multiple passphrases found in %s, " "specify one on the command line with -p" % - Bcfg2.Options.setup.configfile) + Bcfg2.Options.setup.config) return (pname, passphrase) def _write(self, filename, data): @@ -579,7 +579,7 @@ class CLI(object): if data is None: data = getattr(tool, mode)() - if not data: + if data is None: self.logger.error("Failed to %s %s, skipping" % (mode, fname)) continue if Bcfg2.Options.setup.stdout: diff --git a/src/lib/Bcfg2/Server/Info.py b/src/lib/Bcfg2/Server/Info.py index 64621aa53..3a1ed7433 100644 --- a/src/lib/Bcfg2/Server/Info.py +++ b/src/lib/Bcfg2/Server/Info.py @@ -92,7 +92,7 @@ class InfoCmd(Bcfg2.Options.Subcommand): # pylint: disable=W0223 """ Given a list of globs, select the items from candidates that match the globs """ # special cases to speed things up: - if globs is None or '*' in globs: + if not globs or '*' in globs: return candidates has_wildcards = False for glob in globs: diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 2f245561b..de7ae038a 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -39,8 +39,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Cfg/**/pubkey.xml": "pubkey.xsd", "Cfg/**/authorizedkeys.xml": "authorizedkeys.xsd", "Cfg/**/authorized_keys.xml": "authorizedkeys.xsd", + "Cfg/**/sslcert.xml": "sslca-cert.xsd", + "Cfg/**/sslkey.xml": "sslca-key.xsd", "SSHbase/**/info.xml": "info.xsd", - "SSLCA/**/info.xml": "info.xsd", "TGenshi/**/info.xml": "info.xsd", "TCheetah/**/info.xml": "info.xsd", "Bundler/*.xml": "bundle.xsd", @@ -55,8 +56,6 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "GroupPatterns/config.xml": "grouppatterns.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", - "SSLCA/**/cert.xml": "sslca-cert.xsd", - "SSLCA/**/key.xml": "sslca-key.xsd", "GroupLogic/groups.xml": "grouplogic.xsd" } diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 517140178..c42009bdd 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -16,32 +16,15 @@ import threading import lxml.etree import multiprocessing import Bcfg2.Options +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin from itertools import cycle -from Bcfg2.Server.Cache import Cache from Bcfg2.Compat import Queue, Empty, wraps from Bcfg2.Server.Core import Core, exposed from Bcfg2.Server.BuiltinCore import BuiltinCore from multiprocessing.connection import Listener, Client -class DispatchingCache(Cache, Bcfg2.Server.Plugin.Debuggable): - """ Implementation of :class:`Bcfg2.Cache.Cache` that propagates - cache expiration events to child nodes. """ - - #: The method to send over the pipe to expire the cache - method = "expire_metadata_cache" - - def __init__(self, *args, **kwargs): - self.rpc_q = kwargs.pop("queue") - Bcfg2.Server.Plugin.Debuggable.__init__(self) - Cache.__init__(self, *args, **kwargs) - - def expire(self, key=None): - self.rpc_q.publish(self.method, args=[key]) - Cache.expire(self, key=key) - - class RPCQueue(Bcfg2.Server.Plugin.Debuggable): """ An implementation of a :class:`multiprocessing.Queue` designed for several additional use patterns: @@ -304,16 +287,9 @@ class ChildCore(Core): return rmi @exposed - def expire_metadata_cache(self, client=None): - """ Expire the metadata cache for a client """ - self.metadata_cache.expire(client) - - @exposed - def RecvProbeData(self, address, _): - """ Expire the probe cache for a client """ - self.expire_caches_by_type(Bcfg2.Server.Plugin.Probing, - key=self.resolve_client(address, - metadata=False)[0]) + def expire_cache(self, *tags, **kwargs): + """ Expire cached data """ + Bcfg2.Server.Cache.expire(*tags, exact=kwargs.pop("exact", False)) @exposed def GetConfig(self, client): @@ -368,8 +344,6 @@ class MultiprocessingCore(BuiltinCore): #: used to send or publish commands to children. self.rpc_q = RPCQueue() - self.metadata_cache = DispatchingCache(queue=self.rpc_q) - #: A list of children that will be cycled through self._all_children = [] @@ -392,6 +366,7 @@ class MultiprocessingCore(BuiltinCore): self.logger.debug("Started %s children: %s" % (len(self._all_children), self._all_children)) self.children = cycle(self._all_children) + Bcfg2.Server.Cache.add_expire_hook(self.cache_dispatch) return BuiltinCore._run(self) def shutdown(self): @@ -464,16 +439,11 @@ class MultiprocessingCore(BuiltinCore): def set_debug(self, address, debug): self.rpc_q.set_debug(debug) self.rpc_q.publish("set_debug", args=[address, debug]) - self.metadata_cache.set_debug(debug) return BuiltinCore.set_debug(self, address, debug) - @exposed - def RecvProbeData(self, address, probedata): - rv = BuiltinCore.RecvProbeData(self, address, probedata) - # we don't want the children to actually process probe data, - # so we don't send the data, just the fact that we got some. - self.rpc_q.publish("RecvProbeData", args=[address, None]) - return rv + def cache_dispatch(self, tags, exact, _): + """ Publish cache expiration events to child nodes. """ + self.rpc_q.publish("expire_cache", args=tags, kwargs=dict(exact=exact)) @exposed def GetConfig(self, address): diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 2d157eba9..7a3d887fe 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -597,17 +597,14 @@ class XMLFileBacked(FileBacked): Index.__doc__ = FileBacked.Index.__doc__ def add_monitor(self, fpath): - """ Add a FAM monitor to a file that has been XIncluded. This - is only done if the constructor got both a ``fam`` object and - ``should_monitor`` set to True. + """ Add a FAM monitor to a file that has been XIncluded. :param fpath: The full path to the file to monitor :type fpath: string :returns: None """ self.extra_monitors.append(fpath) - if self.should_monitor: - self.fam.AddMonitor(fpath, self) + self.fam.AddMonitor(fpath, self) def __iter__(self): return iter(self.entries) @@ -631,6 +628,9 @@ class StructFile(XMLFileBacked): #: the file being cached __identifier__ = None + #: Whether or not to enable encryption + encryption = True + #: Callbacks used to determine if children of items with the given #: tags should be included in the return value of #: :func:`Bcfg2.Server.Plugin.helpers.StructFile.Match` and @@ -674,7 +674,7 @@ class StructFile(XMLFileBacked): self.logger.error('Genshi parse error in %s: %s' % (self.name, err)) - if HAS_CRYPTO: + if HAS_CRYPTO and self.encryption: lax_decrypt = self.xdata.get( "lax_decryption", str(Bcfg2.Options.setup.lax_decryption)).lower() == "true" @@ -925,6 +925,7 @@ class PriorityStructFile(StructFile): __init__.__doc__ = StructFile.__init__.__doc__ def Index(self): + StructFile.Index(self) try: self.priority = int(self.xdata.get('priority')) except (ValueError, TypeError): @@ -955,13 +956,13 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): def HandleEvent(self, event): XMLDirectoryBacked.HandleEvent(self, event) self.Entries = {} - for src in list(self.entries.values()): - for itype, children in list(src.items.items()): - for child in children: - try: - self.Entries[itype][child] = self.BindEntry - except KeyError: - self.Entries[itype] = {child: self.BindEntry} + for src in self.entries.values(): + for child in src.xdata.iterchildren(): + if child.tag in ['Group', 'Client']: + continue + if child.tag not in self.Entries: + self.Entries[child.tag] = dict() + self.Entries[child.tag][child.get("name")] = self.BindEntry HandleEvent.__doc__ = XMLDirectoryBacked.HandleEvent.__doc__ def _matches(self, entry, metadata, candidate): # pylint: disable=W0613 diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 30275f6ad..522c6a220 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -221,10 +221,32 @@ class Connector(object): def get_additional_groups(self, metadata): # pylint: disable=W0613 """ Return a list of additional groups for the given client. + Each group can be either the name of a group (a string), or a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + that defines other data besides just the name. Note that you + cannot return a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + that clobbers a group defined by another plugin; the original + group will be used instead. For instance, assume the + following in ``Metadata/groups.xml``: + + .. code-block:: xml + + <Groups> + ... + <Group name="foo" public="false"/> + </Groups> + + You could not subsequently return a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + with ``public=True``; a warning would be issued, and the + original (non-public) ``foo`` group would be used. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :return: list of strings + :return: list of strings or + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` + objects. """ return list() @@ -632,22 +654,3 @@ class ClientACLs(object): :returns: bool """ return True - - -class Caching(object): - """ A plugin that caches more than just the data received from the - FAM. This presents a unified interface to clear the cache. """ - - def expire_cache(self, key=None): - """ Expire the cache associated with the given key. - - :param key: The key to expire the cache for. Because cache - implementations vary tremendously between plugins, - this could be any number of things, but generally - a hostname. It also may or may not be possible to - expire the cache for a single host; this interface - does not require any guarantee about that. - :type key: varies - :returns: None - """ - raise NotImplementedError diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index f91bac634..b3824fb57 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -52,15 +52,12 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data) #: Bundles by bundle name, rather than filename self.bundles = dict() - __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def HandleEvent(self, event): Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) self.bundles = dict([(b.bundle_name, b) for b in self.entries.values()]) - HandleEvent.__doc__ = \ - Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent.__doc__ def BuildStructures(self, metadata): bundleset = [] @@ -121,5 +118,3 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, data.remove(child) bundleset.append(data) return bundleset - BuildStructures.__doc__ = \ - Bcfg2.Server.Plugin.Structure.BuildStructures.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py index c08d3ec44..895752c9c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py @@ -5,7 +5,7 @@ access. """ import lxml.etree import Bcfg2.Options from Bcfg2.Server.Plugin import StructFile, PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator, CFG +from Bcfg2.Server.Plugins.Cfg import CfgGenerator, get_cfg from Bcfg2.Server.Plugins.Metadata import ClientMetadata @@ -25,7 +25,7 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): CfgGenerator.__init__(self, fname, None) StructFile.__init__(self, fname) self.cache = dict() - self.core = CFG.core + self.core = get_cfg().core __init__.__doc__ = CfgGenerator.__init__.__doc__ def handle_event(self, event): @@ -38,10 +38,13 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): spec = self.XMLMatch(metadata) rv = [] for allow in spec.findall("Allow"): - params = '' - if allow.find("Params") is not None: - params = ",".join("=".join(p) - for p in allow.find("Params").attrib.items()) + options = [] + for opt in allow.findall("Option"): + if opt.get("value"): + options.append("%s=%s" % (opt.get("name"), + opt.get("value"))) + else: + options.append(opt.get("name")) pubkey_name = allow.get("from") if pubkey_name: @@ -85,6 +88,6 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): (metadata.hostname, lxml.etree.tostring(allow))) continue - rv.append(" ".join([params, pubkey]).strip()) + rv.append(" ".join([",".join(options), pubkey]).strip()) return "\n".join(rv) get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index 7bb5d3cf5..e5611d50b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -5,17 +5,11 @@ import shutil import tempfile import Bcfg2.Options from Bcfg2.Utils import Executor -from Bcfg2.Server.Plugin import StructFile -from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError +from Bcfg2.Server.Plugins.Cfg import XMLCfgCreator, CfgCreationError from Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator import CfgPublicKeyCreator -try: - import Bcfg2.Server.Encryption - HAS_CRYPTO = True -except ImportError: - HAS_CRYPTO = False -class CfgPrivateKeyCreator(CfgCreator, StructFile): +class CfgPrivateKeyCreator(XMLCfgCreator): """The CfgPrivateKeyCreator creates SSH keys on the fly. """ #: Different configurations for different clients/groups can be @@ -25,6 +19,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): #: Handle XML specifications of private keys __basenames__ = ['privkey.xml'] + cfg_section = "sshkeys" options = [ Bcfg2.Options.Option( cf=("sshkeys", "category"), dest="sshkeys_category", @@ -34,27 +29,12 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): help="Passphrase used to encrypt generated SSH private keys")] def __init__(self, fname): - CfgCreator.__init__(self, fname) - StructFile.__init__(self, fname) - + XMLCfgCreator.__init__(self, fname) pubkey_path = os.path.dirname(self.name) + ".pub" pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path)) self.pubkey_creator = CfgPublicKeyCreator(pubkey_name) self.cmd = Executor() - __init__.__doc__ = CfgCreator.__init__.__doc__ - - @property - def passphrase(self): - """ The passphrase used to encrypt private keys """ - if HAS_CRYPTO and Bcfg2.Options.setup.sshkeys_passphrase: - return Bcfg2.Options.setup.passphrases[ - Bcfg2.Options.setup.sshkeys_passphrase] - return None - - def handle_event(self, event): - CfgCreator.handle_event(self, event) - StructFile.HandleEvent(self, event) - handle_event.__doc__ = CfgCreator.handle_event.__doc__ + __init__.__doc__ = XMLCfgCreator.__init__.__doc__ def _gen_keypair(self, metadata, spec=None): """ Generate a keypair according to the given client medata @@ -117,45 +97,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): shutil.rmtree(tempdir) raise - def get_specificity(self, metadata, spec=None): - """ Get config settings for key generation specificity - (per-host or per-group). - - :param metadata: The client metadata to create data for - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param spec: The key specification to follow when creating the - keys. This should be an XML document that only - contains key specification data that applies to - the given client metadata, and may be obtained by - doing ``self.XMLMatch(metadata)`` - :type spec: lxml.etree._Element - :returns: dict - A dict of specificity arguments suitable for - passing to - :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.write_data` - or - :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.get_filename` - """ - if spec is None: - spec = self.XMLMatch(metadata) - category = spec.get("category", Bcfg2.Options.setup.sshkeys_category) - if category is None: - per_host_default = "true" - else: - per_host_default = "false" - per_host = spec.get("perhost", per_host_default).lower() == "true" - - specificity = dict(host=metadata.hostname) - if category and not per_host: - group = metadata.group_in_category(category) - if group: - specificity = dict(group=group, - prio=int(spec.get("priority", 50))) - else: - self.logger.info("Cfg: %s has no group in category %s, " - "creating host-specific key" % - (metadata.hostname, category)) - return specificity - # pylint: disable=W0221 def create_data(self, entry, metadata, return_pair=False): """ Create data for the given entry on the given client @@ -176,7 +117,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): ``return_pair`` is set to True """ spec = self.XMLMatch(metadata) - specificity = self.get_specificity(metadata, spec) + specificity = self.get_specificity(metadata) filename = self._gen_keypair(metadata, spec) try: @@ -190,12 +131,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): # encrypt the private key, write to the proper place, and # return it privkey = open(filename).read() - if HAS_CRYPTO and self.passphrase: - self.debug_log("Cfg: Encrypting key data at %s" % filename) - privkey = Bcfg2.Server.Encryption.ssl_encrypt(privkey, - self.passphrase) - specificity['ext'] = '.crypt' - self.write_data(privkey, **specificity) if return_pair: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py index 4c61e338e..de1848159 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py @@ -4,7 +4,7 @@ to create SSH keys on the fly. """ import lxml.etree from Bcfg2.Server.Plugin import StructFile, PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, CFG +from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, get_cfg class CfgPublicKeyCreator(CfgCreator, StructFile): @@ -17,7 +17,7 @@ class CfgPublicKeyCreator(CfgCreator, StructFile): creation of a keypair when a public key is created. """ #: Different configurations for different clients/groups can be - #: handled with Client and Group tags within privkey.xml + #: handled with Client and Group tags within pubkey.xml __specific__ = False #: Handle XML specifications of private keys @@ -29,7 +29,7 @@ class CfgPublicKeyCreator(CfgCreator, StructFile): def __init__(self, fname): CfgCreator.__init__(self, fname) StructFile.__init__(self, fname) - self.cfg = CFG + self.cfg = get_cfg() __init__.__doc__ = CfgCreator.__init__.__doc__ def create_data(self, entry, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py new file mode 100644 index 000000000..92fcc4cd8 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py @@ -0,0 +1,255 @@ +""" Cfg creator that creates SSL certs """ + +import os +import sys +import tempfile +import lxml.etree +import Bcfg2.Options +from Bcfg2.Utils import Executor +from Bcfg2.Compat import ConfigParser +from Bcfg2.Server.FileMonitor import get_fam +from Bcfg2.Server.Plugin import PluginExecutionError +from Bcfg2.Server.Plugins.Cfg import CfgCreationError, XMLCfgCreator, \ + CfgCreator, CfgVerifier, CfgVerificationError, get_cfg + + +class CfgSSLCACertCreator(XMLCfgCreator, CfgVerifier): + """ This class acts as both a Cfg creator that creates SSL certs, + and as a Cfg verifier that verifies SSL certs. """ + + #: Different configurations for different clients/groups can be + #: handled with Client and Group tags within pubkey.xml + __specific__ = False + + #: Handle XML specifications of private keys + __basenames__ = ['sslcert.xml'] + + cfg_section = "sslca" + options = [ + Bcfg2.Options.Option( + cf=("sslca", "category"), dest="sslca_category", + help="Metadata category that generated SSL keys are specific to"), + Bcfg2.Options.Option( + cf=("sslca", "passphrase"), dest="sslca_passphrase", + help="Passphrase used to encrypt generated SSL keys"), + Bcfg2.Options.WildcardSectionGroup( + Bcfg2.Options.PathOption( + cf=("sslca_*", "config"), + help="Path to the openssl config for the CA"), + Bcfg2.Options.Option( + cf=("sslca_*", "passphrase"), + help="Passphrase for the CA private key"), + Bcfg2.Options.PathOption( + cf=("sslca_*", "chaincert"), + help="Path to the SSL chaining certificate for verification"), + Bcfg2.Options.BooleanOption( + cf=("sslca_*", "root_ca"), + help="Whether or not <chaincert> is a root CA (as opposed to " + "an intermediate cert"), + prefix="")] + + def __init__(self, fname): + XMLCfgCreator.__init__(self, fname) + CfgVerifier.__init__(self, fname, None) + self.cmd = Executor() + self.cfg = get_cfg() + + def build_req_config(self, metadata): + """ Generates a temporary openssl configuration file that is + used to generate the required certificate request. """ + fd, fname = tempfile.mkstemp() + cfp = ConfigParser.ConfigParser({}) + cfp.optionxform = str + defaults = dict( + req=dict( + default_md='sha1', + distinguished_name='req_distinguished_name', + req_extensions='v3_req', + x509_extensions='v3_req', + prompt='no'), + req_distinguished_name=dict(), + v3_req=dict(subjectAltName='@alt_names'), + alt_names=dict()) + for section in list(defaults.keys()): + cfp.add_section(section) + for key in defaults[section]: + cfp.set(section, key, defaults[section][key]) + spec = self.XMLMatch(metadata) + cert = spec.find("Cert") + altnamenum = 1 + altnames = spec.findall('subjectAltName') + altnames.extend(list(metadata.aliases)) + altnames.append(metadata.hostname) + for altname in altnames: + cfp.set('alt_names', 'DNS.' + str(altnamenum), altname) + altnamenum += 1 + for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: + if cert.get(item): + cfp.set('req_distinguished_name', item, cert.get(item)) + cfp.set('req_distinguished_name', 'CN', metadata.hostname) + self.debug_log("Cfg: Writing temporary CSR config to %s" % fname) + try: + cfp.write(os.fdopen(fd, 'w')) + except IOError: + raise CfgCreationError("Cfg: Failed to write temporary CSR config " + "file: %s" % sys.exc_info()[1]) + return fname + + def build_request(self, keyfile, metadata): + """ Create the certificate request """ + req_config = self.build_req_config(metadata) + try: + fd, req = tempfile.mkstemp() + os.close(fd) + cert = self.XMLMatch(metadata).find("Cert") + days = cert.get("days", "365") + cmd = ["openssl", "req", "-new", "-config", req_config, + "-days", days, "-key", keyfile, "-text", "-out", req] + result = self.cmd.run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate CSR: %s" % + result.error) + return req + finally: + try: + os.unlink(req_config) + except OSError: + self.logger.error("Cfg: Failed to unlink temporary CSR " + "config: %s" % sys.exc_info()[1]) + + def get_ca(self, name): + """ get a dict describing a CA from the config file """ + rv = dict() + prefix = "sslca_%s_" % name + for attr in dir(Bcfg2.Options.setup): + if attr.startswith(prefix): + rv[attr[len(prefix):]] = getattr(Bcfg2.Options.setup, attr) + return rv + + def create_data(self, entry, metadata): + """ generate a new cert """ + self.logger.info("Cfg: Generating new SSL cert for %s" % self.name) + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get('ca', 'default')) + req = self.build_request(self._get_keyfile(cert, metadata), metadata) + try: + days = cert.get('days', '365') + cmd = ["openssl", "ca", "-config", ca['config'], "-in", req, + "-days", days, "-batch"] + passphrase = ca.get('passphrase') + if passphrase: + cmd.extend(["-passin", "pass:%s" % passphrase]) + result = self.cmd.run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate cert: %s" % + result.error) + except KeyError: + raise CfgCreationError("Cfg: [sslca_%s] section has no 'config' " + "option" % cert.get('ca', 'default')) + finally: + try: + os.unlink(req) + except OSError: + self.logger.error("Cfg: Failed to unlink temporary CSR: %s " % + sys.exc_info()[1]) + data = result.stdout + if cert.get('append_chain') and 'chaincert' in ca: + data += open(ca['chaincert']).read() + + self.write_data(data, **self.get_specificity(metadata)) + return data + + def verify_entry(self, entry, metadata, data): + fd, fname = tempfile.mkstemp() + self.debug_log("Cfg: Writing SSL cert %s to temporary file %s for " + "verification" % (entry.get("name"), fname)) + os.fdopen(fd, 'w').write(data) + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get('ca', 'default')) + try: + if ca.get('chaincert'): + self.verify_cert_against_ca(fname, entry, metadata) + self.verify_cert_against_key(fname, + self._get_keyfile(cert, metadata)) + finally: + os.unlink(fname) + + def _get_keyfile(self, cert, metadata): + """ Given a <Cert/> element and client metadata, return the + full path to the file on the filesystem that the key lives in.""" + keypath = cert.get("key") + eset = self.cfg.entries[keypath] + try: + return eset.best_matching(metadata).name + except PluginExecutionError: + # SSL key needs to be created + try: + creator = eset.best_matching(metadata, + eset.get_handlers(metadata, + CfgCreator)) + except PluginExecutionError: + raise CfgCreationError("Cfg: No SSL key or key creator " + "defined for %s" % keypath) + + keyentry = lxml.etree.Element("Path", name=keypath) + creator.create_data(keyentry, metadata) + + tries = 0 + while True: + if tries >= 10: + raise CfgCreationError("Cfg: Timed out waiting for event " + "on SSL key at %s" % keypath) + get_fam().handle_events_in_interval(1) + try: + return eset.best_matching(metadata).name + except PluginExecutionError: + tries += 1 + continue + + def verify_cert_against_ca(self, filename, entry, metadata): + """ + check that a certificate validates against the ca cert, + and that it has not expired. + """ + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get("ca", "default")) + chaincert = ca.get('chaincert') + cmd = ["openssl", "verify"] + is_root = ca.get('root_ca', "false").lower() == 'true' + if is_root: + cmd.append("-CAfile") + else: + # verifying based on an intermediate cert + cmd.extend(["-purpose", "sslserver", "-untrusted"]) + cmd.extend([chaincert, filename]) + self.debug_log("Cfg: Verifying %s against CA" % entry.get("name")) + result = self.cmd.run(cmd) + if result.stdout == cert + ": OK\n": + self.debug_log("Cfg: %s verified successfully against CA" % + entry.get("name")) + else: + raise CfgVerificationError("%s failed verification against CA: %s" + % (entry.get("name"), result.error)) + + def _get_modulus(self, fname, ftype="x509"): + """ get the modulus from the given file """ + cmd = ["openssl", ftype, "-noout", "-modulus", "-in", fname] + self.debug_log("Cfg: Getting modulus of %s for verification: %s" % + (fname, " ".join(cmd))) + result = self.cmd.run(cmd) + if not result.success: + raise CfgVerificationError("Failed to get modulus of %s: %s" % + (fname, result.error)) + return result.stdout.strip() + + def verify_cert_against_key(self, filename, keyfile): + """ check that a certificate validates against its private + key. """ + cert = self._get_modulus(filename) + key = self._get_modulus(keyfile, ftype="rsa") + if cert == key: + self.debug_log("Cfg: %s verified successfully against key %s" % + (filename, keyfile)) + else: + raise CfgVerificationError("%s failed verification against key %s" + % (filename, keyfile)) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py new file mode 100644 index 000000000..a158302be --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py @@ -0,0 +1,36 @@ +""" Cfg creator that creates SSL keys """ + +from Bcfg2.Utils import Executor +from Bcfg2.Server.Plugins.Cfg import CfgCreationError, XMLCfgCreator + + +class CfgSSLCAKeyCreator(XMLCfgCreator): + """ Cfg creator that creates SSL keys """ + + #: Different configurations for different clients/groups can be + #: handled with Client and Group tags within sslkey.xml + __specific__ = False + + __basenames__ = ["sslkey.xml"] + + cfg_section = "sslca" + + def create_data(self, entry, metadata): + self.logger.info("Cfg: Generating new SSL key for %s" % self.name) + spec = self.XMLMatch(metadata) + key = spec.find("Key") + if not key: + key = dict() + ktype = key.get('type', 'rsa') + bits = key.get('bits', '2048') + if ktype == 'rsa': + cmd = ["openssl", "genrsa", bits] + elif ktype == 'dsa': + cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] + result = Executor().run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate key %s for %s: %s" % + (self.name, metadata.hostname, + result.error)) + self.write_data(result.stdout, **self.get_specificity(metadata)) + return result.stdout diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 99afac7eb..eea0a3456 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -10,16 +10,29 @@ import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 -from Bcfg2.Compat import u_str, unicode, b64encode, any, oct_mode +from Bcfg2.Compat import u_str, unicode, b64encode, any, walk_packages # pylint: enable=W0622 -#: CFG is a reference to the :class:`Bcfg2.Server.Plugins.Cfg.Cfg` -#: plugin object created by the Bcfg2 core. This is provided so that -#: the handler objects can access it as necessary, since the existing -#: :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` and -#: :class:`Bcfg2.Server.Plugin.helpers.EntrySet` classes have no -#: facility for passing it otherwise. -CFG = None +try: + import Bcfg2.Server.Encryption + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False + +_handlers = [m[1] # pylint: disable=C0103 + for m in walk_packages(path=__path__)] + +_CFG = None + + +def get_cfg(): + """ Get the :class:`Bcfg2.Server.Plugins.Cfg.Cfg` plugin object + created by the Bcfg2 core. This is provided so that the handler + objects can access it as necessary, since the existing + :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` and + :class:`Bcfg2.Server.Plugin.helpers.EntrySet` classes have no + facility for passing it otherwise.""" + return _CFG class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): @@ -288,7 +301,7 @@ class CfgCreator(CfgBaseFileMatcher): :type name: string .. ----- - .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgCreator.__specific__ + .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgInfo.__specific__ """ CfgBaseFileMatcher.__init__(self, fname, None) @@ -310,7 +323,9 @@ class CfgCreator(CfgBaseFileMatcher): ``host`` is given, it will be host-specific. It will be group-specific if ``group`` and ``prio`` are given. If neither ``host`` nor ``group`` is given, the filename will be - non-specific. + non-specific. In general, this will be called as:: + + self.get_filename(**self.get_specificity(metadata)) :param host: The file applies to the given host :type host: bool @@ -341,6 +356,9 @@ class CfgCreator(CfgBaseFileMatcher): written as a host-specific file, or as a group-specific file if ``group`` and ``prio`` are given. If neither ``host`` nor ``group`` is given, it will be written as a non-specific file. + In general, this will be called as:: + + self.write_data(data, **self.get_specificity(metadata)) :param data: The data to write :type data: string @@ -360,7 +378,7 @@ class CfgCreator(CfgBaseFileMatcher): :raises: :exc:`Bcfg2.Server.Plugins.Cfg.CfgCreationError` """ fileloc = self.get_filename(host=host, group=group, prio=prio, ext=ext) - self.debug_log("%s: Writing new file %s" % (self.name, fileloc)) + self.debug_log("Cfg: Writing new file %s" % fileloc) try: os.makedirs(os.path.dirname(fileloc)) except OSError: @@ -376,6 +394,95 @@ class CfgCreator(CfgBaseFileMatcher): raise CfgCreationError("Could not write %s: %s" % (fileloc, err)) +class XMLCfgCreator(CfgCreator, # pylint: disable=W0223 + Bcfg2.Server.Plugin.StructFile): + """ A CfgCreator that uses XML to describe how data should be + generated. """ + + #: Whether or not the created data from this class can be + #: encrypted + encryptable = True + + #: Encryption and creation settings can be stored in bcfg2.conf, + #: either under the [cfg] section, or under the named section. + cfg_section = None + + def __init__(self, name): + CfgCreator.__init__(self, name) + Bcfg2.Server.Plugin.StructFile.__init__(self, name) + + def handle_event(self, event): + CfgCreator.handle_event(self, event) + Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event) + + @property + def passphrase(self): + """ The passphrase used to encrypt created data """ + if self.cfg_section: + localopt = "%s_passphrase" % self.cfg_section + passphrase = getattr(Bcfg2.Options.setup, localopt, + Bcfg2.Options.setup.cfg_passphrase) + else: + passphrase = Bcfg2.Options.setup.cfg_passphrase + if passphrase is None: + return None + try: + return Bcfg2.Options.setup.passphrases[passphrase] + except KeyError: + raise CfgCreationError("%s: No such passphrase: %s" % + (self.__class__.__name__, passphrase)) + + @property + def category(self): + """ The category to which created data is specific """ + if self.cfg_section: + localopt = "%s_category" % self.cfg_section + return getattr(Bcfg2.Options.setup, localopt, + Bcfg2.Options.setup.cfg_category) + else: + return Bcfg2.Options.setup.cfg_category + + def write_data(self, data, host=None, group=None, prio=0, ext=''): + if HAS_CRYPTO and self.encryptable and self.passphrase: + self.debug_log("Cfg: Encrypting created data") + data = Bcfg2.Server.Encryption.ssl_encrypt(data, self.passphrase) + ext = '.crypt' + CfgCreator.write_data(self, data, host=host, group=group, prio=prio, + ext=ext) + + def get_specificity(self, metadata): + """ Get config settings for key generation specificity + (per-host or per-group). + + :param metadata: The client metadata to create data for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: dict - A dict of specificity arguments suitable for + passing to + :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.write_data` + or + :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.get_filename` + """ + category = self.xdata.get("category", self.category) + if category is None: + per_host_default = "true" + else: + per_host_default = "false" + per_host = self.xdata.get("perhost", + per_host_default).lower() == "true" + + specificity = dict(host=metadata.hostname) + if category and not per_host: + group = metadata.group_in_category(category) + if group: + specificity = dict(group=group, + prio=int(self.xdata.get("priority", 50))) + else: + self.logger.info("Cfg: %s has no group in category %s, " + "creating host-specific data" % + (metadata.hostname, category)) + return specificity + + class CfgVerificationError(Exception): """ Raised by :func:`Bcfg2.Server.Plugins.Cfg.CfgVerifier.verify_entry` when an @@ -411,7 +518,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): def __init__(self, basename, path, entry_type): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type) self.specific = None - self._handlers = None __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ def set_debug(self, debug): @@ -420,14 +526,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): entry.set_debug(debug) return rv - @property - def handlers(self): - """ A list of Cfg handler classes. """ - if self._handlers is None: - self._handlers = Bcfg2.Options.setup.cfg_handlers - self._handlers.sort(key=operator.attrgetter("__priority__")) - return self._handlers - def handle_event(self, event): """ Dispatch a FAM event to :func:`entry_init` or the appropriate child handler object. @@ -444,7 +542,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): # process a bogus changed event like a created return - for hdlr in self.handlers: + for hdlr in Bcfg2.Options.setup.cfg_handlers: if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -783,32 +881,27 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, '--cfg-validation', cf=('cfg', 'validation'), default=True, help='Run validation on Cfg files'), Bcfg2.Options.Option( + cf=('cfg', 'category'), dest="cfg_category", + help='The default name of the metadata category that created data ' + 'is specific to'), + Bcfg2.Options.Option( + cf=('cfg', 'passphrase'), dest="cfg_passphrase", + help='The default passphrase name used to encrypt created data'), + Bcfg2.Options.Option( cf=("cfg", "handlers"), dest="cfg_handlers", help="Cfg handlers to load", type=Bcfg2.Options.Types.comma_list, action=CfgHandlerAction, - default=['CfgAuthorizedKeysGenerator', 'CfgEncryptedGenerator', - 'CfgCheetahGenerator', 'CfgEncryptedCheetahGenerator', - 'CfgGenshiGenerator', 'CfgEncryptedGenshiGenerator', - 'CfgExternalCommandVerifier', 'CfgInfoXML', - 'CfgPlaintextGenerator', - 'CfgPrivateKeyCreator', 'CfgPublicKeyCreator'])] + default=_handlers)] def __init__(self, core, datastore): - global CFG # pylint: disable=W0603 + global _CFG # pylint: disable=W0603 Bcfg2.Server.Plugin.GroupSpool.__init__(self, core, datastore) Bcfg2.Server.Plugin.PullTarget.__init__(self) - self._handlers = None - CFG = self + Bcfg2.Options.setup.cfg_handlers.sort( + key=operator.attrgetter("__priority__")) + _CFG = self __init__.__doc__ = Bcfg2.Server.Plugin.GroupSpool.__init__.__doc__ - @property - def handlers(self): - """ A list of Cfg handler classes. """ - if self._handlers is None: - self._handlers = Bcfg2.Options.setup.cfg_handlers - self._handlers.sort(key=operator.attrgetter("__priority__")) - return self._handlers - def has_generator(self, entry, metadata): """ Return True if the given entry can be generated for the given metadata; False otherwise diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index aa336ff23..1da7c8fec 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -4,6 +4,7 @@ template to dynamically set additional groups for clients. """ import os import lxml.etree import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Metadata import MetadataGroup class GroupLogicConfig(Bcfg2.Server.Plugin.StructFile): @@ -11,10 +12,17 @@ class GroupLogicConfig(Bcfg2.Server.Plugin.StructFile): create = lxml.etree.Element("GroupLogic", nsmap=dict(py="http://genshi.edgewall.org/")) - def _match(self, item, metadata): + def _match(self, item, metadata, *args): if item.tag == 'Group' and not len(item.getchildren()): return [item] - return Bcfg2.Server.Plugin.StructFile._match(self, item, metadata) + return Bcfg2.Server.Plugin.StructFile._match(self, item, metadata, + *args) + + def _xml_match(self, item, metadata, *args): + if item.tag == 'Group' and not len(item.getchildren()): + return [item] + return Bcfg2.Server.Plugin.StructFile._xml_match(self, item, metadata, + *args) class GroupLogic(Bcfg2.Server.Plugin.Plugin, @@ -30,5 +38,11 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, should_monitor=True) def get_additional_groups(self, metadata): - return [el.get("name") - for el in self.config.XMLMatch(metadata).findall("Group")] + rv = [] + for el in self.config.XMLMatch(metadata).findall("Group"): + if el.get("category"): + rv.append(MetadataGroup(el.get("name"), + category=el.get("category"))) + else: + rv.append(el.get("name")) + return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 12ece1f19..db104b27e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -16,10 +16,10 @@ import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked +from Bcfg2.Server.Cache import Cache from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 from Bcfg2.version import Bcfg2VersionInfo - # pylint: disable=C0103 ClientVersions = None MetadataClientModel = None @@ -89,7 +89,7 @@ def load_django_models(): def keys(self): """ Get keys for the mapping """ - return [c.hostname for c in MetadataClientModel.objects.all()] + return list(iter(self)) def __contains__(self, key): try: @@ -102,17 +102,12 @@ def load_django_models(): class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): """Handles xml config files and all XInclude statements""" - def __init__(self, metadata, watch_clients, basefile): - # we tell XMLFileBacked _not_ to add a monitor for this file, - # because the main Metadata plugin has already added one. - # then we immediately set should_monitor to the proper value, - # so that XInclude'd files get properly watched + def __init__(self, metadata, basefile): fpath = os.path.join(metadata.data, basefile) toptag = os.path.splitext(basefile)[0].title() Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath, should_monitor=False, create=toptag) - self.should_monitor = watch_clients self.metadata = metadata self.basefile = basefile self.data = None @@ -257,8 +252,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): def add_monitor(self, fpath): self.extras.append(fpath) - if self.should_monitor: - self.fam.AddMonitor(fpath, self.metadata) + self.fam.AddMonitor(fpath, self.metadata) def HandleEvent(self, event=None): """Handle fam events""" @@ -500,7 +494,6 @@ class MetadataGroup(tuple): # pylint: disable=E0012,R0924 class Metadata(Bcfg2.Server.Plugin.Metadata, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" @@ -518,12 +511,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, help='Default client authentication method')] options_parsed_hook = staticmethod(load_django_models) - def __init__(self, core, datastore, watch_clients=True): + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Metadata.__init__(self) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) - self.watch_clients = watch_clients self.states = dict() self.extra = dict() self.handlers = dict() @@ -554,22 +545,26 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.raliases = {} # mapping of groupname -> MetadataGroup object self.groups = {} - # mappings of predicate -> MetadataGroup object + # mappings of groupname -> [predicates] self.group_membership = dict() self.negated_groups = dict() + # list of group names in document order + self.ordered_groups = [] # mapping of hostname -> version string if self._use_db: self.versions = ClientVersions(core, # pylint: disable=E1102 datastore) else: self.versions = dict() + self.uuid = {} self.session_cache = {} + self.cache = Cache("Metadata") self.default = None self.pdirty = False self.password = Bcfg2.Options.setup.password self.query = MetadataQuery(core.build_metadata, - lambda: list(self.clients), + self.list_clients, self.get_client_names_by_groups, self.get_client_names_by_profiles, self.get_all_group_names, @@ -595,17 +590,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _handle_file(self, fname): """ set up the necessary magic for handling a metadata file (clients.xml or groups.xml, e.g.) """ - if self.watch_clients: - try: - Bcfg2.Server.FileMonitor.get_fam().AddMonitor( - os.path.join(self.data, fname), self) - except: - err = sys.exc_info()[1] - msg = "Unable to add file monitor for %s: %s" % (fname, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - self.states[fname] = False - xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname) + try: + Bcfg2.Server.FileMonitor.get_fam().AddMonitor( + os.path.join(self.data, fname), self) + except: + err = sys.exc_info()[1] + msg = "Unable to add file monitor for %s: %s" % (fname, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) + self.states[fname] = False + xmlcfg = XMLMetadataConfig(self, fname) aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(fname)) self.handlers[xmlcfg.HandleEvent] = getattr(self, "_handle_%s_event" % aname) @@ -860,51 +854,34 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if self._use_db: self.clients = self.list_clients() + def _get_condition(self, element): + """ Return a predicate that returns True if a client meets + the condition specified in the given Group or Client + element """ + negate = element.get('negate', 'false').lower() == 'true' + pname = element.get("name") + if element.tag == 'Group': + return lambda c, g, _: negate != (pname in g) + elif element.tag == 'Client': + return lambda c, g, _: negate != (pname == c) + + def _get_category_condition(self, grpname): + """ get a predicate that returns False if a client is already + a member of a group in the given group's category, True + otherwise""" + return lambda client, _, categories: \ + bool(self._check_category(client, grpname, categories)) + + def _aggregate_conditions(self, conditions): + """ aggregate all conditions on a given group declaration + into a single predicate """ + return lambda client, groups, cats: \ + all(cond(client, groups, cats) for cond in conditions) + def _handle_groups_xml_event(self, _): # pylint: disable=R0912 """ re-read groups.xml on any event on it """ self.groups = {} - # these three functions must be separate functions in order to - # ensure that the scope is right for the closures they return - def get_condition(element): - """ Return a predicate that returns True if a client meets - the condition specified in the given Group or Client - element """ - negate = element.get('negate', 'false').lower() == 'true' - pname = element.get("name") - if element.tag == 'Group': - return lambda c, g, _: negate != (pname in g) - elif element.tag == 'Client': - return lambda c, g, _: negate != (pname == c) - - def get_category_condition(category, gname): - """ get a predicate that returns False if a client is - already a member of a group in the given category, True - otherwise """ - def in_cat(client, groups, categories): # pylint: disable=W0613 - """ return True if the client is already a member of a - group in the category given in the enclosing function, - False otherwise """ - if category in categories: - if (gname not in self.groups or - client not in self.groups[gname].warned): - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, gname, category, - client, categories[category])) - if gname in self.groups: - self.groups[gname].warned.append(client) - return False - return True - return in_cat - - def aggregate_conditions(conditions): - """ aggregate all conditions on a given group declaration - into a single predicate """ - return lambda client, groups, cats: \ - all(cond(client, groups, cats) for cond in conditions) - # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of # parsing groups.xml didn't support nested groups; in the old @@ -930,6 +907,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership = dict() self.negated_groups = dict() + self.ordered_groups = [] # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is @@ -940,40 +918,44 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # XPath. We do the same thing for Client tags. for el in self.groups_xml.xdata.xpath("//Groups/Group//*") + \ self.groups_xml.xdata.xpath("//Groups/Client//*"): - if ((el.tag != 'Group' and el.tag != 'Client') or - el.getchildren()): + if (el.tag != 'Group' and el.tag != 'Client') or el.getchildren(): continue conditions = [] for parent in el.iterancestors(): - cond = get_condition(parent) + cond = self._get_condition(parent) if cond: conditions.append(cond) gname = el.get("name") if el.get("negate", "false").lower() == "true": - self.negated_groups[aggregate_conditions(conditions)] = \ - self.groups[gname] + self.negated_groups.setdefault(gname, []) + self.negated_groups[gname].append( + self._aggregate_conditions(conditions)) else: if self.groups[gname].category: - conditions.append( - get_category_condition(self.groups[gname].category, - gname)) + conditions.append(self._get_category_condition(gname)) - self.group_membership[aggregate_conditions(conditions)] = \ - self.groups[gname] + if gname not in self.ordered_groups: + self.ordered_groups.append(gname) + self.group_membership.setdefault(gname, []) + self.group_membership[gname].append( + self._aggregate_conditions(conditions)) self.states['groups.xml'] = True - def expire_cache(self, key=None): - self.core.metadata_cache.expire(key) - def HandleEvent(self, event): """Handle update events for data files.""" for handles, event_handler in self.handlers.items(): if handles(event): # clear the entire cache when we get an event for any # metadata file - self.expire_cache() + self.cache.expire() + + # clear out the list of category suppressions that + # have been warned about, since this may change when + # clients.xml or groups.xml changes. + for group in self.groups.values(): + group.warned = [] event_handler(event) if False not in list(self.states.values()) and self.debug_flag: @@ -1112,30 +1094,85 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, categories = dict() while numgroups != len(groups): numgroups = len(groups) - for predicate, group in self.group_membership.items(): - if group.name in groups: + newgroups = set() + removegroups = set() + for grpname in self.ordered_groups: + if grpname in groups: continue - if predicate(client, groups, categories): - groups.add(group.name) - if group.category: - categories[group.category] = group.name - for predicate, group in self.negated_groups.items(): - if group.name not in groups: + if any(p(client, groups, categories) + for p in self.group_membership[grpname]): + newgroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + categories[self.groups[grpname].category] = grpname + groups.update(newgroups) + for grpname, predicates in self.negated_groups.items(): + if grpname not in groups: continue - if predicate(client, groups, categories): - groups.remove(group.name) - if group.category: - del categories[group.category] + if any(p(client, groups, categories) for p in predicates): + removegroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + del categories[self.groups[grpname].category] + groups.difference_update(removegroups) return (groups, categories) + def _check_category(self, client, grpname, categories): + """ Determine if the given client is already a member of a + group in the same category as the named group. + + The return value is one of three possibilities: + + * If the client is already a member of a group in the same + category, then False is returned (i.e., the category check + failed); + * If the group is not in any categories, then True is returned; + * If the group is not a member of a group in the category, + then the name of the category is returned. This makes it + easy to add the category to the ClientMetadata object (or + other category list). + + If a pure boolean value is required, you can do + ``bool(self._check_category(...))``. + """ + if grpname not in self.groups: + return True + category = self.groups[grpname].category + if not category: + return True + if category in categories: + if client not in self.groups[grpname].warned: + self.logger.warning("%s: Group %s suppressed by category %s; " + "%s already a member of %s" % + (self.name, grpname, category, + client, categories[category])) + self.groups[grpname].warned.append(client) + return False + return category + + def _check_and_add_category(self, client, grpname, categories): + """ If the client is not a member of a group in the same + category as the named group, then the category is added to + ``categories``. + :func:`Bcfg2.Server.Plugins.Metadata._check_category` is used + to determine if the category can be added. + + If the category check failed, returns False; otherwise, + returns True. """ + rv = self._check_category(client, grpname, categories) + if rv and rv is not True: + categories[rv] = grpname + return True + return rv + def get_initial_metadata(self, client): # pylint: disable=R0914,R0912 """Return the metadata for a given client.""" if False in list(self.states.values()): raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " "been read yet") client = client.lower() - if client in self.core.metadata_cache: - return self.core.metadata_cache[client] + if client in self.cache: + return self.cache[client] if client in self.aliases: client = self.aliases[client] @@ -1149,30 +1186,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, Handles setting categories and category suppression. Returns the new profile for the client (which might be unchanged). """ - groups.add(grpname) if grpname in self.groups: - group = self.groups[grpname] - category = group.category - if category: - if category in categories: - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, grpname, category, - client, categories[category])) - return - categories[category] = grpname - if not profile and group.is_profile: + if not self._check_and_add_category(client, grpname, + categories): + return profile + groups.add(grpname) + if not profile and self.groups[grpname].is_profile: return grpname else: return profile + else: + groups.add(grpname) + return profile if client not in self.clients: pgroup = None if client in self.clientgroups: pgroup = self.clientgroups[client][0] + self.debug_log("%s: Adding new client with profile %s" % + (self.name, pgroup)) elif self.default: pgroup = self.default + self.debug_log("%s: Adding new client with default profile %s" + % (self.name, pgroup)) if pgroup: self.set_profile(client, pgroup, (None, None), @@ -1189,6 +1225,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.groups[cgroup] = MetadataGroup(cgroup) profile = _add_group(cgroup) + # we do this before setting the default because there may be + # groups set in <Client> tags in groups.xml that we want to + # set groups, categories = self._merge_groups(client, groups, categories=categories) @@ -1230,15 +1269,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, addresses, categories, uuid, password, version, self.query) if self.core.metadata_cache_mode == 'initial': - self.core.metadata_cache[client] = rv + self.cache[client] = rv return rv def get_all_group_names(self): """ return a list of all group names """ all_groups = set() all_groups.update(self.groups.keys()) - all_groups.update([g.name for g in self.group_membership.values()]) - all_groups.update([g.name for g in self.negated_groups.values()]) + all_groups.update(self.group_membership.keys()) + all_groups.update(self.negated_groups.keys()) for grp in self.clientgroups.values(): all_groups.update(grp) return all_groups @@ -1251,7 +1290,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_profiles(self, profiles): """ return a list of names of clients in the given profile groups """ rv = [] - for client in list(self.clients): + for client in self.list_clients(): mdata = self.core.build_metadata(client) if mdata.profile in profiles: rv.append(client) @@ -1259,34 +1298,33 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_groups(self, groups): """ return a list of names of clients in the given groups """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.groups.issuperset(groups)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.groups.issuperset(groups): + rv.append(client) + return rv def get_client_names_by_bundles(self, bundles): """ given a list of bundles, return a list of names of clients that use those bundles """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.bundles.issuperset(bundles): + rv.append(client) + return rv def merge_additional_groups(self, imd, groups): for group in groups: if group in imd.groups: continue - if group in self.groups and self.groups[group].category: - category = self.groups[group].category - if self.groups[group].category in imd.categories: - self.logger.warning("%s: Group %s suppressed by category " - "%s; %s already a member of %s" % - (self.name, group, category, - imd.hostname, - imd.categories[category])) - continue - imd.categories[category] = group + if not self._check_and_add_category(imd.hostname, group, + imd.categories): + continue imd.groups.add(group) - self._merge_groups(imd.hostname, imd.groups, - categories=imd.categories) - + self._merge_groups(imd.hostname, imd.groups, categories=imd.categories) for group in imd.groups: if group in self.groups: imd.bundles.update(self.groups[group].bundles) @@ -1451,7 +1489,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, instances = {} rv = [] - for client in list(self.clients): + for client in list(self.list_clients()): if not include_client(client): continue if client in self.clientgroups: diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index dcd495d77..a27664215 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -21,9 +21,9 @@ class NagiosGen(Plugin, Generator): self.config = \ StructFile(os.path.join(self.data, 'config.xml'), should_monitor=True, create=self.name) - self.Entries = {'Path': - {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} + self.Entries = { + 'Path': {'/etc/nagiosgen.status': self.createhostconfig, + '/etc/nagios/conf.d/bcfg2.cfg': self.createserverconfig}} self.client_attrib = {'encoding': 'ascii', 'owner': 'root', diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 5af9c1591..56285705a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -8,6 +8,7 @@ import glob import shutil import lxml.etree import Bcfg2.Options +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin from Bcfg2.Compat import urlopen, HTTPError, URLError, MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ @@ -81,7 +82,6 @@ class OnDemandDict(MutableMapping): class Packages(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, @@ -136,12 +136,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] - __child_rmi__ = Bcfg2.Server.Plugin.Plugin.__child_rmi__ + \ - [('Refresh', 'expire_cache'), ('Reload', 'expire_cache')] - def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) @@ -185,7 +181,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`, #: a unique key identifying the collection by its *config*, #: which could be shared among multiple clients. - self.collections = dict() + self.collections = Bcfg2.Server.Cache.Cache("Packages", "collections") #: clients is a cache mapping of hostname -> #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` @@ -193,21 +189,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. - self.clients = dict() - - #: groupcache caches group lookups. It maps Collections (via - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) - #: to sets of package groups, and thence to the packages - #: indicated by those groups. - self.groupcache = dict() - - #: pkgcache caches complete package sets. It maps Collections - #: (via - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) - #: to sets of initial packages, and thence to the final - #: (complete) package selections resolved from the initial - #: packages - self.pkgcache = dict() + self.clients = Bcfg2.Server.Cache.Cache("Packages", "cache") + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ @@ -400,11 +383,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, groups.sort() # check for this set of groups in the group cache + gcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_groups", + collection.cachekey) gkey = hash(tuple(groups)) - if gkey not in self.groupcache[collection.cachekey]: - self.groupcache[collection.cachekey][gkey] = \ - collection.get_groups(groups) - for pkgs in self.groupcache[collection.cachekey][gkey].values(): + if gkey not in gcache: + gcache[gkey] = collection.get_groups(groups) + for pkgs in gcache[gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution @@ -412,10 +396,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # check for this set of packages in the package cache pkey = hash(tuple(base)) - if pkey not in self.pkgcache[collection.cachekey]: - self.pkgcache[collection.cachekey][pkey] = \ - collection.complete(base) - packages, unknown = self.pkgcache[collection.cachekey][pkey] + pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", + collection.cachekey) + if pkey not in pcache: + pcache[pkey] = collection.complete(base) + packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) @@ -441,7 +426,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config() return True - def expire_cache(self, _=None): + def child_reload(self, _=None): + """ Reload the Packages configuration on a child process. """ self.Reload() def _load_config(self, force_update=False): @@ -472,10 +458,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, collection.setup_data(force_update) # clear Collection and package caches - self.clients = dict() - self.collections = dict() - self.groupcache = dict() - self.pkgcache = dict() + Bcfg2.Server.Cache.expire("Packages") for source in self.sources.entries: cachefiles.add(source.cachefile) @@ -551,11 +534,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object - collection = Collection(metadata, [], self.cachepath, self.data) - ckey = collection.cachekey - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) - return collection + return Collection(metadata, [], self.cachepath, self.data) if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] @@ -592,8 +571,6 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) return collection def get_additional_data(self, metadata): @@ -642,8 +619,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ - if metadata.hostname in self.clients: - del self.clients[metadata.hostname] + self.clients.expire(metadata.hostname) def end_statistics(self, metadata): """ Hook to clear the cache for this client in :attr:`clients` diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index f75d88d8f..560546c70 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -8,8 +8,11 @@ import copy import operator import lxml.etree import Bcfg2.Server +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin +from Bcfg2.Compat import unicode # pylint: disable=W0622 import Bcfg2.Server.FileMonitor +from Bcfg2.Logger import Debuggable from Bcfg2.Server.Statistics import track_statistics HAS_DJANGO = False @@ -63,6 +66,215 @@ except ImportError: HAS_YAML = False +class ProbeStore(Debuggable): + """ Caching abstraction layer between persistent probe data + storage and the Probes plugin.""" + + def __init__(self, core, datastore): # pylint: disable=W0613 + Debuggable.__init__(self) + self._groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + self._datacache = Bcfg2.Server.Cache.Cache("Probes", "probedata") + + def get_groups(self, hostname): + """ Get the list of groups for the given host """ + if hostname not in self._groupcache: + self._load_groups(hostname) + return self._groupcache.get(hostname, []) + + def set_groups(self, hostname, groups): + """ Set the list of groups for the given host """ + raise NotImplementedError + + def get_data(self, hostname): + """ Get a dict of probe data for the given host """ + if hostname not in self._datacache: + self._load_data(hostname) + return self._datacache.get(hostname, dict()) + + def set_data(self, hostname, data): + """ Set probe data for the given host """ + raise NotImplementedError + + def _load_groups(self, hostname): + """ When probe groups are not found in the cache, this + function is called to load them from the backend (XML or + database). """ + raise NotImplementedError + + def _load_data(self, hostname): + """ When probe groups are not found in the cache, this + function is called to load them from the backend (XML or + database). """ + raise NotImplementedError + + def commit(self): + """ Commit the current data in the cache to the persistent + backend store. This is not used with the + :class:`Bcfg2.Server.Plugins.Probes.DBProbeStore`, because it + commits on every change. """ + pass + + +class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): + """ Caching abstraction layer between the database and the Probes + plugin. """ + create = False + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) + ProbeStore.__init__(self, core, datastore) + + def _load_groups(self, hostname): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + groupdata = ProbesGroupsModel.objects.filter(hostname=hostname) + self._groupcache[hostname] = list(set(r.group for r in groupdata)) + Bcfg2.Server.Cache.expire("Metadata", hostname) + + @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock + def set_groups(self, hostname, groups): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + olddata = self._groupcache.get(hostname, []) + self._groupcache[hostname] = groups + for group in groups: + try: + ProbesGroupsModel.objects.get_or_create( + hostname=hostname, + group=group) + except ProbesGroupsModel.MultipleObjectsReturned: + ProbesGroupsModel.objects.filter(hostname=hostname, + group=group).delete() + ProbesGroupsModel.objects.get_or_create( + hostname=hostname, + group=group) + ProbesGroupsModel.objects.filter( + hostname=hostname).exclude(group__in=groups).delete() + if olddata != groups: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + def _load_data(self, hostname): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + ts_set = False + for pdata in ProbesDataModel.objects.filter(hostname=hostname): + if not ts_set: + self._datacache[hostname].timestamp = \ + time.mktime(pdata.timestamp.timetuple()) + ts_set = True + self._datacache[hostname][pdata.probe] = ProbeData(pdata.data) + Bcfg2.Server.Cache.expire("Metadata", hostname) + + @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock + def set_data(self, hostname, data): + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + expire_metadata = False + for probe, pdata in data.items(): + self._datacache[hostname][probe] = pdata + record, created = ProbesDataModel.objects.get_or_create( + hostname=hostname, + probe=probe) + expire_metadata |= created + if record.data != pdata: + record.data = pdata + record.save() + expire_metadata = True + qset = ProbesDataModel.objects.filter( + hostname=hostname).exclude(probe__in=data.keys()) + if len(qset): + qset.delete() + expire_metadata = True + if expire_metadata: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + +class XMLProbeStore(ProbeStore): + """ Caching abstraction layer between ``probed.xml`` and the + Probes plugin.""" + def __init__(self, core, datastore): + ProbeStore.__init__(self, core, datastore) + self._fname = os.path.join(datastore, 'probed.xml') + self._load_data() + + def _load_data(self, _=None): + """ Load probe data from probed.xml """ + Bcfg2.Server.Cache.expire("Probes", "probegroups") + Bcfg2.Server.Cache.expire("Probes", "probedata") + if not os.path.exists(self._fname): + self.commit() + try: + data = lxml.etree.parse(self._fname, + parser=Bcfg2.Server.XMLParser).getroot() + except (IOError, lxml.etree.XMLSyntaxError): + err = sys.exc_info()[1] + self.logger.error("Failed to read file probed.xml: %s" % err) + return + for client in data.getchildren(): + self._datacache[client.get('name')] = \ + ClientProbeDataSet(timestamp=client.get("timestamp")) + self._groupcache[client.get('name')] = [] + for pdata in client: + if pdata.tag == 'Probe': + self._datacache[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get("value")) + elif pdata.tag == 'Group': + self._groupcache[client.get('name')].append( + pdata.get('name')) + + Bcfg2.Server.Cache.expire("Metadata") + + def _load_groups(self, hostname): + self._load_data(hostname) + + def commit(self): + """ Write received probe data to probed.xml """ + top = lxml.etree.Element("Probed") + for client, probed in sorted(self._datacache.items()): + # make a copy of probe data for this client in case it + # submits probe data while we're trying to write + # probed.xml + probedata = copy.copy(probed) + ctag = \ + lxml.etree.SubElement(top, 'Client', name=client, + timestamp=str(int(probedata.timestamp))) + for probe in sorted(probedata): + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self._datacache[client][probe].decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self._datacache[client][probe]) + for group in sorted(self._groupcache[client]): + lxml.etree.SubElement(ctag, "Group", name=group) + try: + top.getroottree().write(self._fname, + xml_declaration=False, + pretty_print='true') + except IOError: + err = sys.exc_info()[1] + self.logger.error("Failed to write probed.xml: %s" % err) + + def set_groups(self, hostname, groups): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + olddata = self._groupcache.get(hostname, []) + self._groupcache[hostname] = groups + if olddata != groups: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + def set_data(self, hostname, data): + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + expire_metadata = False + for probe, pdata in data.items(): + olddata = self._datacache[hostname].get(probe, ProbeData('')) + self._datacache[hostname][probe] = pdata + expire_metadata |= olddata != data + if expire_metadata: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + class ClientProbeDataSet(dict): """ dict of probe => [probe data] that records a timestamp for each host """ @@ -79,7 +291,10 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data) + if isinstance(data, unicode): + return str.__new__(cls, data.encode('utf-8')) + else: + return str.__new__(cls, data) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -195,12 +410,13 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' + groupline_re = re.compile(r'^group:\s*(?P<groupname>\S+)\s*') + options = [ Bcfg2.Options.BooleanOption( cf=('probes', 'use_database'), dest="probes_db", @@ -209,7 +425,6 @@ class Probes(Bcfg2.Server.Plugin.Probing, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Probing.__init__(self) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) @@ -219,191 +434,48 @@ class Probes(Bcfg2.Server.Plugin.Probing, err = sys.exc_info()[1] raise Bcfg2.Server.Plugin.PluginInitError(err) - self.probedata = dict() - self.cgroups = dict() - self.load_data() - __init__.__doc__ = Bcfg2.Server.Plugin.DatabaseBacked.__init__.__doc__ - - @track_statistics() - def write_data(self, client): - """ Write probe data out for use with bcfg2-info """ - if self._use_db: - return self._write_data_db(client) - else: - return self._write_data_xml(client) - - def _write_data_xml(self, _): - """ Write received probe data to probed.xml """ - top = lxml.etree.Element("Probed") - for client, probed in sorted(self.probedata.items()): - # make a copy of probe data for this client in case it - # submits probe data while we're trying to write - # probed.xml - probedata = copy.copy(probed) - ctag = \ - lxml.etree.SubElement(top, 'Client', name=client, - timestamp=str(int(probedata.timestamp))) - for probe in sorted(probedata): - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=self.probedata[client][probe]) - for group in sorted(self.cgroups[client]): - lxml.etree.SubElement(ctag, "Group", name=group) - try: - top.getroottree().write(os.path.join(self.data, 'probed.xml'), - xml_declaration=False, - pretty_print='true') - except IOError: - err = sys.exc_info()[1] - self.logger.error("Failed to write probed.xml: %s" % err) - - @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock - def _write_data_db(self, client): - """ Write received probe data to the database """ - for probe, data in self.probedata[client.hostname].items(): - pdata = \ - ProbesDataModel.objects.get_or_create(hostname=client.hostname, - probe=probe)[0] - if pdata.data != data: - pdata.data = data - pdata.save() - - ProbesDataModel.objects.filter( - hostname=client.hostname).exclude( - probe__in=self.probedata[client.hostname]).delete() - - for group in self.cgroups[client.hostname]: - try: - ProbesGroupsModel.objects.get_or_create( - hostname=client.hostname, - group=group) - except ProbesGroupsModel.MultipleObjectsReturned: - ProbesGroupsModel.objects.filter(hostname=client.hostname, - group=group).delete() - ProbesGroupsModel.objects.get_or_create( - hostname=client.hostname, - group=group) - ProbesGroupsModel.objects.filter( - hostname=client.hostname).exclude( - group__in=self.cgroups[client.hostname]).delete() - - def expire_cache(self, key=None): - self.load_data(client=key) - - def load_data(self, client=None): - """ Load probe data from the appropriate backend (probed.xml - or the database) """ if self._use_db: - return self._load_data_db(client=client) - else: - # the XML backend doesn't support loading data for single - # clients, so it reloads all data - return self._load_data_xml() - - def _load_data_xml(self): - """ Load probe data from probed.xml """ - try: - data = lxml.etree.parse(os.path.join(self.data, 'probed.xml'), - parser=Bcfg2.Server.XMLParser).getroot() - except (IOError, lxml.etree.XMLSyntaxError): - err = sys.exc_info()[1] - self.logger.error("Failed to read file probed.xml: %s" % err) - return - self.probedata = {} - self.cgroups = {} - for client in data.getchildren(): - self.probedata[client.get('name')] = \ - ClientProbeDataSet(timestamp=client.get("timestamp")) - self.cgroups[client.get('name')] = [] - for pdata in client: - if pdata.tag == 'Probe': - self.probedata[client.get('name')][pdata.get('name')] = \ - ProbeData(pdata.get("value")) - elif pdata.tag == 'Group': - self.cgroups[client.get('name')].append(pdata.get('name')) - - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) - - def _load_data_db(self, client=None): - """ Load probe data from the database """ - if client is None: - self.probedata = {} - self.cgroups = {} - probedata = ProbesDataModel.objects.all() - groupdata = ProbesGroupsModel.objects.all() + self.probestore = DBProbeStore(core, datastore) else: - self.probedata.pop(client, None) - self.cgroups.pop(client, None) - probedata = ProbesDataModel.objects.filter(hostname=client) - groupdata = ProbesGroupsModel.objects.filter(hostname=client) - - for pdata in probedata: - if pdata.hostname not in self.probedata: - self.probedata[pdata.hostname] = ClientProbeDataSet( - timestamp=time.mktime(pdata.timestamp.timetuple())) - self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) - for pgroup in groupdata: - if pgroup.hostname not in self.cgroups: - self.cgroups[pgroup.hostname] = [] - self.cgroups[pgroup.hostname].append(pgroup.group) - - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, - key=client) + self.probestore = XMLProbeStore(core, datastore) @track_statistics() - def GetProbes(self, meta): - return self.probes.get_probe_data(meta) - GetProbes.__doc__ = Bcfg2.Server.Plugin.Probing.GetProbes.__doc__ + def GetProbes(self, metadata): + return self.probes.get_probe_data(metadata) @track_statistics() def ReceiveData(self, client, datalist): - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - if client.hostname in self.cgroups: - olddata = copy.copy(self.cgroups[client.hostname]) - else: - olddata = [] - - cgroups = [] - cprobedata = ClientProbeDataSet() + cgroups = set() + cdata = dict() for data in datalist: - self.ReceiveDataItem(client, data, cgroups, cprobedata) - self.cgroups[client.hostname] = cgroups - self.probedata[client.hostname] = cprobedata - - if (self.core.metadata_cache_mode in ['cautious', 'aggressive'] and - olddata != self.cgroups[client.hostname]): - self.core.metadata_cache.expire(client.hostname) - self.write_data(client) - ReceiveData.__doc__ = Bcfg2.Server.Plugin.Probing.ReceiveData.__doc__ - - def ReceiveDataItem(self, client, data, cgroups, cprobedata): - """Receive probe results pertaining to client.""" + groups, cdata[data.get("name")] = \ + self.ReceiveDataItem(client, data) + cgroups.update(groups) + self.probestore.set_groups(client.hostname, list(cgroups)) + self.probestore.set_data(client.hostname, cdata) + self.probestore.commit() + + def ReceiveDataItem(self, client, data): + """ Receive probe results pertaining to client. Returns a + tuple of (<probe groups>, <probe data>). """ if data.text is None: self.logger.info("Got null response to probe %s from %s" % (data.get('name'), client.hostname)) - cprobedata[data.get('name')] = ProbeData('') - return + return [], '' dlines = data.text.split('\n') self.logger.debug("Processing probe from %s: %s:%s" % (client.hostname, data.get('name'), [line.strip() for line in dlines])) + groups = [] for line in dlines[:]: - if line.split(':')[0] == 'group': - newgroup = line.split(':')[1].strip() - if newgroup not in cgroups: - cgroups.append(newgroup) + match = self.groupline_re.match(line) + if match: + groups.append(match.group("groupname")) dlines.remove(line) - dobj = ProbeData("\n".join(dlines)) - cprobedata[data.get('name')] = dobj - - def get_additional_groups(self, meta): - return self.cgroups.get(meta.hostname, list()) - get_additional_groups.__doc__ = \ - Bcfg2.Server.Plugin.Connector.get_additional_groups.__doc__ - - def get_additional_data(self, meta): - return self.probedata.get(meta.hostname, ClientProbeDataSet()) - get_additional_data.__doc__ = \ - Bcfg2.Server.Plugin.Connector.get_additional_data.__doc__ + return (groups, ProbeData("\n".join(dlines))) + + def get_additional_groups(self, metadata): + return self.probestore.get_groups(metadata.hostname) + + def get_additional_data(self, metadata): + return self.probestore.get_data(metadata.hostname) diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index 541116db3..b5c60c875 100644 --- a/src/lib/Bcfg2/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -19,9 +19,10 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): self._regex_cache = dict() def HandlesEntry(self, entry, metadata): - if entry.tag in self.Entries: - return self._matches(entry, metadata, - self.Entries[entry.tag].keys()) + for src in self.entries.values(): + for candidate in src.XMLMatch(metadata).xpath("//%s" % entry.tag): + if self._matches(entry, metadata, candidate): + return True return False HandleEntry = Bcfg2.Server.Plugin.PrioDir.BindEntry diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 8ce4e8a54..f3f711b77 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -5,7 +5,6 @@ import os import sys import socket import shutil -import logging import tempfile import Bcfg2.Options import Bcfg2.Server.Plugin @@ -14,16 +13,10 @@ from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Compat import any, u_str, b64encode # pylint: disable=W0622 -LOGGER = logging.getLogger(__name__) - class KeyData(Bcfg2.Server.Plugin.SpecificData): """ class to handle key data for HostKeyEntrySet """ - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific) - self.encoding = encoding - def __lt__(self, other): return self.name < other.name @@ -40,19 +33,20 @@ class KeyData(Bcfg2.Server.Plugin.SpecificData): entry.text = b64encode(self.data) else: try: - entry.text = u_str(self.data, self.encoding) + entry.text = u_str(self.data, Bcfg2.Options.setup.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) - LOGGER.error(msg) - LOGGER.error("Please verify you are using the proper encoding") + self.logger.error(msg) + self.logger.error("Please verify you are using the proper " + "encoding") raise Bcfg2.Server.Plugin.PluginExecutionError(msg) except ValueError: msg = "Error in specification for %s: %s" % (entry.get('name'), sys.exc_info()[1]) - LOGGER.error(msg) - LOGGER.error("You need to specify base64 encoding for %s" % - entry.get('name')) + self.logger.error(msg) + self.logger.error("You need to specify base64 encoding for %s" + % entry.get('name')) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) if entry.text in ['', None]: entry.set('empty', 'true') @@ -61,16 +55,12 @@ class KeyData(Bcfg2.Server.Plugin.SpecificData): class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet): """ EntrySet to handle all kinds of host keys """ def __init__(self, basename, path): - if basename.startswith("ssh_host_key"): - self.encoding = "base64" - else: - self.encoding = None Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData) self.metadata = {'owner': 'root', 'group': 'root', 'type': 'file'} - if self.encoding is not None: - self.metadata['encoding'] = self.encoding + if basename.startswith("ssh_host_key"): + self.metadata['encoding'] = "base64" if basename.endswith('.pub'): self.metadata['mode'] = '0644' else: @@ -89,7 +79,6 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): class SSHbase(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.PullTarget): """ @@ -123,7 +112,6 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.PullTarget.__init__(self) self.ipcache = {} @@ -150,9 +138,6 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk self.cmd = Executor() - def expire_cache(self, key=None): - self.__skn = False - def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py deleted file mode 100644 index 74d8833f4..000000000 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ /dev/null @@ -1,387 +0,0 @@ -""" The SSLCA generator handles the creation and management of ssl -certificates and their keys. """ - -import os -import sys -import tempfile -import lxml.etree -import Bcfg2.Server.Plugin -from Bcfg2.Utils import Executor -from Bcfg2.Compat import ConfigParser -from Bcfg2.Server.Plugin import PluginExecutionError - - -class SSLCAXMLSpec(Bcfg2.Server.Plugin.StructFile): - """ Base class to handle key.xml and cert.xml """ - encryption = False - attrs = dict() - tag = None - - def get_spec(self, metadata): - """ Get a specification for the type of object described by - this SSLCA XML file for the given client metadata object """ - entries = [e for e in self.Match(metadata) if e.tag == self.tag] - if len(entries) == 0: - raise PluginExecutionError("No matching %s entry found for %s " - "in %s" % (self.tag, - metadata.hostname, - self.name)) - elif len(entries) > 1: - self.logger.warning( - "More than one matching %s entry found for %s in %s; " - "using first match" % (self.tag, metadata.hostname, self.name)) - rv = dict() - for attr, default in self.attrs.items(): - val = entries[0].get(attr.lower(), default) - if default in ['true', 'false']: - rv[attr] = val == 'true' - else: - rv[attr] = val - return rv - - -class SSLCAKeySpec(SSLCAXMLSpec): - """ Handle key.xml files """ - attrs = dict(bits='2048', type='rsa') - tag = 'Key' - - -class SSLCACertSpec(SSLCAXMLSpec): - """ Handle cert.xml files """ - attrs = dict(ca='default', - format='pem', - key=None, - days='365', - C=None, - L=None, - ST=None, - OU=None, - O=None, - emailAddress=None, - append_chain='false') - tag = 'Cert' - - def get_spec(self, metadata): - rv = SSLCAXMLSpec.get_spec(self, metadata) - rv['subjectaltname'] = [e.text for e in self.Match(metadata) - if e.tag == "subjectAltName"] - return rv - - -class SSLCADataFile(Bcfg2.Server.Plugin.SpecificData): - """ Handle key and cert files """ - def bind_entry(self, entry, _): - """ Bind the data in the file to the given abstract entry """ - entry.text = self.data - entry.set("type", "file") - return entry - - -class SSLCAEntrySet(Bcfg2.Server.Plugin.EntrySet): - """ Entry set to handle SSLCA entries and XML files """ - def __init__(self, _, path, entry_type, parent=None): - Bcfg2.Server.Plugin.EntrySet.__init__(self, os.path.basename(path), - path, entry_type) - self.parent = parent - self.key = None - self.cert = None - self.cmd = Executor(timeout=120) - - def handle_event(self, event): - action = event.code2str() - fpath = os.path.join(self.path, event.filename) - - if event.filename == 'key.xml': - if action in ['exists', 'created', 'changed']: - self.key = SSLCAKeySpec(fpath) - self.key.HandleEvent(event) - elif event.filename == 'cert.xml': - if action in ['exists', 'created', 'changed']: - self.cert = SSLCACertSpec(fpath) - self.cert.HandleEvent(event) - else: - Bcfg2.Server.Plugin.EntrySet.handle_event(self, event) - - def build_key(self, entry, metadata): - """ - either grabs a prexisting key hostfile, or triggers the generation - of a new key if one doesn't exist. - """ - # TODO: verify key fits the specs - filename = "%s.H_%s" % (os.path.basename(entry.get('name')), - metadata.hostname) - self.logger.info("SSLCA: Generating new key %s" % filename) - key_spec = self.key.get_spec(metadata) - ktype = key_spec['type'] - bits = key_spec['bits'] - if ktype == 'rsa': - cmd = ["openssl", "genrsa", bits] - elif ktype == 'dsa': - cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] - self.debug_log("SSLCA: Generating new key: %s" % " ".join(cmd)) - result = self.cmd.run(cmd) - if not result.success: - raise PluginExecutionError("SSLCA: Failed to generate key %s for " - "%s: %s" % (entry.get("name"), - metadata.hostname, - result.error)) - open(os.path.join(self.path, filename), 'w').write(result.stdout) - return result.stdout - - def build_cert(self, entry, metadata, keyfile): - """ generate a new cert """ - filename = "%s.H_%s" % (os.path.basename(entry.get('name')), - metadata.hostname) - self.logger.info("SSLCA: Generating new cert %s" % filename) - cert_spec = self.cert.get_spec(metadata) - ca = self.parent.get_ca(cert_spec['ca']) - req_config = None - req = None - try: - req_config = self.build_req_config(metadata) - req = self.build_request(keyfile, req_config, metadata) - days = cert_spec['days'] - cmd = ["openssl", "ca", "-config", ca['config'], "-in", req, - "-days", days, "-batch"] - passphrase = ca.get('passphrase') - if passphrase: - cmd.extend(["-passin", "pass:%s" % passphrase]) - - def _scrub_pass(arg): - """ helper to scrub the passphrase from the - argument list """ - if arg.startswith("pass:"): - return "pass:******" - else: - return arg - else: - _scrub_pass = lambda a: a - - self.debug_log("SSLCA: Generating new certificate: %s" % - " ".join(_scrub_pass(a) for a in cmd)) - result = self.cmd.run(cmd) - if not result.success: - raise PluginExecutionError("SSLCA: Failed to generate cert: %s" - % result.error) - finally: - try: - if req_config and os.path.exists(req_config): - os.unlink(req_config) - if req and os.path.exists(req): - os.unlink(req) - except OSError: - self.logger.error("SSLCA: Failed to unlink temporary files: %s" - % sys.exc_info()[1]) - cert = result.stdout - if cert_spec['append_chain'] and 'chaincert' in ca: - cert += open(ca['chaincert']).read() - - open(os.path.join(self.path, filename), 'w').write(cert) - return cert - - def build_req_config(self, metadata): - """ - generates a temporary openssl configuration file that is - used to generate the required certificate request - """ - # create temp request config file - fd, fname = tempfile.mkstemp() - cfp = ConfigParser.ConfigParser({}) - cfp.optionxform = str - defaults = { - 'req': { - 'default_md': 'sha1', - 'distinguished_name': 'req_distinguished_name', - 'req_extensions': 'v3_req', - 'x509_extensions': 'v3_req', - 'prompt': 'no' - }, - 'req_distinguished_name': {}, - 'v3_req': { - 'subjectAltName': '@alt_names' - }, - 'alt_names': {} - } - for section in list(defaults.keys()): - cfp.add_section(section) - for key in defaults[section]: - cfp.set(section, key, defaults[section][key]) - cert_spec = self.cert.get_spec(metadata) - altnamenum = 1 - altnames = cert_spec['subjectaltname'] - altnames.extend(list(metadata.aliases)) - altnames.append(metadata.hostname) - for altname in altnames: - cfp.set('alt_names', 'DNS.' + str(altnamenum), altname) - altnamenum += 1 - for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: - if cert_spec[item]: - cfp.set('req_distinguished_name', item, cert_spec[item]) - cfp.set('req_distinguished_name', 'CN', metadata.hostname) - self.debug_log("SSLCA: Writing temporary request config to %s" % fname) - try: - cfp.write(os.fdopen(fd, 'w')) - except IOError: - raise PluginExecutionError("SSLCA: Failed to write temporary CSR " - "config file: %s" % sys.exc_info()[1]) - return fname - - def build_request(self, keyfile, req_config, metadata): - """ - creates the certificate request - """ - fd, req = tempfile.mkstemp() - os.close(fd) - days = self.cert.get_spec(metadata)['days'] - cmd = ["openssl", "req", "-new", "-config", req_config, - "-days", days, "-key", keyfile, "-text", "-out", req] - self.debug_log("SSLCA: Generating new CSR: %s" % " ".join(cmd)) - result = self.cmd.run(cmd) - if not result.success: - raise PluginExecutionError("SSLCA: Failed to generate CSR: %s" % - result.error) - return req - - def verify_cert(self, filename, keyfile, entry, metadata): - """ Perform certification verification against the CA and - against the key """ - ca = self.parent.get_ca(self.cert.get_spec(metadata)['ca']) - do_verify = ca.get('chaincert') - if do_verify: - return (self.verify_cert_against_ca(filename, entry, metadata) and - self.verify_cert_against_key(filename, keyfile)) - return True - - def verify_cert_against_ca(self, filename, entry, metadata): - """ - check that a certificate validates against the ca cert, - and that it has not expired. - """ - ca = self.parent.get_ca(self.cert.get_spec(metadata)['ca']) - chaincert = ca.get('chaincert') - cert = os.path.join(self.path, filename) - cmd = ["openssl", "verify"] - is_root = ca.get('root_ca', "false").lower() == 'true' - if is_root: - cmd.append("-CAfile") - else: - # verifying based on an intermediate cert - cmd.extend(["-purpose", "sslserver", "-untrusted"]) - cmd.extend([chaincert, cert]) - self.debug_log("SSLCA: Verifying %s against CA: %s" % - (entry.get("name"), " ".join(cmd))) - result = self.cmd.run(cmd) - if result.stdout == cert + ": OK\n": - self.debug_log("SSLCA: %s verified successfully against CA" % - entry.get("name")) - return True - self.logger.warning("SSLCA: %s failed verification against CA: %s" % - (entry.get("name"), result.error)) - return False - - def _get_modulus(self, fname, ftype="x509"): - """ get the modulus from the given file """ - cmd = ["openssl", ftype, "-noout", "-modulus", "-in", fname] - self.debug_log("SSLCA: Getting modulus of %s for verification: %s" % - (fname, " ".join(cmd))) - result = self.cmd.run(cmd) - if not result.success: - self.logger.warning("SSLCA: Failed to get modulus of %s: %s" % - (fname, result.error)) - return result.stdout.strip() - - def verify_cert_against_key(self, filename, keyfile): - """ - check that a certificate validates against its private key. - """ - - certfile = os.path.join(self.path, filename) - cert = self._get_modulus(certfile) - key = self._get_modulus(keyfile, ftype="rsa") - if cert == key: - self.debug_log("SSLCA: %s verified successfully against key %s" % - (filename, keyfile)) - return True - self.logger.warning("SSLCA: %s failed verification against key %s" % - (filename, keyfile)) - return False - - def bind_entry(self, entry, metadata): - if self.key: - self.bind_info_to_entry(entry, metadata) - try: - return self.best_matching(metadata).bind_entry(entry, metadata) - except PluginExecutionError: - entry.text = self.build_key(entry, metadata) - entry.set("type", "file") - return entry - elif self.cert: - key = self.cert.get_spec(metadata)['key'] - cleanup_keyfile = False - try: - keyfile = self.parent.entries[key].best_matching(metadata).name - except PluginExecutionError: - cleanup_keyfile = True - # create a temp file with the key in it - fd, keyfile = tempfile.mkstemp() - os.chmod(keyfile, 384) # 0600 - el = lxml.etree.Element('Path', name=key) - self.parent.core.Bind(el, metadata) - os.fdopen(fd, 'w').write(el.text) - - try: - self.bind_info_to_entry(entry, metadata) - try: - best = self.best_matching(metadata) - if self.verify_cert(best.name, keyfile, entry, metadata): - return best.bind_entry(entry, metadata) - except PluginExecutionError: - pass - # if we get here, it's because either a) there was no best - # matching entry; or b) the existing cert did not verify - entry.text = self.build_cert(entry, metadata, keyfile) - entry.set("type", "file") - return entry - finally: - if cleanup_keyfile: - try: - os.unlink(keyfile) - except OSError: - err = sys.exc_info()[1] - self.logger.error("SSLCA: Failed to unlink temporary " - "key %s: %s" % (keyfile, err)) - - -class SSLCA(Bcfg2.Server.Plugin.GroupSpool): - """ The SSLCA generator handles the creation and management of ssl - certificates and their keys. """ - __author__ = 'g.hagger@gmail.com' - - options = Bcfg2.Server.Plugin.GroupSpool.options + [ - Bcfg2.Options.WildcardSectionGroup( - Bcfg2.Options.PathOption( - cf=("sslca_*", "config"), - help="Path to the openssl config for the CA"), - Bcfg2.Options.Option( - cf=("sslca_*", "passphrase"), - help="Passphrase for the CA private key"), - Bcfg2.Options.PathOption( - cf=("sslca_*", "chaincert"), - help="Path to the SSL chaining certificate for verification"), - Bcfg2.Options.BooleanOption( - cf=("sslca_*", "root_ca"), - help="Whether or not <chaincert> is a root CA (as opposed to " - "an intermediate cert"))] - - # python 2.5 doesn't support mixing *magic and keyword arguments - es_cls = lambda self, *args: SSLCAEntrySet(*args, **dict(parent=self)) - es_child_cls = SSLCADataFile - - def get_ca(self, name): - """ get a dict describing a CA from the config file """ - rv = dict() - prefix = "sslca_%s_" % name - for attr in dir(Bcfg2.Options.setup): - if attr.startswith(prefix): - rv[attr[len(prefix):]] = getattr(Bcfg2.Options.setup, attr) - return rv diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 646124fcc..6a3948f40 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -5,7 +5,6 @@ better. """ import os import sys import socket -import select import signal import logging import ssl @@ -237,22 +236,22 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): return False return True - ### need to override do_POST here def do_POST(self): try: max_chunk_size = 10 * 1024 * 1024 size_remaining = int(self.headers["content-length"]) L = [] while size_remaining: - try: - select.select([self.rfile.fileno()], [], [], 3) - except select.error: - self.logger.error("Got select timeout") - raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size).decode('utf-8')) + chunk = self.rfile.read(chunk_size).decode('utf-8') + if not chunk: + break + L.append(chunk) size_remaining -= len(L[-1]) data = ''.join(L) + if data is None: + return # response has been sent + response = self.server._marshaled_dispatch(self.client_address, data) if sys.hexversion >= 0x03000000: @@ -265,6 +264,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): (self.client_address, sys.exc_info()[1])) try: self.send_response(500) + self.send_header("Content-length", "0") self.end_headers() except: (etype, msg) = sys.exc_info()[:2] @@ -321,14 +321,11 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): def finish(self): # shut down the connection - if not self.wfile.closed: - try: - self.wfile.flush() - self.wfile.close() - except socket.error: - err = sys.exc_info()[1] - self.logger.warning("Error closing connection: %s" % err) - self.rfile.close() + try: + SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.finish(self) + except socket.error: + err = sys.exc_info()[1] + self.logger.warning("Error closing connection: %s" % err) class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, @@ -446,8 +443,6 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, self.handle_request() except socket.timeout: pass - except select.error: - pass except: self.logger.error("Got unexpected error in handle_request", exc_info=1) diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 236f87d0a..10057b63e 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -2,12 +2,13 @@ used by both client and server. Stuff that doesn't fit anywhere else. """ +import fcntl +import logging import os import re -import sys -import fcntl import select -import logging +import shlex +import sys import subprocess import threading from Bcfg2.Compat import input, any # pylint: disable=W0622 @@ -216,12 +217,17 @@ class Executor(object): :type timeout: float :returns: :class:`Bcfg2.Utils.ExecutorResult` """ + shell = False + if 'shell' in kwargs: + shell = kwargs['shell'] if isinstance(command, str): cmdstr = command + if not shell: + command = shlex.split(cmdstr) else: cmdstr = " ".join(command) self.logger.debug("Running: %s" % cmdstr) - args = dict(shell=False, bufsize=16384, close_fds=True) + args = dict(shell=shell, bufsize=16384, close_fds=True) args.update(kwargs) args.update(stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 42d415232..7ddf58aed 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -165,7 +165,8 @@ class _OptionContainer(object): dest='db_schema'), Bcfg2.Options.Option( cf=('database', 'options'), help='Database options', - dest='db_opts', type=Bcfg2.Options.Types.comma_dict), + dest='db_opts', type=Bcfg2.Options.Types.comma_dict, + default=dict()), Bcfg2.Options.Option( cf=('reporting', 'timezone'), help='Django timezone'), Bcfg2.Options.BooleanOption( |