diff options
Diffstat (limited to 'src/lib/Bcfg2/Server')
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Init.py | 27 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Core.py | 133 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Inotify.py | 23 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/__init__.py | 14 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin/base.py | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin/helpers.py | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin/interfaces.py | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/DBStats.py | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 119 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/POSIXCompat.py | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 85 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 23 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Reporting.py | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/SSLServer.py | 9 |
16 files changed, 387 insertions, 74 deletions
diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 724da124b..884405786 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -12,6 +12,7 @@ from Bcfg2.Utils import Executor import Bcfg2.Server.Admin import Bcfg2.Server.Plugin import Bcfg2.Options +import Bcfg2.Server.Plugins.Metadata from Bcfg2.Compat import input # pylint: disable=W0622 # default config file @@ -171,8 +172,6 @@ class Init(Bcfg2.Server.Admin.Mode): self.data['certpath'] = os.path.join(basepath, 'bcfg2.crt') def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - # Parse options setup = Bcfg2.Options.get_option_parser() setup.add_options(dict(configfile=Bcfg2.Options.CFILE, @@ -218,7 +217,7 @@ class Init(Bcfg2.Server.Admin.Mode): """Ask for the repository path.""" while True: newrepo = safe_input("Location of Bcfg2 repository [%s]: " % - self.data['repopath']) + self.data['repopath']) if newrepo != '': self.data['repopath'] = os.path.abspath(newrepo) if os.path.isdir(self.data['repopath']): @@ -296,7 +295,7 @@ class Init(Bcfg2.Server.Admin.Mode): "created [%s]: " % self.data['keypath']) if keypath: self.data['keypath'] = keypath - certpath = safe_input("Path where Bcfg2 server cert will be created" + certpath = safe_input("Path where Bcfg2 server cert will be created " "[%s]: " % self.data['certpath']) if certpath: self.data['certpath'] = certpath @@ -324,6 +323,16 @@ class Init(Bcfg2.Server.Admin.Mode): def init_repo(self): """Setup a new repo and create the content of the configuration file.""" + # Create the repository + path = os.path.join(self.data['repopath'], 'etc') + try: + os.makedirs(path) + self._init_plugins() + print("Repository created successfuly in %s" % + self.data['repopath']) + except OSError: + print("Failed to create %s." % path) + confdata = CONFIG % (self.data['repopath'], ','.join(self.plugins), self.data['sendmail'], @@ -339,13 +348,3 @@ class Init(Bcfg2.Server.Admin.Mode): create_key(self.data['shostname'], self.data['keypath'], self.data['certpath'], self.data['country'], self.data['state'], self.data['location']) - - # Create the repository - path = os.path.join(self.data['repopath'], 'etc') - try: - os.makedirs(path) - self._init_plugins() - print("Repository created successfuly in %s" % - self.data['repopath']) - except OSError: - print("Failed to create %s." % path) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 07f9e0588..c69e8b055 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -4,8 +4,9 @@ implementations inherit from. """ import os import sys import time -import select import atexit +import select +import signal import logging import inspect import threading @@ -119,13 +120,36 @@ class BaseCore(object): #: A :class:`logging.Logger` object for use by the core self.logger = logging.getLogger('bcfg2-server') + #: Log levels for the various logging handlers with debug True + #: and False. Each loglevel dict is a dict of ``logger name + #: => log level``; the logger names are set in + #: :mod:`Bcfg2.Logger`. The logger name ``default`` is + #: special, and will be used for any log handlers whose name + #: does not appear elsewhere in the dict. At a minimum, + #: ``default`` must be provided. + self._loglevels = {True: dict(default=logging.DEBUG), + False: dict(console=logging.INFO, + default=level)} + + #: Used to keep track of the current debug state of the core. + self.debug_flag = False + + # enable debugging on the core now. debugging is enabled on + # everything else later + if self.setup['debug']: + self.set_core_debug(None, setup['debug']) + if 'ignore' not in self.setup: self.setup.add_option('ignore', SERVER_FAM_IGNORE) self.setup.reparse() + famargs = dict(filemonitor=self.setup['filemonitor'], debug=self.setup['debug'], ignore=self.setup['ignore']) - if self.setup['filemonitor'] not in Bcfg2.Server.FileMonitor.available: + try: + filemonitor = \ + Bcfg2.Server.FileMonitor.available[setup['filemonitor']] + except KeyError: self.logger.error("File monitor driver %s not available; " "forcing to default" % self.setup['filemonitor']) famargs['filemonitor'] = 'default' @@ -281,6 +305,14 @@ class BaseCore(object): #: The CA that signed the server cert self.ca = self.setup['ca'] + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + self.shutdown() + os._exit(1) # pylint: disable=W0212 + + signal.signal(signal.SIGINT, hdlr) + #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ @@ -295,6 +327,10 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() + if self.debug_flag: + # enable debugging on everything else. + self.plugins[plugin].set_debug(self.debug_flag) + def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -392,6 +428,7 @@ class BaseCore(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ + self.logger.debug("Shutting down core...") if not self.terminate.isSet(): self.terminate.set() self.fam.shutdown() @@ -427,6 +464,8 @@ class BaseCore(object): hook. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ + self.logger.debug("Running %s hooks for %s" % (hook, + metadata.hostname)) start = time.time() try: for plugin in \ @@ -460,6 +499,7 @@ class BaseCore(object): client :type data: list of lxml.etree._Element objects """ + self.logger.debug("Validating structures for %s" % metadata.hostname) for plugin in \ self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): try: @@ -486,6 +526,7 @@ class BaseCore(object): client :type data: list of lxml.etree._Element objects """ + self.logger.debug("Validating goals for %s" % metadata.hostname) for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): try: plugin.validate_goals(metadata, data) @@ -506,6 +547,7 @@ class BaseCore(object): :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: list of :class:`lxml.etree._Element` objects """ + self.logger.debug("Getting structures for %s" % metadata.hostname) structures = list(chain(*[struct.BuildStructures(metadata) for struct in self.structures])) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] @@ -528,6 +570,7 @@ class BaseCore(object): structures to. Modified in-place. :type config: lxml.etree._Element """ + self.logger.debug("Binding structures for %s" % metadata.hostname) for astruct in structures: try: self.BindStructure(astruct, metadata) @@ -544,6 +587,9 @@ class BaseCore(object): :param metadata: Client metadata to bind structure for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ + self.logger.debug("Binding structure %s for %s" % + (structure.get("name", "unknown"), + metadata.hostname)) for entry in structure.getchildren(): if entry.tag.startswith("Bound"): entry.tag = entry.tag[5:] @@ -619,6 +665,7 @@ class BaseCore(object): :type client: string :returns: :class:`lxml.etree._Element` - A complete Bcfg2 configuration document """ + self.logger.debug("Building configuration for %s" % client) start = time.time() config = lxml.etree.Element("Configuration", version='2.0', revision=self.revision) @@ -718,6 +765,12 @@ class BaseCore(object): self.shutdown() raise + if self.setup['fam_blocking']: + time.sleep(1) + while self.fam.pending() != 0: + time.sleep(1) + + self.set_debug(None, self.debug_flag) self._block() def _daemonize(self): @@ -746,6 +799,7 @@ class BaseCore(object): :type mode: string :returns: list of Decision tuples ``(<entry tag>, <entry name>)`` """ + self.logger.debug("Getting decision list for %s" % metadata.hostname) result = [] for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): try: @@ -816,6 +870,7 @@ class BaseCore(object): else: imd = self.metadata_cache.get(client_name, None) if not imd: + self.logger.debug("Building metadata for %s" % client_name) imd = self.metadata.get_initial_metadata(client_name) for conn in self.connectors: grps = conn.get_additional_groups(imd) @@ -837,6 +892,7 @@ class BaseCore(object): :param statistics: The statistics document to process :type statistics: lxml.etree._Element """ + self.logger.debug("Processing statistics for %s" % client_name) meta = self.build_metadata(client_name) state = statistics.find(".//Statistics") if state.get('version') >= '2.0': @@ -907,10 +963,12 @@ class BaseCore(object): def _get_rmi(self): """ Get a list of RMI calls exposed by plugins """ rmi = dict() - if self.plugins: - for pname, pinst in list(self.plugins.items()): - for mname in pinst.__rmi__: - rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) + for pname, pinst in list(self.plugins.items()): + for mname in pinst.__rmi__: + rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) + famname = self.fam.__class__.__name__ + for mname in self.fam.__rmi__: + rmi["%s.%s" % (famname, mname)] = getattr(self.fam, mname) return rmi def _resolve_exposed_method(self, method_name): @@ -964,6 +1022,7 @@ class BaseCore(object): return func.__doc__ @exposed + @track_statistics() def DeclareVersion(self, address, version): """ Declare the client version. @@ -974,7 +1033,9 @@ class BaseCore(object): :returns: bool - True on success :raises: :exc:`xmlrpclib.Fault` """ - client = self.resolve_client(address)[0] + client = self.resolve_client(address, metadata=False)[0] + self.logger.debug("%s is running Bcfg2 client version %s" % (client, + version)) try: self.metadata.set_version(client, version) except (Bcfg2.Server.Plugin.MetadataConsistencyError, @@ -996,6 +1057,7 @@ class BaseCore(object): """ resp = lxml.etree.Element('probes') client, metadata = self.resolve_client(address, cleanup_cache=True) + self.logger.debug("Getting probes for %s" % client) try: for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): for probe in plugin.GetProbes(metadata): @@ -1017,6 +1079,7 @@ class BaseCore(object): :raises: :exc:`xmlrpclib.Fault` """ client, metadata = self.resolve_client(address) + self.logger.debug("Receiving probe data from %s" % client) if self.metadata_cache_mode == 'cautious': # clear the metadata cache right after building the # metadata object; that way the cache is cleared for any @@ -1063,6 +1126,7 @@ class BaseCore(object): :raises: :exc:`xmlrpclib.Fault` """ client = self.resolve_client(address, metadata=False)[0] + self.logger.debug("%s sets its profile to %s" % (client, profile)) try: self.metadata.set_profile(client, profile, address) except (Bcfg2.Server.Plugin.MetadataConsistencyError, @@ -1171,22 +1235,35 @@ class BaseCore(object): :type address: tuple :returns: bool - The new debug state of the FAM """ - for plugin in self.plugins.values(): - plugin.toggle_debug() - return self.toggle_fam_debug(address) + return self.set_debug(address, not self.debug_flag) @exposed - def toggle_fam_debug(self, _): + def toggle_core_debug(self, address): + """ Toggle debug status of the server core + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: bool - The new debug state of the FAM + """ + return self.set_core_debug(address, not self.debug_flag) + + @exposed + def toggle_fam_debug(self, address): """ Toggle debug status of the FAM :returns: bool - The new debug state of the FAM """ - return self.fam.toggle_debug() + self.logger.warning("Deprecated method set_fam_debug called by %s" % + address[0]) + return "This method is deprecated and will be removed in a future " + \ + "release\n%s" % self.fam.toggle_debug() @exposed def set_debug(self, address, debug): """ Explicitly set debug status of the FAM and all plugins + :param address: Client (address, hostname) pair + :type address: tuple :param debug: The new debug status. This can either be a boolean, or a string describing the state (e.g., "true" or "false"; case-insensitive) @@ -1197,10 +1274,33 @@ class BaseCore(object): debug = debug.lower() == "true" for plugin in self.plugins.values(): plugin.set_debug(debug) - return self.set_fam_debug(address, debug) + rv = self.set_core_debug(address, debug) + return self.fam.set_debug(debug) and rv + + @exposed + def set_core_debug(self, _, debug): + """ Explicity set debug status of the server core + + :param debug: The new debug status. This can either be a + boolean, or a string describing the state (e.g., + "true" or "false"; case-insensitive) + :type debug: bool or string + :returns: bool - The new debug state of the FAM + """ + if debug not in [True, False]: + debug = debug.lower() == "true" + self.debug_flag = debug + self.logger.info("Core: debug = %s" % debug) + levels = self._loglevels[self.debug_flag] + for handler in logging.root.handlers: + level = levels.get(handler.name, levels['default']) + self.logger.debug("Setting %s log handler to %s" % + (handler.name, logging.getLevelName(level))) + handler.setLevel(level) + return self.debug_flag @exposed - def set_fam_debug(self, _, debug): + def set_fam_debug(self, address, debug): """ Explicitly set debug status of the FAM :param debug: The new debug status of the FAM. This can @@ -1212,4 +1312,7 @@ class BaseCore(object): """ if debug not in [True, False]: debug = debug.lower() == "true" - return self.fam.set_debug(debug) + self.logger.warning("Deprecated method set_fam_debug called by %s" % + address[0]) + return "This method is deprecated and will be removed in a future " + \ + "release\n%s" % self.fam.set_debug(debug) diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index 178a47b1a..cdd52dbb9 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -2,6 +2,7 @@ support. """ import os +import errno import logging import pyinotify from Bcfg2.Compat import reduce # pylint: disable=W0622 @@ -15,6 +16,8 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): """ File monitor backend with `inotify <http://inotify.aiken.cz/>`_ support. """ + __rmi__ = Pseudo.__rmi__ + ["list_watches", "list_paths"] + #: Inotify is the best FAM backend, so it gets a very high #: priority __priority__ = 99 @@ -182,6 +185,9 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): try: watchdir = self.watches_by_path[watch_path] except KeyError: + if not os.path.exists(watch_path): + raise OSError(errno.ENOENT, + "No such file or directory: '%s'" % path) watchdir = self.watchmgr.add_watch(watch_path, self.mask, quiet=False)[watch_path] self.watches_by_path[watch_path] = watchdir @@ -211,3 +217,20 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): if self.notifier: self.notifier.stop() shutdown.__doc__ = Pseudo.shutdown.__doc__ + + def list_watches(self): + """ XML-RPC that returns a list of current inotify watches for + debugging purposes. """ + return list(self.watches_by_path.keys()) + + def list_paths(self): + """ XML-RPC that returns a list of paths that are handled for + debugging purposes. Because inotify doesn't like watching + files, but prefers to watch directories, this will be + different from + :func:`Bcfg2.Server.FileMonitor.Inotify.Inotify.ListWatches`. For + instance, if a plugin adds a monitor to + ``/var/lib/bcfg2/Plugin/foo.xml``, :func:`ListPaths` will + return ``/var/lib/bcfg2/Plugin/foo.xml``, while + :func:`ListWatches` will return ``/var/lib/bcfg2/Plugin``. """ + return list(self.handles.keys()) diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index d77f21b93..522ddb705 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -116,6 +116,9 @@ class FileMonitor(Debuggable): #: should have higher priorities. __priority__ = -1 + #: List of names of methods to be exposed as XML-RPC functions + __rmi__ = Debuggable.__rmi__ + ["list_event_handlers"] + def __init__(self, ignore=None, debug=False): """ :param ignore: A list of filename globs describing events that @@ -288,6 +291,8 @@ class FileMonitor(Debuggable): def shutdown(self): """ Handle any tasks required to shut down the monitor. """ + self.debug_log("Shutting down %s file monitor" % + self.__class__.__name__) self.started = False def AddMonitor(self, path, obj, handleID=None): @@ -310,6 +315,15 @@ class FileMonitor(Debuggable): """ raise NotImplementedError + def list_event_handlers(self): + """ XML-RPC that returns + :attr:`Bcfg2.Server.FileMonitor.FileMonitor.handles` for + debugging purposes. """ + rv = dict() + for watch, handler in self.handles.items(): + rv[watch] = getattr(handler, "name", handler.__class__.__name__) + return rv + #: A module-level FAM object that all plugins, etc., can use. This #: should not be used directly, but retrieved via :func:`get_fam`. diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 60525d5a1..497e8fac6 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -37,7 +37,7 @@ def is_octal_mode(val): def is_username(val): """ Return True if val is a string giving either a positive integer uid, or a valid Unix username """ - return re.match(r'^([a-z]\w{0,30}|\d+)$', val) + return re.match(r'^([A-z][-_A-z0-9]{0,30}|\d+)$', val) def is_device_mode(val): diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index 25a687874..f7bc08717 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -34,8 +34,8 @@ class Debuggable(object): :returns: bool - The new value of the debug flag """ self.debug_flag = debug - self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__, - self.debug_flag), + self.debug_log("%s: debug = %s" % (self.__class__.__name__, + self.debug_flag), flag=True) return debug @@ -122,6 +122,7 @@ class Plugin(Debuggable): """ Perform shutdown tasks for the plugin :returns: None """ + self.debug_log("Shutting down %s plugin" % self.name) self.running = False def __str__(self): diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 187c594fd..ded7dd8dc 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -478,6 +478,7 @@ class XMLFileBacked(FileBacked): def Index(self): self.xdata = lxml.etree.XML(self.data, base_url=self.name, parser=Bcfg2.Server.XMLParser) + self.extras = [] self._follow_xincludes() if self.extras: try: diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 3ef29775d..11a61ff9c 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -313,6 +313,7 @@ class Threaded(object): """ raise NotImplementedError + class ThreadedStatistics(Statistics, Threaded, threading.Thread): """ ThreadedStatistics plugins process client statistics in a separate thread. """ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index 5f10879be..b3781e299 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -77,6 +77,10 @@ class CfgGenshiGenerator(CfgGenerator): __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): + if self.template is None: + raise PluginExecutionError("Failed to load template %s" % + self.name) + fname = entry.get('realname', entry.get('name')) stream = self.template.generate( name=fname, diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index e0794f019..e6ef50fa1 100644 --- a/src/lib/Bcfg2/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -9,7 +9,6 @@ class DBStats(Bcfg2.Server.Plugin.Plugin): def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) self.logger.error("DBStats has been replaced with Reporting") - self.logger.error("DBStats: Be sure to migrate your data "\ - "before running the report collector") + self.logger.error("DBStats: Be sure to migrate your data " + "before running the report collector") raise Bcfg2.Server.Plugin.PluginInitError - diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index b053e65d3..7f8db7b6d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -138,7 +138,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): self.logger.error('Failed to parse %s' % self.basefile) return self.extras = [] - self.basedata = copy.copy(xdata) + self.basedata = copy.deepcopy(xdata) self._follow_xincludes(xdata=xdata) if self.extras: try: @@ -263,31 +263,63 @@ class ClientMetadata(object): # pylint: disable=R0913 def __init__(self, client, profile, groups, bundles, aliases, addresses, categories, uuid, password, version, query): + #: The client hostname (as a string) self.hostname = client + + #: The client profile (as a string) self.profile = profile + + #: The set of all bundles this client gets self.bundles = bundles + + #: A list of all client aliases self.aliases = aliases + + #: A list of all addresses this client is known by self.addresses = addresses + + #: A list of groups this client is a member of self.groups = groups + + #: A dict of categories of this client's groups. Keys are + #: category names, values are corresponding group names. self.categories = categories + + #: The UUID identifier for this client self.uuid = uuid + + #: The Bcfg2 password for this client self.password = password + + #: Connector plugins known to this client self.connectors = [] + + #: The version of the Bcfg2 client this client is running, as + #: a string self.version = version try: + #: The version of the Bcfg2 client this client is running, + #: as a :class:`Bcfg2.version.Bcfg2VersionInfo` object. self.version_info = Bcfg2VersionInfo(version) except (ValueError, AttributeError): self.version_info = None + + #: A :class:`Bcfg2.Server.Plugins.Metadata.MetadataQuery` + #: object for this client. self.query = query # pylint: enable=R0913 def inGroup(self, group): - """Test to see if client is a member of group.""" + """Test to see if client is a member of group. + + :returns: bool """ return group in self.groups def group_in_category(self, category): - """ return the group in the given category that the client is - a member of, or the empty string """ + """ Return the group in the given category that the client is + a member of, or an empty string. + + :returns: string """ for grp in self.query.all_groups_in_category(category): if grp in self.groups: return grp @@ -295,17 +327,59 @@ class ClientMetadata(object): class MetadataQuery(object): - """ object supplied to client metadata to allow client metadata - objects to query metadata without being able to modify it """ + """ This class provides query methods for the metadata of all + clients known to the Bcfg2 server, without being able to modify + that data. + + Note that ``*by_groups()`` and ``*by_profiles()`` behave + differently; for a client to be included in the return value of a + ``*by_groups()`` method, it must be a member of *all* groups + listed in the argument; for a client to be included in the return + value of a ``*by_profiles()`` method, it must have *any* group + listed as its profile group. """ def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): - # resolver is set later + #: Get :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` + #: object for the given hostname. + #: + #: :returns: Bcfg2.Server.Plugins.Metadata.ClientMetadata self.by_name = by_name + + #: Get a list of hostnames of clients that are in all given + #: groups. + #: + #: :param groups: The groups to check clients for membership in + #: :type groups: list + #: + #: :returns: list of strings self.names_by_groups = self._warn_string(by_groups) + + #: Get a list of hostnames of clients whose profile matches + #: any given profile group. + #: + #: :param profiles: The profiles to check clients for + #: membership in. + #: :type profiles: list + #: :returns: list of strings self.names_by_profiles = self._warn_string(by_profiles) + + #: Get all known client hostnames. + #: + #: :returns: list of strings self.all_clients = get_clients + + #: Get all known group names. + #: + #: :returns: list of strings self.all_groups = all_groups + + #: Get the names of all groups in the given category. + #: + #: :param category: The category to query for groups that + #: belong to it. + #: :type category: string + #: :returns: list of strings self.all_groups_in_category = all_groups_in_category def _warn_string(self, func): @@ -326,22 +400,41 @@ class MetadataQuery(object): return inner def by_groups(self, groups): - """ get a list of ClientMetadata objects that are in all given - groups """ + """ Get a list of + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects + that are in all given groups. + + :param groups: The groups to check clients for membership in. + :type groups: list + :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata + objects + """ # don't need to decorate this with _warn_string because # names_by_groups is decorated return [self.by_name(name) for name in self.names_by_groups(groups)] def by_profiles(self, profiles): - """ get a list of ClientMetadata objects that are in any of - the given profiles """ + """ Get a list of + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects + that have any of the given groups as their profile. + + :param profiles: The profiles to check clients for membership + in. + :type profiles: list + :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata + objects + """ # don't need to decorate this with _warn_string because # names_by_profiles is decorated return [self.by_name(name) for name in self.names_by_profiles(profiles)] def all(self): - """ get a list of all ClientMetadata objects """ + """ Get a list of all + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects. + + :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ return [self.by_name(name) for name in self.all_clients()] @@ -1255,7 +1348,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, - self.core.setup('authentication')) == 'bootstrap': + self.core.setup['authentication']) == 'bootstrap': self.update_client(metadata.hostname, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 0dd42c9cb..490ee6f20 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -15,6 +15,11 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, def validate_goals(self, metadata, goals): """Verify that we are generating correct old POSIX entries.""" + if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0): + # do not care about a client that is _any_ 1.3.0 release + # (including prereleases and RCs) + return + for goal in goals: for entry in goal.getchildren(): if entry.tag == 'Path' and 'mode' in entry.keys(): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 3799b1723..4535fb76d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -313,9 +313,7 @@ class YumCollection(Collection): @property def __package_groups__(self): - """ YumCollections support package groups only if - :attr:`use_yum` is True """ - return self.use_yum + return True @property def helper(self): @@ -663,11 +661,6 @@ class YumCollection(Collection): In this implementation the packages may be strings or tuples. See :ref:`yum-pkg-objects` for more information. """ - if not self.use_yum: - self.logger.warning("Packages: Package groups are not supported " - "by Bcfg2's internal Yum dependency generator") - return dict() - if not grouplist: return dict() @@ -679,7 +672,15 @@ class YumCollection(Collection): ptype = "default" gdicts.append(dict(group=group, type=ptype)) - return self.call_helper("get_groups", inputdata=gdicts) + if self.use_yum: + return self.call_helper("get_groups", inputdata=gdicts) + else: + pkgs = dict() + for gdict in gdicts: + pkgs[gdict['group']] = Collection.get_group(self, + gdict['group'], + gdict['type']) + return pkgs def _element_to_pkg(self, el, name): """ Convert a Package or Instance element to a package tuple """ @@ -975,6 +976,7 @@ class YumSource(Source): for x in ['global'] + self.arches]) self.needed_paths = set() self.file_to_arch = dict() + self.yumgroups = dict() __init__.__doc__ = Source.__init__.__doc__ @property @@ -992,7 +994,8 @@ class YumSource(Source): if not self.use_yum: cache = open(self.cachefile, 'wb') cPickle.dump((self.packages, self.deps, self.provides, - self.filemap, self.url_map), cache, 2) + self.filemap, self.url_map, + self.yumgroups), cache, 2) cache.close() def load_state(self): @@ -1002,7 +1005,7 @@ class YumSource(Source): if not self.use_yum: data = open(self.cachefile) (self.packages, self.deps, self.provides, - self.filemap, self.url_map) = cPickle.load(data) + self.filemap, self.url_map, self.yumgroups) = cPickle.load(data) @property def urls(self): @@ -1057,7 +1060,7 @@ class YumSource(Source): urls = [] for elt in xdata.findall(RPO + 'data'): - if elt.get('type') in ['filelists', 'primary']: + if elt.get('type') in ['filelists', 'primary', 'group']: floc = elt.find(RPO + 'location') fullurl = url + floc.get('href') urls.append(fullurl) @@ -1074,11 +1077,14 @@ class YumSource(Source): # we have to read primary.xml first, and filelists.xml afterwards; primaries = list() filelists = list() + groups = list() for fname in self.files: if fname.endswith('primary.xml.gz'): primaries.append(fname) elif fname.endswith('filelists.xml.gz'): filelists.append(fname) + elif fname.find('comps'): + groups.append(fname) for fname in primaries: farch = self.file_to_arch[fname] @@ -1088,6 +1094,9 @@ class YumSource(Source): farch = self.file_to_arch[fname] fdata = lxml.etree.parse(fname).getroot() self.parse_filelist(fdata, farch) + for fname in groups: + fdata = lxml.etree.parse(fname).getroot() + self.parse_group(fdata) # merge data sdata = list(self.packages.values()) @@ -1151,6 +1160,35 @@ class YumSource(Source): self.provides[arch][prov] = list() self.provides[arch][prov].append(pkgname) + @Bcfg2.Server.Plugin.track_statistics() + def parse_group(self, data): + """ parse comps.xml.gz data """ + for group in data.getchildren(): + if not group.tag.endswith('group'): + continue + try: + groupid = group.xpath('id')[0].text + self.yumgroups[groupid] = {'mandatory': list(), + 'default': list(), + 'optional': list(), + 'conditional': list()} + except IndexError: + continue + try: + packagelist = group.xpath('packagelist')[0] + except IndexError: + continue + for pkgreq in packagelist.getchildren(): + pkgtype = pkgreq.get('type', None) + if pkgtype == 'mandatory': + self.yumgroups[groupid]['mandatory'].append(pkgreq.text) + elif pkgtype == 'default': + self.yumgroups[groupid]['default'].append(pkgreq.text) + elif pkgtype == 'optional': + self.yumgroups[groupid]['optional'].append(pkgreq.text) + elif pkgtype == 'conditional': + self.yumgroups[groupid]['conditional'].append(pkgreq.text) + def is_package(self, metadata, package): arch = [a for a in self.arches if a in metadata.groups] if not arch: @@ -1230,3 +1268,26 @@ class YumSource(Source): return self.pulp_id else: return Source.get_repo_name(self, url_map) + + def get_group(self, metadata, group, ptype=None): # pylint: disable=W0613 + """ Get the list of packages of the given type in a package + group. + + :param group: The name of the group to query + :type group: string + :param ptype: The type of packages to get, for backends that + support multiple package types in package groups + (e.g., "recommended," "optional," etc.) + :type ptype: string + :returns: list of strings - package names + """ + try: + yumgroup = self.yumgroups[group] + except KeyError: + return [] + packages = yumgroup['conditional'] + yumgroup['mandatory'] + if ptype in ['default', 'optional', 'all']: + packages += yumgroup['default'] + if ptype in ['optional', 'all']: + packages += yumgroup['optional'] + return packages diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 5d3fbae2e..2175cf0aa 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -5,6 +5,7 @@ determine the completeness of the client configuration. """ import os import sys import glob +import copy import shutil import lxml.etree import Bcfg2.Logger @@ -296,20 +297,22 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ if self.disableResolver: # Config requests no resolver + for struct in structures: + for pkg in struct.xpath('//Package | //BoundPackage'): + if pkg.get("group"): + if pkg.get("type"): + pkg.set("choose", pkg.get("type")) return if collection is None: collection = self.get_collection(metadata) - # base is the set of initial packages -- explicitly - # given in the specification, from expanded package groups, - # and essential to the distribution - base = set() + initial = set() to_remove = [] groups = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): - base.update(collection.packages_from_entry(pkg)) + initial.update(collection.packages_from_entry(pkg)) elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -321,6 +324,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, pkg, xml_declaration=False).decode('UTF-8')) + # base is the set of initial packages explicitly given in the + # specification, packages from expanded package groups, and + # packages essential to the distribution + base = set(initial) + # remove package groups for el in to_remove: el.getparent().remove(el) @@ -336,7 +344,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) - newpkgs = collection.get_new_packages(base, packages) + newpkgs = collection.get_new_packages(initial, packages) self.debug_log("Packages: %d base, %d complete, %d new" % (len(base), len(packages), len(newpkgs))) newpkgs.sort() @@ -514,7 +522,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :return: dict of lists of ``url_map`` data """ collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data()) + return dict(sources=collection.get_additional_data(), + allsources=copy.deepcopy(self.sources)) def end_client_run(self, metadata): """ Hook to clear the cache for this client in diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index d072f1a33..a6dc2c1ef 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -65,10 +65,13 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): (self.name, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) raise PluginInitError(msg) + if self.debug_flag: + self.transport.set_debug(self.debug_flag) def set_debug(self, debug): rv = Debuggable.set_debug(self, debug) - self.transport.set_debug(debug) + if self.transport is not None: + self.transport.set_debug(debug) return rv def process_statistics(self, client, xdata): diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 0d1246d85..28450aa1a 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -412,12 +412,9 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, name = instance.name except AttributeError: name = "unknown" - if hasattr(instance, 'plugins'): - for pname, pinst in list(instance.plugins.items()): - for mname in pinst.__rmi__: - xmname = "%s.%s" % (pname, mname) - fn = getattr(pinst, mname) - self.register_function(fn, name=xmname) + if hasattr(instance, '_get_rmi'): + for fname, func in instance._get_rmi().items(): + self.register_function(func, name=fname) self.logger.info("serving %s at %s" % (name, self.url)) def serve_forever(self): |