diff options
Diffstat (limited to 'src/lib/Bcfg2/Server')
| -rw-r--r-- | src/lib/Bcfg2/Server/Core.py | 93 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/__init__.py | 2 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugin/base.py | 18 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugin/helpers.py | 31 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugin/interfaces.py | 1 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/DBStats.py | 5 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/GroupPatterns.py | 23 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 59 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 2 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Reporting.py | 5 |
10 files changed, 141 insertions, 98 deletions
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index ee1006d1a..0ded7ac26 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -100,9 +100,7 @@ class BaseCore(object): #: The Bcfg2 repository directory self.datastore = setup['repo'] - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: + if setup['verbose']: level = logging.INFO else: level = logging.WARNING @@ -120,6 +118,25 @@ class BaseCore(object): #: A :class:`logging.Logger` object for use by the core self.logger = logging.getLogger('bcfg2-server') + #: Log levels for the various logging handlers with debug True + #: and False. Each loglevel dict is a dict of ``logger name + #: => log level``; the logger names are set in + #: :mod:`Bcfg2.Logger`. The logger name ``default`` is + #: special, and will be used for any log handlers whose name + #: does not appear elsewhere in the dict. At a minimum, + #: ``default`` must be provided. + self._loglevels = {True: dict(default=logging.DEBUG), + False: dict(console=logging.INFO, + default=level)} + + #: Used to keep track of the current debug state of the core. + self.debug_flag = False + + # enable debugging on the core now. debugging is enabled on + # everything else later + if setup['debug']: + self.set_core_debug(None, setup['debug']) + try: filemonitor = \ Bcfg2.Server.FileMonitor.available[setup['filemonitor']] @@ -308,6 +325,11 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() + if self.debug_flag: + # enable debugging on everything else. + self.plugins[plugin].set_debug(self.debug_flag) + + def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -405,6 +427,7 @@ class BaseCore(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ + self.logger.debug("Shutting down core...") if not self.terminate.isSet(): self.terminate.set() self.fam.shutdown() @@ -438,6 +461,8 @@ class BaseCore(object): hook. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ + self.logger.debug("Running %s hooks for %s" % (hook, + metadata.hostname)) start = time.time() try: for plugin in \ @@ -471,6 +496,7 @@ class BaseCore(object): client :type data: list of lxml.etree._Element objects """ + self.logger.debug("Validating structures for %s" % metadata.hostname) for plugin in \ self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): try: @@ -497,6 +523,7 @@ class BaseCore(object): client :type data: list of lxml.etree._Element objects """ + self.logger.debug("Validating goals for %s" % metadata.hostname) for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): try: plugin.validate_goals(metadata, data) @@ -517,6 +544,7 @@ class BaseCore(object): :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: list of :class:`lxml.etree._Element` objects """ + self.logger.debug("Getting structures for %s" % metadata.hostname) structures = list(chain(*[struct.BuildStructures(metadata) for struct in self.structures])) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] @@ -539,6 +567,7 @@ class BaseCore(object): structures to. Modified in-place. :type config: lxml.etree._Element """ + self.logger.debug("Binding structures for %s" % metadata.hostname) for astruct in structures: try: self.BindStructure(astruct, metadata) @@ -555,6 +584,9 @@ class BaseCore(object): :param metadata: Client metadata to bind structure for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ + self.logger.debug("Binding structure %s for %s" % + (structure.get("name", "unknown"), + metadata.hostname)) for entry in structure.getchildren(): if entry.tag.startswith("Bound"): entry.tag = entry.tag[5:] @@ -630,6 +662,7 @@ class BaseCore(object): :type client: string :returns: :class:`lxml.etree._Element` - A complete Bcfg2 configuration document """ + self.logger.debug("Building configuration for %s" % client) start = time.time() config = lxml.etree.Element("Configuration", version='2.0', revision=self.revision) @@ -729,6 +762,7 @@ class BaseCore(object): self.shutdown() raise + self.set_debug(None, self.debug_flag) self._block() def _daemonize(self): @@ -757,6 +791,7 @@ class BaseCore(object): :type mode: string :returns: list of Decision tuples ``(<entry tag>, <entry name>)`` """ + self.logger.debug("Getting decision list for %s" % metadata.hostname) result = [] for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): try: @@ -785,6 +820,7 @@ class BaseCore(object): else: imd = self.metadata_cache.get(client_name, None) if not imd: + self.logger.debug("Building metadata for %s" % client_name) imd = self.metadata.get_initial_metadata(client_name) for conn in self.connectors: grps = conn.get_additional_groups(imd) @@ -806,6 +842,7 @@ class BaseCore(object): :param statistics: The statistics document to process :type statistics: lxml.etree._Element """ + self.logger.debug("Processing statistics for %s" % client_name) meta = self.build_metadata(client_name) state = statistics.find(".//Statistics") if state.get('version') >= '2.0': @@ -933,6 +970,7 @@ class BaseCore(object): return func.__doc__ @exposed + @track_statistics() def DeclareVersion(self, address, version): """ Declare the client version. @@ -943,7 +981,9 @@ class BaseCore(object): :returns: bool - True on success :raises: :exc:`xmlrpclib.Fault` """ - client = self.resolve_client(address)[0] + client = self.resolve_client(address, metadata=False)[0] + self.logger.debug("%s is running Bcfg2 client version %s" % (client, + version)) try: self.metadata.set_version(client, version) except (Bcfg2.Server.Plugin.MetadataConsistencyError, @@ -965,6 +1005,7 @@ class BaseCore(object): """ resp = lxml.etree.Element('probes') client, metadata = self.resolve_client(address, cleanup_cache=True) + self.logger.debug("Getting probes for %s" % client) try: for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): for probe in plugin.GetProbes(metadata): @@ -986,6 +1027,7 @@ class BaseCore(object): :raises: :exc:`xmlrpclib.Fault` """ client, metadata = self.resolve_client(address) + self.logger.debug("Receiving probe data from %s" % client) if self.metadata_cache_mode == 'cautious': # clear the metadata cache right after building the # metadata object; that way the cache is cleared for any @@ -1032,6 +1074,7 @@ class BaseCore(object): :raises: :exc:`xmlrpclib.Fault` """ client = self.resolve_client(address, metadata=False)[0] + self.logger.debug("%s sets its profile to %s" % (client, profile)) try: self.metadata.set_profile(client, profile, address) except (Bcfg2.Server.Plugin.MetadataConsistencyError, @@ -1132,9 +1175,17 @@ class BaseCore(object): :type address: tuple :returns: bool - The new debug state of the FAM """ - for plugin in self.plugins.values(): - plugin.toggle_debug() - return self.toggle_fam_debug(address) + return self.set_debug(address, not self.debug_flag) + + @exposed + def toggle_core_debug(self, address): + """ Toggle debug status of the server core + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: bool - The new debug state of the FAM + """ + return self.set_core_debug(address, not self.debug_flag) @exposed def toggle_fam_debug(self, address): @@ -1151,6 +1202,8 @@ class BaseCore(object): def set_debug(self, address, debug): """ Explicitly set debug status of the FAM and all plugins + :param address: Client (address, hostname) pair + :type address: tuple :param debug: The new debug status. This can either be a boolean, or a string describing the state (e.g., "true" or "false"; case-insensitive) @@ -1161,7 +1214,31 @@ class BaseCore(object): debug = debug.lower() == "true" for plugin in self.plugins.values(): plugin.set_debug(debug) - return self.set_fam_debug(address, debug) + rv = self.set_core_debug(address, debug) + return self.set_fam_debug(address, debug) and rv + + @exposed + def set_core_debug(self, _, debug): + """ Explicity set debug status of the server core + + :param debug: The new debug status. This can either be a + boolean, or a string describing the state (e.g., + "true" or "false"; case-insensitive) + :type debug: bool or string + :returns: bool - The new debug state of the FAM + """ + if debug not in [True, False]: + debug = debug.lower() == "true" + self.debug_flag = debug + self.logger.info("Core: debug = %s" % debug) + levels = self._loglevels[self.debug_flag] + for handler in logging.root.handlers: + level = levels.get(handler.get_name(), levels['default']) + self.logger.debug("Setting %s log handler to %s" % + (handler.get_name(), + logging.getLevelName(level))) + handler.setLevel(level) + return self.debug_flag @exposed def set_fam_debug(self, address, debug): diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 58144958e..e430e3160 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -291,6 +291,8 @@ class FileMonitor(Debuggable): def shutdown(self): """ Handle any tasks required to shut down the monitor. """ + self.debug_log("Shutting down %s file monitor" % + self.__class__.__name__) self.started = False def AddMonitor(self, path, obj, handleID=None): diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index e74909ee9..f7bc08717 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -2,6 +2,7 @@ import os import logging +from Bcfg2.Utils import ClassName class Debuggable(object): @@ -33,8 +34,8 @@ class Debuggable(object): :returns: bool - The new value of the debug flag """ self.debug_flag = debug - self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__, - self.debug_flag), + self.debug_log("%s: debug = %s" % (self.__class__.__name__, + self.debug_flag), flag=True) return debug @@ -59,18 +60,6 @@ class Debuggable(object): self.logger.error(message) -class ClassName(object): - """ This very simple descriptor class exists only to get the name - of the owner class. This is used because, for historical reasons, - we expect every plugin to have a ``name`` attribute that is in - almost all cases the same as the ``__class__.__name__`` attribute - of the plugin object. This makes that more dynamic so that each - plugin isn't repeating its own name. """ - - def __get__(self, inst, owner): - return owner.__name__ - - class Plugin(Debuggable): """ The base class for all Bcfg2 Server plugins. """ @@ -133,6 +122,7 @@ class Plugin(Debuggable): """ Perform shutdown tasks for the plugin :returns: None """ + self.debug_log("Shutting down %s plugin" % self.name) self.running = False def __str__(self): diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 41c450b4e..c2252f956 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -5,6 +5,7 @@ import re import sys import copy import time +import glob import logging import operator import lxml.etree @@ -503,13 +504,14 @@ class XMLFileBacked(FileBacked): def _follow_xincludes(self, fname=None, xdata=None): """ follow xincludes, adding included files to self.extras """ + xinclude = '%sinclude' % Bcfg2.Server.XI_NAMESPACE + if xdata is None: if fname is None: xdata = self.xdata.getroottree() else: xdata = lxml.etree.parse(fname) - included = [el for el in xdata.findall('//%sinclude' % - Bcfg2.Server.XI_NAMESPACE)] + included = [el for el in xdata.findall('//' + xinclude)] for el in included: name = el.get("href") if name.startswith("/"): @@ -520,16 +522,23 @@ class XMLFileBacked(FileBacked): else: rel = self.name fpath = os.path.join(os.path.dirname(rel), name) - if fpath not in self.extras: - if os.path.exists(fpath): - self._follow_xincludes(fname=fpath) - self.add_monitor(fpath) + + # expand globs in xinclude, a bcfg2-specific extension + extras = glob.glob(fpath) + if not extras: + msg = "%s: %s does not exist, skipping" % (self.name, name) + if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): + LOGGER.debug(msg) else: - msg = "%s: %s does not exist, skipping" % (self.name, name) - if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): - LOGGER.debug(msg) - else: - LOGGER.warning(msg) + LOGGER.warning(msg) + + parent = el.getparent() + parent.remove(el) + for extra in extras: + if extra != self.name and extra not in self.extras: + self.add_monitor(extra) + lxml.etree.SubElement(parent, xinclude, href=extra) + self._follow_xincludes(fname=extra) def Index(self): self.xdata = lxml.etree.XML(self.data, base_url=self.name, diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index f42ada773..cb996b1ca 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -313,6 +313,7 @@ class Threaded(object): """ raise NotImplementedError + class ThreadedStatistics(Statistics, Threaded, threading.Thread): """ ThreadedStatistics plugins process client statistics in a separate thread. """ diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index e0794f019..e6ef50fa1 100644 --- a/src/lib/Bcfg2/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -9,7 +9,6 @@ class DBStats(Bcfg2.Server.Plugin.Plugin): def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) self.logger.error("DBStats has been replaced with Reporting") - self.logger.error("DBStats: Be sure to migrate your data "\ - "before running the report collector") + self.logger.error("DBStats: Be sure to migrate your data " + "before running the report collector") raise Bcfg2.Server.Plugin.PluginInitError - diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 1b12e590a..5716a134f 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -6,28 +6,7 @@ import sys import logging import Bcfg2.Server.Lint import Bcfg2.Server.Plugin -from Bcfg2.Compat import any # pylint: disable=W0622 - - -class PackedDigitRange(object): - """ Helper object for NameRange entries """ - - def __init__(self, digit_range): - self.sparse = list() - self.ranges = list() - for item in digit_range.split(','): - if '-' in item: - self.ranges.append(tuple([int(x) for x in item.split('-')])) - else: - self.sparse.append(int(item)) - - def includes(self, other): - """ return True if other is included in this range """ - iother = int(other) - if iother in self.sparse: - return True - return any(iother in range(start, end + 1) - for start, end in self.ranges) +from Bcfg2.Utils import PackedDigitRange class PatternMap(object): diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index fe8dd86a6..a81139b5d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -15,6 +15,7 @@ import Bcfg2.Server import Bcfg2.Server.Lint import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor +from Bcfg2.Utils import locked from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 from Bcfg2.version import Bcfg2VersionInfo @@ -27,15 +28,6 @@ except ImportError: LOGGER = logging.getLogger(__name__) -def locked(fd): - """ Acquire a lock on a file """ - try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - return True - return False - - if HAS_DJANGO: class MetadataClientModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): @@ -195,7 +187,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): newcontents = lxml.etree.tostring(dataroot, xml_declaration=False, pretty_print=True).decode('UTF-8') - while locked(fd) == True: + while locked(fd): pass try: datafile.write(newcontents) @@ -392,7 +384,7 @@ class MetadataGroup(tuple): class Metadata(Bcfg2.Server.Plugin.Metadata, - Bcfg2.Server.Plugin.Statistics, + Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" __author__ = 'bcfg-dev@mcs.anl.gov' @@ -400,7 +392,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Metadata.__init__(self) - Bcfg2.Server.Plugin.Statistics.__init__(self, core, datastore) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) self.watch_clients = watch_clients self.states = dict() @@ -685,8 +677,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.raddresses[clname] = set() self.raddresses[clname].add(caddr) if 'auth' in client.attrib: - self.auth[client.get('name')] = client.get('auth', - 'cert+password') + self.auth[client.get('name')] = client.get('auth') if 'uuid' in client.attrib: self.uuid[client.get('uuid')] = clname if client.get('secure', 'false').lower() == 'true': @@ -1200,7 +1191,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # look at cert.cN client = certinfo['commonName'] self.debug_log("Got cN %s; using as client name" % client) - auth_type = self.auth.get(client, 'cert+password') + auth_type = self.auth.get(client, + self.core.setup['authentication']) elif user == 'root': id_method = 'address' try: @@ -1223,12 +1215,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Authenticating client %s" % client) # next we validate the address - if id_method == 'uuid': - addr_is_valid = True - else: - addr_is_valid = self.validate_client_address(client, address) - - if not addr_is_valid: + if (id_method != 'uuid' and + not self.validate_client_address(client, address)): return False if id_method == 'cert' and auth_type != 'cert+password': @@ -1238,23 +1226,19 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # we are done if cert+password not required return True - if client not in self.passwords: - if client in self.secure: - self.logger.error("Client %s in secure mode but has no " - "password" % address[0]) - return False - if password != self.password: - self.logger.error("Client %s used incorrect global password" % - address[0]) - return False + if client not in self.passwords and client in self.secure: + self.logger.error("Client %s in secure mode but has no password" % + address[0]) + return False + if client not in self.secure: if client in self.passwords: plist = [self.password, self.passwords[client]] else: plist = [self.password] if password not in plist: - self.logger.error("Client %s failed to use either allowed " - "password" % address[0]) + self.logger.error("Client %s failed to use an allowed password" + % address[0]) return False else: # client in secure mode and has a client password @@ -1268,12 +1252,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return True # pylint: enable=R0911,R0912 - def process_statistics(self, meta, _): - """ Hook into statistics interface to toggle clients in - bootstrap mode """ - client = meta.hostname - if client in self.auth and self.auth[client] == 'bootstrap': - self.update_client(client, dict(auth='cert')) + def end_statistics(self, metadata): + """ Hook to toggle clients in bootstrap mode """ + if self.auth.get(metadata.hostname, + self.core.setup['authentication']) == 'bootstrap': + self.update_client(metadata.hostname, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): """Admin mode viz support.""" diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index ec0d8e828..27f493677 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -35,7 +35,7 @@ class AptCollection(Collection): for source in self: if source.rawurl: - self.logger.info("Packages: Skipping rawurl %s" % + self.logger.info("Packages: Skipping rawurl %s" % source.rawurl) else: lines.append("deb %s %s %s" % (source.url, source.version, diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index d072f1a33..a6dc2c1ef 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -65,10 +65,13 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): (self.name, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) raise PluginInitError(msg) + if self.debug_flag: + self.transport.set_debug(self.debug_flag) def set_debug(self, debug): rv = Debuggable.set_debug(self, debug) - self.transport.set_debug(debug) + if self.transport is not None: + self.transport.set_debug(debug) return rv def process_statistics(self, client, xdata): |