diff options
Diffstat (limited to 'src/lib/Bcfg2/Server')
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Fam.py | 31 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Gamin.py | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Inotify.py | 39 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Pseudo.py | 11 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/__init__.py | 45 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/__init__.py | 26 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 320 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Probes.py | 6 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/__init__.py | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/models.py | 29 |
10 files changed, 311 insertions, 204 deletions
diff --git a/src/lib/Bcfg2/Server/FileMonitor/Fam.py b/src/lib/Bcfg2/Server/FileMonitor/Fam.py index aef74add4..9c6031be9 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Fam.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Fam.py @@ -7,21 +7,24 @@ import logging from time import time from Bcfg2.Server.FileMonitor import FileMonitor -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class Fam(FileMonitor): + """ file monitor with support for FAM """ + __priority__ = 90 def __init__(self, ignore=None, debug=False): FileMonitor.__init__(self, ignore=ignore, debug=debug) - self.fm = _fam.open() + self.filemonitor = _fam.open() self.users = {} def fileno(self): """Return fam file handle number.""" - return self.fm.fileno() + return self.filemonitor.fileno() - def handle_event_set(self, _): + def handle_event_set(self, _=None): self.Service() def handle_events_in_interval(self, interval): @@ -30,13 +33,13 @@ class Fam(FileMonitor): if self.Service(): now = time() - def AddMonitor(self, path, obj): + def AddMonitor(self, path, obj, _=None): """Add a monitor to path, installing a callback to obj.HandleEvent.""" mode = os.stat(path)[stat.ST_MODE] if stat.S_ISDIR(mode): - handle = self.fm.monitorDirectory(path, None) + handle = self.filemonitor.monitorDirectory(path, None) else: - handle = self.fm.monitorFile(path, None) + handle = self.filemonitor.monitorFile(path, None) self.handles[handle.requestID()] = handle if obj != None: self.users[handle.requestID()] = obj @@ -50,10 +53,10 @@ class Fam(FileMonitor): start = time() now = time() while (time() - now) < interval: - if self.fm.pending(): - while self.fm.pending(): + if self.filemonitor.pending(): + while self.filemonitor.pending(): count += 1 - rawevents.append(self.fm.nextEvent()) + rawevents.append(self.filemonitor.nextEvent()) now = time() unique = [] bookkeeping = [] @@ -73,10 +76,10 @@ class Fam(FileMonitor): if event.requestID in self.users: try: self.users[event.requestID].HandleEvent(event) - except: - logger.error("Handling event for file %s" % event.filename, + except: # pylint: disable=W0702 + LOGGER.error("Handling event for file %s" % event.filename, exc_info=1) end = time() - logger.info("Processed %s fam events in %03.03f seconds. %s coalesced" % - (count, (end - start), collapsed)) + LOGGER.info("Processed %s fam events in %03.03f seconds. " + "%s coalesced" % (count, (end - start), collapsed)) return count diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 9d4330e89..d0ba59cd8 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -2,15 +2,12 @@ import os import stat -import logging # pylint: disable=F0401 from gamin import WatchMonitor, GAMCreated, GAMExists, GAMEndExist, \ GAMChanged, GAMDeleted # pylint: enable=F0401 from Bcfg2.Server.FileMonitor import Event, FileMonitor -logger = logging.getLogger(__name__) - class GaminEvent(Event): """ @@ -28,6 +25,7 @@ class GaminEvent(Event): class Gamin(FileMonitor): + """ file monitor with gamin support """ __priority__ = 10 def __init__(self, ignore=None, debug=False): diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index 75eff3bc5..5a8a1e1c6 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -1,17 +1,18 @@ """ Inotify driver for file alteration events """ import os -import sys import logging import pyinotify # pylint: disable=F0401 -from Bcfg2.Compat import reduce +from Bcfg2.Compat import reduce # pylint: disable=W0622 from Bcfg2.Server.FileMonitor import Event from Bcfg2.Server.FileMonitor.Pseudo import Pseudo -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class Inotify(Pseudo, pyinotify.ProcessEvent): + """ file monitor with inotify support """ + __priority__ = 1 # pylint: disable=E1101 action_map = {pyinotify.IN_CREATE: 'created', @@ -24,18 +25,18 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): def __init__(self, ignore=None, debug=False): Pseudo.__init__(self, ignore=ignore, debug=debug) - pyinotify.ProcessEvent(self) + pyinotify.ProcessEvent.__init__(self) self.event_filter = dict() self.watches_by_path = dict() # these are created in start() after the server is done forking self.notifier = None - self.wm = None + self.watchmgr = None self.add_q = [] def start(self): Pseudo.start(self) - self.wm = pyinotify.WatchManager() - self.notifier = pyinotify.ThreadedNotifier(self.wm, self) + self.watchmgr = pyinotify.WatchManager() + self.notifier = pyinotify.ThreadedNotifier(self.watchmgr, self) self.notifier.start() for monitor in self.add_q: self.AddMonitor(*monitor) @@ -43,7 +44,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): def fileno(self): if self.started: - return self.wm.get_fd() + return self.watchmgr.get_fd() else: return None @@ -54,9 +55,9 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): action = aname break try: - watch = self.wm.watches[ievent.wd] + watch = self.watchmgr.watches[ievent.wd] except KeyError: - logger.error("Error handling event for %s: Watch %s not found" % + LOGGER.error("Error handling event for %s: Watch %s not found" % (ievent.pathname, ievent.wd)) return # FAM-style file monitors return the full path to the parent @@ -87,7 +88,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): ievent.pathname in self.event_filter[ievent.wd]): self.events.append(evt) - def AddMonitor(self, path, obj): + def AddMonitor(self, path, obj, handleID=None): # strip trailing slashes path = path.rstrip("/") @@ -116,18 +117,18 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): # see if this path is already being watched try: - wd = self.watches_by_path[watch_path] + watchdir = self.watches_by_path[watch_path] except KeyError: - wd = self.wm.add_watch(watch_path, self.mask, - quiet=False)[watch_path] - self.watches_by_path[watch_path] = wd + watchdir = self.watchmgr.add_watch(watch_path, self.mask, + quiet=False)[watch_path] + self.watches_by_path[watch_path] = watchdir produce_exists = True if not is_dir: - if wd not in self.event_filter: - self.event_filter[wd] = [path] - elif path not in self.event_filter[wd]: - self.event_filter[wd].append(path) + if watchdir not in self.event_filter: + self.event_filter[watchdir] = [path] + elif path not in self.event_filter[watchdir]: + self.event_filter[watchdir].append(path) else: # we've been asked to watch a file that we're already # watching, so we don't need to produce 'exists' diff --git a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py index 089d4cf0f..9062cbfd8 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py @@ -1,12 +1,13 @@ """ Pseudo provides static monitor support for file alteration events """ import os -import logging from Bcfg2.Server.FileMonitor import FileMonitor, Event -logger = logging.getLogger(__name__) class Pseudo(FileMonitor): + """ file monitor that only produces events on server startup and + doesn't actually monitor at all """ + __priority__ = 99 def AddMonitor(self, path, obj, handleID=None): @@ -15,9 +16,9 @@ class Pseudo(FileMonitor): handleID = len(list(self.handles.keys())) self.events.append(Event(handleID, path, 'exists')) if os.path.isdir(path): - dirList = os.listdir(path) - for includedFile in dirList: - self.events.append(Event(handleID, includedFile, 'exists')) + dirlist = os.listdir(path) + for fname in dirlist: + self.events.append(Event(handleID, fname, 'exists')) self.events.append(Event(handleID, path, 'endExist')) if obj != None: diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index fd0cb66f1..1b12ab703 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -4,12 +4,13 @@ import os import sys import fnmatch import logging -import pkgutil from time import sleep, time -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class Event(object): + """ Base class for all FAM events """ def __init__(self, request_id, filename, code): self.requestID = request_id self.filename = filename @@ -53,33 +54,40 @@ class FileMonitor(object): self.started = True def debug_log(self, msg): + """ log a debug message """ if self.debug: - logger.info(msg) + LOGGER.info(msg) def should_ignore(self, event): + """ returns true if an event should be ignored """ for pattern in self.ignore: - if (fnmatch.fnmatch(event.filename, pattern) or + if (fnmatch.fnmatch(event.filename, pattern) or fnmatch.fnmatch(os.path.split(event.filename)[-1], pattern)): self.debug_log("Ignoring %s" % event) return True return False def pending(self): + """ returns True if there are pending events """ return bool(self.events) def get_event(self): + """ get the oldest pending event """ return self.events.pop(0) def fileno(self): + """ get the file descriptor of the file monitor thread """ return 0 def handle_one_event(self, event): + """ handle the given event by dispatching it to the object + that handles events for the path """ if not self.started: self.start() if self.should_ignore(event): return if event.requestID not in self.handles: - logger.info("Got event for unexpected id %s, file %s" % + LOGGER.info("Got event for unexpected id %s, file %s" % (event.requestID, event.filename)) return self.debug_log("Dispatching event %s %s to obj %s" % @@ -87,12 +95,13 @@ class FileMonitor(object): self.handles[event.requestID])) try: self.handles[event.requestID].HandleEvent(event) - except: + except: # pylint: disable=W0702 err = sys.exc_info()[1] - logger.error("Error in handling of event %s for %s: %s" % + LOGGER.error("Error in handling of event %s for %s: %s" % (event.code2str(), event.filename, err)) def handle_event_set(self, lock=None): + """ Handle all pending events """ if not self.started: self.start() count = 1 @@ -100,19 +109,18 @@ class FileMonitor(object): start = time() if lock: lock.acquire() - try: - self.handle_one_event(event) - while self.pending(): - self.handle_one_event(self.get_event()) - count += 1 - except: - pass + self.handle_one_event(event) + while self.pending(): + self.handle_one_event(self.get_event()) + count += 1 if lock: lock.release() end = time() - logger.info("Handled %d events in %.03fs" % (count, (end - start))) + LOGGER.info("Handled %d events in %.03fs" % (count, (end - start))) def handle_events_in_interval(self, interval): + """ handle events for the specified period of time (in + seconds) """ if not self.started: self.start() end = time() + interval @@ -124,10 +132,15 @@ class FileMonitor(object): sleep(0.5) def shutdown(self): + """ shutdown the monitor """ self.started = False + def AddMonitor(self, path, obj, handleID=None): + """ watch the specified path, alerting obj to events """ + raise NotImplementedError + -available = dict() +available = dict() # pylint: disable=C0103 # todo: loading the monitor drivers should be automatic from Bcfg2.Server.FileMonitor.Pseudo import Pseudo diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index eea205b75..a59214048 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -10,6 +10,9 @@ import fcntl import termios import struct from Bcfg2.Server import XI_NAMESPACE +from Bcfg2.Compat import walk_packages + +__all__ = [m[1] for m in walk_packages(path=__path__)] def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -99,6 +102,7 @@ class Plugin(object): class ErrorHandler (object): """ a class to handle errors for bcfg2-lint plugins """ + def __init__(self, config=None): self.errors = 0 self.warnings = 0 @@ -114,32 +118,26 @@ class ErrorHandler (object): else: self._wrapper = lambda s: [s] - self._handlers = {} + self.errors = dict() if config is not None: - for err, action in config.items(): - if "warn" in action: - self._handlers[err] = self.warn - elif "err" in action: - self._handlers[err] = self.error - else: - self._handlers[err] = self.debug + self.RegisterErrors(config.items()) def RegisterErrors(self, errors): """ Register a dict of errors (name: default level) that a plugin may raise """ for err, action in errors.items(): - if err not in self._handlers: + if err not in self.errors: if "warn" in action: - self._handlers[err] = self.warn + self.errors[err] = self.warn elif "err" in action: - self._handlers[err] = self.error + self.errors[err] = self.error else: - self._handlers[err] = self.debug + self.errors[err] = self.debug def dispatch(self, err, msg): """ Dispatch an error to the correct handler """ - if err in self._handlers: - self._handlers[err](msg) + if err in self.errors: + self.errors[err](msg) self.logger.debug(" (%s)" % err) else: # assume that it's an error, but complain diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 468d1f190..477f88b82 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -14,20 +14,20 @@ import Bcfg2.Server import Bcfg2.Server.Lint import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor -from Bcfg2.Compat import MutableMapping, all # pylint: disable=W0622 +from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 from Bcfg2.version import Bcfg2VersionInfo try: from django.db import models - has_django = True + HAS_DJANGO = True except ImportError: - has_django = False + HAS_DJANGO = False -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) def locked(fd): - """Aquire a lock on a file""" + """ Acquire a lock on a file """ try: fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: @@ -35,13 +35,17 @@ def locked(fd): return False -if has_django: +if HAS_DJANGO: class MetadataClientModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): + """ django model for storing clients in the database """ hostname = models.CharField(max_length=255, primary_key=True) version = models.CharField(max_length=31, null=True) - class ClientVersions(MutableMapping): + class ClientVersions(MutableMapping, object): + """ dict-like object to make it easier to access client bcfg2 + versions from the database """ + def __getitem__(self, key): try: return MetadataClientModel.objects.get(hostname=key).version @@ -77,7 +81,7 @@ if has_django: def __contains__(self, key): try: - client = MetadataClientModel.objects.get(hostname=key) + MetadataClientModel.objects.get(hostname=key) return True except MetadataClientModel.DoesNotExist: return False @@ -85,6 +89,7 @@ if has_django: class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): """Handles xml config files and all XInclude statements""" + def __init__(self, metadata, watch_clients, basefile): # we tell XMLFileBacked _not_ to add a monitor for this file, # because the main Metadata plugin has already added one. @@ -105,18 +110,23 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): Bcfg2.Server.FileMonitor.Pseudo) def _get_xdata(self): + """ getter for xdata property """ if not self.data: raise Bcfg2.Server.Plugin.MetadataRuntimeError("%s has no data" % self.basefile) return self.data def _set_xdata(self, val): + """ setter for xdata property. in practice this should only be + used by the test suite """ self.data = val xdata = property(_get_xdata, _set_xdata) @property def base_xdata(self): + """ property to get the data of the base file (without any + xincludes processed) """ if not self.basedata: raise Bcfg2.Server.Plugin.MetadataRuntimeError("%s has no data" % self.basefile) @@ -160,7 +170,6 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): newcontents = lxml.etree.tostring(dataroot, xml_declaration=False, pretty_print=True).decode('UTF-8') - fd = datafile.fileno() while locked(fd) == True: pass @@ -198,7 +207,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): 'xmltree': self.basedata, 'xquery': cli} else: - """Try to find the data in included files""" + # Try to find the data in included files for included in self.extras: try: xdata = lxml.etree.parse(included, @@ -217,7 +226,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): if self.fam and self.should_monitor: self.fam.AddMonitor(fpath, self.metadata) - def HandleEvent(self, event): + def HandleEvent(self, event=None): """Handle fam events""" filename = os.path.basename(event.filename) if event.filename in self.extras: @@ -233,7 +242,8 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): class ClientMetadata(object): """This object contains client metadata.""" - def __init__(self, client, profile, groups, bundles, aliases, addresses, + # pylint: disable=R0913 + def __init__(self, client, profile, groups, bundles, aliases, addresses, categories, uuid, password, version, query): self.hostname = client self.profile = profile @@ -248,15 +258,18 @@ class ClientMetadata(object): self.version = version try: self.version_info = Bcfg2VersionInfo(version) - except: + except (ValueError, AttributeError): self.version_info = None self.query = query + # pylint: enable=R0913 def inGroup(self, group): """Test to see if client is a member of group.""" return group in self.groups def group_in_category(self, category): + """ return the group in the given category that the client is + a member of, or the empty string """ for grp in self.query.all_groups_in_category(category): if grp in self.groups: return grp @@ -264,6 +277,9 @@ class ClientMetadata(object): class MetadataQuery(object): + """ object supplied to client metadata to allow client metadata + objects to query metadata without being able to modify it """ + def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): # resolver is set later @@ -275,37 +291,52 @@ class MetadataQuery(object): self.all_groups_in_category = all_groups_in_category def _warn_string(self, func): - # it's a common mistake to call by_groups, etc., in templates with - # a single string argument instead of a list. that doesn't cause - # errors because strings are iterables. this decorator warns - # about that usage. + """ decorator to warn that a MetadataQuery function that + expects a list has been called with a single string argument + instead. this is a common mistake in templates, and it + doesn't cause errors because strings are iterables """ + + # pylint: disable=C0111 + @wraps(func) def inner(arg): if isinstance(arg, str): - logger.warning("%s: %s takes a list as argument, not a string" % - (self.__class__.__name__, func.__name__)) + LOGGER.warning("%s: %s takes a list as argument, not a string" + % (self.__class__.__name__, func.__name__)) return func(arg) + # pylint: enable=C0111 + return inner def by_groups(self, groups): + """ get a list of ClientMetadata objects that are in all given + groups """ # don't need to decorate this with _warn_string because # names_by_groups is decorated return [self.by_name(name) for name in self.names_by_groups(groups)] def by_profiles(self, profiles): + """ get a list of ClientMetadata objects that are in any of + the given profiles """ # don't need to decorate this with _warn_string because # names_by_profiles is decorated - return [self.by_name(name) for name in self.names_by_profiles(profiles)] + return [self.by_name(name) + for name in self.names_by_profiles(profiles)] def all(self): + """ get a list of all ClientMetadata objects """ return [self.by_name(name) for name in self.all_clients()] class MetadataGroup(tuple): + """ representation of a metadata group. basically just a named tuple """ + + # pylint: disable=R0913,W0613 def __new__(cls, name, bundles=None, category=None, is_profile=False, is_public=False, is_private=False): if bundles is None: bundles = set() return tuple.__new__(cls, (bundles, category)) + # pylint: enable=W0613 def __init__(self, name, bundles=None, category=None, is_profile=False, is_public=False, is_private=False): @@ -320,6 +351,7 @@ class MetadataGroup(tuple): self.is_private = is_private # record which clients we've warned about category suppression self.warned = [] + # pylint: enable=R0913 def __str__(self): return repr(self) @@ -347,7 +379,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.watch_clients = watch_clients self.states = dict() self.extra = dict() - self.handlers = [] + self.handlers = dict() self.groups_xml = self._handle_file("groups.xml") if (self._use_db and os.path.exists(os.path.join(self.data, "clients.xml"))): @@ -407,6 +439,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, "w").write(kwargs[aname]) def _handle_file(self, fname): + """ set up the necessary magic for handling a metadata file + (clients.xml or groups.xml, e.g.) """ if self.watch_clients: try: self.core.fam.AddMonitor(os.path.join(self.data, fname), self) @@ -416,13 +450,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) self.states[fname] = False - aname = re.sub(r'[^A-z0-9_]', '_', fname) xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname) - self.handlers.append(xmlcfg.HandleEvent) + aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(fname)) + self.handlers[xmlcfg.HandleEvent] = getattr(self, + "_handle_%s_event" % aname) self.extra[fname] = [] return xmlcfg def _search_xdata(self, tag, name, tree, alias=False): + """ Generic method to find XML data (group, client, etc.) """ for node in tree.findall("//%s" % tag): if node.get("name") == name: return node @@ -442,9 +478,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return self._search_xdata("Bundle", bundle_name, tree) def search_client(self, client_name, tree): + """ find a client in the given XML tree """ return self._search_xdata("Client", client_name, tree, alias=True) def _add_xdata(self, config, tag, name, attribs=None, alias=False): + """ Generic method to add XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) if node != None: self.logger.error("%s \"%s\" already exists" % (tag, name)) @@ -491,6 +529,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, attribs=attribs, alias=True) def _update_xdata(self, config, tag, name, attribs, alias=False): + """ Generic method to modify XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) if node == None: self.logger.error("%s \"%s\" does not exist" % (tag, name)) @@ -534,7 +573,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, else: return self.clients - def _remove_xdata(self, config, tag, name, alias=False): + def _remove_xdata(self, config, tag, name): + """ Generic method to remove XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata) if node == None: self.logger.error("%s \"%s\" does not exist" % (tag, name)) @@ -561,10 +601,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def remove_bundle(self, bundle_name): """Remove a bundle.""" if self._use_db: - msg = "Metadata does not support removing bundles with " + \ - "use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support removing bundles with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) @@ -582,7 +622,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, else: return self._remove_xdata(self.clients_xml, "Client", client_name) - def _handle_clients_xml_event(self, event): + def _handle_clients_xml_event(self, _): + """ handle all events for clients.xml and files xincluded from + clients.xml """ xdata = self.clients_xml.xdata self.clients = [] self.clientgroups = {} @@ -640,12 +682,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if self._use_db: self.clients = self.list_clients() - def _handle_groups_xml_event(self, event): + def _handle_groups_xml_event(self, _): # pylint: disable=R0912 + """ re-read groups.xml on any event on it """ self.groups = {} # these three functions must be separate functions in order to # ensure that the scope is right for the closures they return def get_condition(element): + """ Return a predicate that returns True if a client meets + the condition specified in the given Group or Client + element """ negate = element.get('negate', 'false').lower() == 'true' pname = element.get("name") if element.tag == 'Group': @@ -654,7 +700,13 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return lambda c, g, _: negate != (pname == c) def get_category_condition(category, gname): - def in_cat(client, groups, categories): + """ get a predicate that returns False if a client is + already a member of a group in the given category, True + otherwise """ + def in_cat(client, groups, categories): # pylint: disable=W0613 + """ return True if the client is already a member of a + group in the category given in the enclosing function, + False otherwise """ if category in categories: if (gname not in self.groups or client not in self.groups[gname].warned): @@ -670,6 +722,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return in_cat def aggregate_conditions(conditions): + """ aggregate all conditions on a given group declaration + into a single predicate """ return lambda client, groups, cats: \ all(cond(client, groups, cats) for cond in conditions) @@ -735,14 +789,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def HandleEvent(self, event): """Handle update events for data files.""" - for hdlr in self.handlers: - aname = re.sub(r'[^A-z0-9_]', '_', - os.path.basename(event.filename)) - if hdlr(event): + for handles, event_handler in self.handlers.items(): + if handles(event): # clear the entire cache when we get an event for any # metadata file self.core.metadata_cache.expire() - getattr(self, "_handle_%s_event" % aname)(event) + event_handler(event) if False not in list(self.states.values()) and self.debug_flag: # check that all groups are real and complete. this is @@ -759,18 +811,19 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Group %s set as nonexistent group %s" % (gname, group)) - def set_profile(self, client, profile, addresspair, force=False): + def set_profile(self, client, profile, addresspair): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) if False in list(self.states.values()): - raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not been read yet") - if not force and profile not in self.groups: + raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " + "been read yet") + if profile not in self.groups: msg = "Profile group %s does not exist" % profile self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] - if not force and not group.is_public: + if not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) @@ -788,8 +841,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, (client, profiles, profile)) self.update_client(client, dict(profile=profile)) if client in self.clientgroups: - for p in profiles: - self.clientgroups[client].remove(p) + for prof in profiles: + self.clientgroups[client].remove(prof) self.clientgroups[client].append(profile) else: self.clientgroups[client] = [profile] @@ -872,7 +925,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ set group membership based on the contents of groups.xml and initial group membership of this client. Returns a tuple of (allgroups, categories)""" - numgroups = -1 # force one initial pass + numgroups = -1 # force one initial pass if categories is None: categories = dict() while numgroups != len(groups): @@ -893,10 +946,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, del categories[group.category] return (groups, categories) - def get_initial_metadata(self, client): + def get_initial_metadata(self, client): # pylint: disable=R0914,R0912 """Return the metadata for a given client.""" if False in list(self.states.values()): - raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not been read yet") + raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " + "been read yet") client = client.lower() if client in self.core.metadata_cache: @@ -904,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if client in self.aliases: client = self.aliases[client] - + groups = set() categories = dict() profile = None @@ -917,7 +971,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = self.default if pgroup: - self.set_profile(client, pgroup, (None, None), force=True) + self.set_profile(client, pgroup, (None, None)) groups.add(pgroup) category = self.groups[pgroup].category if category: @@ -958,8 +1012,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, try: bundles.update(self.groups[group].bundles) except KeyError: - self.logger.warning("%s: %s is a member of undefined group %s" % - (self.name, client, group)) + self.logger.warning("%s: %s is a member of undefined group %s" + % (self.name, client, group)) aliases = self.raliases.get(client, set()) addresses = self.raddresses.get(client, set()) @@ -989,6 +1043,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return rv def get_all_group_names(self): + """ return a list of all group names """ all_groups = set() all_groups.update(self.groups.keys()) all_groups.update([g.name for g in self.group_membership.values()]) @@ -998,10 +1053,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return all_groups def get_all_groups_in_category(self, category): + """ return a list of names of groups in the given category """ return set([g.name for g in self.groups.values() if g.category == category]) def get_client_names_by_profiles(self, profiles): + """ return a list of names of clients in the given profile groups """ rv = [] for client in list(self.clients): mdata = self.get_initial_metadata(client) @@ -1010,13 +1067,14 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return rv def get_client_names_by_groups(self, groups): - mdata = [self.core.build_metadata(client) - for client in list(self.clients)] + """ return a list of names of clients in the given groups """ + mdata = [self.core.build_metadata(client) for client in self.clients] return [md.hostname for md in mdata if md.groups.issuperset(groups)] def get_client_names_by_bundles(self, bundles): - mdata = [self.core.build_metadata(client) - for client in list(self.clients.keys())] + """ given a list of bundles, return a list of names of clients + that use those bundles """ + mdata = [self.core.build_metadata(client) for client in self.clients] return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] def merge_additional_groups(self, imd, groups): @@ -1071,6 +1129,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error("Resolved to %s" % resolved) return False + # pylint: disable=R0911,R0912 def AuthenticateConnection(self, cert, user, password, address): """This function checks auth creds.""" if not isinstance(user, str): @@ -1121,8 +1180,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if client not in self.passwords: if client in self.secure: - self.logger.error("Client %s in secure mode but has no password" - % address[0]) + self.logger.error("Client %s in secure mode but has no " + "password" % address[0]) return False if password != self.password: self.logger.error("Client %s used incorrect global password" % @@ -1147,9 +1206,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if user != 'root': self.session_cache[address] = (time.time(), client) return True + # pylint: enable=R0911,R0912 def process_statistics(self, meta, _): - """Hook into statistics interface to toggle clients in bootstrap mode.""" + """ Hook into statistics interface to toggle clients in + bootstrap mode """ client = meta.hostname if client in self.auth and self.auth[client] == 'bootstrap': self.update_client(client, dict(auth='cert')) @@ -1159,23 +1220,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if only_client: clientmeta = self.core.build_metadata(only_client) - def include_client(client): - return not only_client or client != only_client - - def include_bundle(bundle): - return not only_client or bundle in clientmeta.bundles - - def include_group(group): - return not only_client or group in clientmeta.groups - - groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml"), - parser=Bcfg2.Server.XMLParser) - try: - groups_tree.xinclude() - except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for groups.xml: %s" % - sys.exc_info()[1]) - groups = groups_tree.getroot() + groups = self.groups_xml.xdata.getroot() categories = {'default': 'grey83'} viz_str = [] egroups = groups.findall("Group") + groups.findall('.//Groups/Group') @@ -1186,37 +1231,72 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if None in categories: del categories[None] if hosts: - instances = {} - for client in list(self.clients): - if include_client(client): - continue - if client in self.clientgroups: - groups = self.clientgroups[client] - elif self.default: - groups = [self.default] - else: - continue - for group in groups: - try: - instances[group].append(client) - except KeyError: - instances[group] = [client] - for group, clist in list(instances.items()): - clist.sort() - viz_str.append('"%s-instances" [ label="%s", shape="record" ];' % - (group, '|'.join(clist))) - viz_str.append('"%s-instances" -> "group-%s";' % - (group, group)) + viz_str.extend(self._viz_hosts(only_client)) if bundles: - bundles = [] - [bundles.append(bund.get('name')) \ - for bund in groups.findall('.//Bundle') \ - if bund.get('name') not in bundles \ - and include_bundle(bund.get('name'))] - bundles.sort() - for bundle in bundles: - viz_str.append('"bundle-%s" [ label="%s", shape="septagon"];' % - (bundle, bundle)) + viz_str.extend(self._viz_bundles(bundles, clientmeta)) + viz_str.extend(self._viz_groups(egroups, bundles, clientmeta)) + if key: + for category in categories: + viz_str.append('"%s" [label="%s", shape="record", ' + 'style="filled", fillcolor="%s"];' % + (category, category, categories[category])) + return "\n".join("\t" + s for s in viz_str) + + def _viz_hosts(self, only_client): + """ add hosts to the viz graph """ + def include_client(client): + """ return True if the given client should be included in + the graph""" + return not only_client or client != only_client + + instances = {} + rv = [] + for client in list(self.clients): + if include_client(client): + continue + if client in self.clientgroups: + grps = self.clientgroups[client] + elif self.default: + grps = [self.default] + else: + continue + for group in grps: + try: + instances[group].append(client) + except KeyError: + instances[group] = [client] + for group, clist in list(instances.items()): + clist.sort() + rv.append('"%s-instances" [ label="%s", shape="record" ];' % + (group, '|'.join(clist))) + rv.append('"%s-instances" -> "group-%s";' % (group, group)) + return rv + + def _viz_bundles(self, bundles, clientmeta): + """ add bundles to the viz graph """ + + def include_bundle(bundle): + """ return True if the given bundle should be included in + the graph""" + return not clientmeta or bundle in clientmeta.bundles + + bundles = list(set(bund.get('name')) + for bund in self.groups_xml.xdata.findall('.//Bundle') + if include_bundle(bund.get('name'))) + bundles.sort() + return ['"bundle-%s" [ label="%s", shape="septagon"];' % (bundle, + bundle) + for bundle in bundles] + + def _viz_groups(self, egroups, bundles, clientmeta): + """ add groups to the viz graph """ + + def include_group(group): + """ return True if the given group should be included in + the graph """ + return not clientmeta or group in clientmeta.groups + + rv = [] gseen = [] for group in egroups: if group.get('profile', 'false') == 'true': @@ -1225,31 +1305,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, style = "filled" gseen.append(group.get('name')) if include_group(group.get('name')): - viz_str.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' % - (group.get('name'), group.get('name'), style, - group.get('color'))) + rv.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' + % (group.get('name'), group.get('name'), style, + group.get('color'))) if bundles: for bundle in group.findall('Bundle'): - viz_str.append('"group-%s" -> "bundle-%s";' % - (group.get('name'), bundle.get('name'))) + rv.append('"group-%s" -> "bundle-%s";' % + (group.get('name'), bundle.get('name'))) gfmt = '"group-%s" [label="%s", style="filled", fillcolor="grey83"];' for group in egroups: for parent in group.findall('Group'): - if parent.get('name') not in gseen and include_group(parent.get('name')): - viz_str.append(gfmt % (parent.get('name'), - parent.get('name'))) + if (parent.get('name') not in gseen and + include_group(parent.get('name'))): + rv.append(gfmt % (parent.get('name'), + parent.get('name'))) gseen.append(parent.get("name")) if include_group(group.get('name')): - viz_str.append('"group-%s" -> "group-%s";' % - (group.get('name'), parent.get('name'))) - if key: - for category in categories: - viz_str.append('"%s" [label="%s", shape="record", style="filled", fillcolor="%s"];' % - (category, category, categories[category])) - return "\n".join("\t" + s for s in viz_str) + rv.append('"group-%s" -> "group-%s";' % + (group.get('name'), parent.get('name'))) class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): + """ bcfg2-lint plugin for Metadata """ + def Run(self): self.nested_clients() self.deprecated_options() @@ -1260,6 +1338,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "deprecated-clients-options": "warning"} def deprecated_options(self): + """ check for the location='floating' option, which has been + deprecated in favor of floating='true' """ clientdata = self.metadata.clients_xml.xdata for el in clientdata.xpath("//Client"): loc = el.get("location") @@ -1271,9 +1351,11 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): self.LintError("deprecated-clients-options", "The location='%s' option is deprecated. " "Please use floating='%s' instead: %s" % - (loc, floating, self.RenderXML(el))) + (loc, floating, self.RenderXML(el))) def nested_clients(self): + """ check for a Client tag inside a Client tag, which doesn't + make any sense """ groupdata = self.metadata.groups_xml.xdata for el in groupdata.xpath("//Client//Client"): self.LintError("nested-client-tags", diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 49e3b5e63..700c5e2e8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -132,6 +132,12 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): encoding) fam.AddMonitor(path, self) + def HandleEvent(self, event): + """ handle events on everything but probed.xml """ + if (event.filename != self.path and + not event.filename.endswith("probed.xml")): + return self.handle_event(event) + def get_probe_data(self, metadata): """ Get an XML description of all probes for a client suitable for sending to that client. diff --git a/src/lib/Bcfg2/Server/__init__.py b/src/lib/Bcfg2/Server/__init__.py index f79b51dd3..3eb300a98 100644 --- a/src/lib/Bcfg2/Server/__init__.py +++ b/src/lib/Bcfg2/Server/__init__.py @@ -6,8 +6,8 @@ __all__ = ["Admin", "Core", "FileMonitor", "Plugin", "Plugins", "Hostbase", "Reports", "Snapshots", "XMLParser", "XI", "XI_NAMESPACE"] -XMLParser = lxml.etree.XMLParser(remove_blank_text=True) - XI = 'http://www.w3.org/2001/XInclude' XI_NAMESPACE = '{%s}' % XI +# pylint: disable=C0103 +XMLParser = lxml.etree.XMLParser(remove_blank_text=True) diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index bae6497a9..0328c6bea 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -1,15 +1,18 @@ +""" Django database models for all plugins """ + import sys import logging import Bcfg2.Options import Bcfg2.Server.Plugins from django.db import models -from Bcfg2.Compat import ConfigParser -logger = logging.getLogger('Bcfg2.Server.models') +LOGGER = logging.getLogger('Bcfg2.Server.models') MODELS = [] + def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): + """ load models from plugins specified in the config """ global MODELS if plugins is None: @@ -19,9 +22,10 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): plugin_opt = Bcfg2.Options.SERVER_PLUGINS plugin_opt.default = Bcfg2.Server.Plugins.__all__ - setup = Bcfg2.Options.OptionParser(dict(plugins=plugin_opt, - configfile=Bcfg2.Options.CFILE), - quiet=quiet) + setup = \ + Bcfg2.Options.OptionParser(dict(plugins=plugin_opt, + configfile=Bcfg2.Options.CFILE), + quiet=quiet) setup.parse([Bcfg2.Options.CFILE.cmd, cfile]) plugins = setup['plugins'] @@ -42,7 +46,7 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): try: err = sys.exc_info()[1] mod = __import__(plugin) - except: + except: # pylint: disable=W0702 if plugins != Bcfg2.Server.Plugins.__all__: # only produce errors if the default plugin list # was not used -- i.e., if the config file was set @@ -50,7 +54,8 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): # all plugins, IOW. the error from the first # attempt to import is probably more accurate than # the second attempt. - logger.error("Failed to load plugin %s: %s" % (plugin, err)) + LOGGER.error("Failed to load plugin %s: %s" % (plugin, + err)) continue for sym in dir(mod): obj = getattr(mod, sym) @@ -62,16 +67,16 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): # and thus that this module will always work. load_models(quiet=True) -# Monitor our internal db version + class InternalDatabaseVersion(models.Model): - """Object that tell us to witch version is the database.""" + """ Object that tell us to which version the database is """ version = models.IntegerField() updated = models.DateTimeField(auto_now_add=True) def __str__(self): - return "version %d updated the %s" % (self.version, self.updated.isoformat()) + return "version %d updated the %s" % (self.version, + self.updated.isoformat()) - class Meta: + class Meta: # pylint: disable=C0111,W0232 app_label = "reports" get_latest_by = "version" - |