diff options
| author | Chris St. Pierre <chris.a.st.pierre@gmail.com> | 2012-09-21 13:55:05 -0400 |
|---|---|---|
| committer | Chris St. Pierre <chris.a.st.pierre@gmail.com> | 2012-09-25 11:58:47 -0400 |
| commit | dd28e90f183972cc2a395094ce3e3f72e861953f (patch) | |
| tree | dfe10fd66e0535763d953333ed49f6467762fbd6 /src | |
| parent | eec8f653c0235bde8d3a754802a4485f0d542ea3 (diff) | |
| download | bcfg2-dd28e90f183972cc2a395094ce3e3f72e861953f.tar.gz bcfg2-dd28e90f183972cc2a395094ce3e3f72e861953f.tar.bz2 bcfg2-dd28e90f183972cc2a395094ce3e3f72e861953f.zip | |
run pylint for errors on almost everything, full runs on some selected stuff
Diffstat (limited to 'src')
69 files changed, 854 insertions, 753 deletions
diff --git a/src/lib/Bcfg2/Client/Tools/APK.py b/src/lib/Bcfg2/Client/Tools/APK.py index d70916792..9c7692dbb 100644 --- a/src/lib/Bcfg2/Client/Tools/APK.py +++ b/src/lib/Bcfg2/Client/Tools/APK.py @@ -42,10 +42,10 @@ class APK(Bcfg2.Client.Tools.PkgTool): #FIXME: Does APK have any sort of verification mechanism? return True else: - self.loggger.info(" pkg %s at version %s, not %s" % - (entry.attrib['name'], - self.installed[entry.attrib['name']], - entry.attrib['version'])) + self.logger.info(" pkg %s at version %s, not %s" % + (entry.attrib['name'], + self.installed[entry.attrib['name']], + entry.attrib['version'])) entry.set('current_version', self.installed[entry.get('name')]) return False entry.set('current_exists', 'false') diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index 31d3a1971..4f4fdc3b2 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -80,7 +80,7 @@ class Action(Bcfg2.Client.Tools.Tool): return True def InstallPostInstall(self, entry): - return self.InstallAction(self, entry) + return self.InstallAction(entry) def BundleUpdated(self, bundle, states): """Run postinstalls when bundles have been updated.""" diff --git a/src/lib/Bcfg2/Client/Tools/IPS.py b/src/lib/Bcfg2/Client/Tools/IPS.py index e30bbd2a4..730f4c2d9 100644 --- a/src/lib/Bcfg2/Client/Tools/IPS.py +++ b/src/lib/Bcfg2/Client/Tools/IPS.py @@ -1,7 +1,9 @@ """This is the Bcfg2 support for OpenSolaris packages.""" +# pylint: disable=F0401 import pkg.client.image as image import pkg.client.progress as progress +# pylint: enable=F0401 import Bcfg2.Client.Tools diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py index a0a8ac9f2..15acd2b3d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py @@ -7,11 +7,8 @@ import shutil from datetime import datetime import Bcfg2.Client.Tools from Bcfg2.Compat import walk_packages -try: - from base import POSIXTool -except ImportError: - # py3k, incompatible syntax with py2.4 - exec("from .base import POSIXTool") +from Bcfg2.Client.Tools.POSIX.base import POSIXTool + class POSIX(Bcfg2.Client.Tools.Tool): """POSIX File support code.""" diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 6952d0f7b..2fe6791ae 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -524,7 +524,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): except (OSError, KeyError): err = sys.exc_info()[1] self.logger.error("POSIX: Lookup of %s %s failed: %s" % - (scope, acl.qualifier, err)) + (atype, acl.qualifier, err)) qual = acl.qualifier existing[(atype, acl.tag_type, qual)] = \ self._norm_acl_perms(acl.permset) diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index c8c05061c..02889a2e7 100644 --- a/src/lib/Bcfg2/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py @@ -1,5 +1,6 @@ """This is the bcfg2 support for pacman""" +import sys import Bcfg2.Client.Tools diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index babdaf6bf..11172d25f 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -235,7 +235,7 @@ class SELinuxEntryHandler(object): return getattr(self, "_%sargs" % method)(entry) elif hasattr(self, "_defaultargs"): # default args - return self._defaultargs(entry) + return self._defaultargs(entry) # pylint: disable=E1101 else: raise NotImplementedError diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index e6081dc1c..790ac787a 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -9,18 +9,21 @@ missing = [] import os import shutil import sys + +# pylint: disable=F0401 # python-dulwich git imports try: import dulwich import dulwich.index from dulwich.errors import NotGitRepository -except: +except ImportError: missing.append('git') # subversion import try: import pysvn -except: +except ImportError: missing.append('svn') +# pylint: enable=F0401 import Bcfg2.Client.Tools @@ -123,13 +126,14 @@ class VCS(Bcfg2.Client.Tools.Tool): def Installsvn(self, entry): """Checkout contents from a svn repository""" + # pylint: disable=E1101 try: client = pysvn.Client.update(entry.get('name'), recurse=True) - except: + except pysvn.ClientError: self.logger.error("Failed to update repository", exc_info=1) return False - return True + # pylint: enable=E1101 def VerifyPath(self, entry, _): vcs = entry.get('vcstype') diff --git a/src/lib/Bcfg2/Client/Tools/YUMng.py b/src/lib/Bcfg2/Client/Tools/YUMng.py index 34029b9fe..a93a48f9a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUMng.py +++ b/src/lib/Bcfg2/Client/Tools/YUMng.py @@ -185,12 +185,14 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): else: debuglevel = 0 + # pylint: disable=E1121 try: self.yb.preconf.debuglevel = debuglevel self.yb._getConfig() except AttributeError: self.yb._getConfig(self.yb.conf.config_file_path, debuglevel=debuglevel) + # pylint: enable=E1121 try: self.yb.doConfigSetup() @@ -711,7 +713,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): if result != 0: self.logger.debug("Unable to install %s-%s" % \ (self.instance_status[inst].get('pkg').get('name'), - self.str_evra(inst))) + nevraString(inst))) return False else: self.logger.debug("Installed %s-%s-%s" % \ @@ -817,7 +819,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): for pkg in self.extra_instances: for inst in pkg: self.logger.info(" %s %s" % \ - ((pkg.get('name'), self.str_evra(inst)))) + ((pkg.get('name'), nevraString(inst)))) # Figure out which instances of the packages actually need something # doing to them and place in the appropriate work 'queue'. diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index b058ae16c..51d3ceecf 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -376,3 +376,6 @@ class SvcTool(Tool): except: self.logger.error("Unexpected failure of install method for entry type %s" % (entry.tag), exc_info=1) + + def InstallService(self, entry): + raise NotImplementedError diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 858479611..d6bbd3b72 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -2,6 +2,8 @@ # library will use lxml, then builtin xml.etree, then ElementTree +# pylint: disable=F0401,E0611 + try: from lxml.etree import Element, SubElement, XML, tostring from lxml.etree import XMLSyntaxError as ParseError @@ -19,7 +21,8 @@ except ImportError: driver = 'etree-py' except ImportError: try: - from elementtree.ElementTree import Element, SubElement, XML, tostring + from elementtree.ElementTree import Element, SubElement, XML, \ + tostring driver = 'etree' import elementtree.ElementTree Element = elementtree.ElementTree.Element @@ -32,5 +35,3 @@ except ImportError: print("Failed to load lxml, xml.etree and elementtree.ElementTree") print("Cannot continue") raise SystemExit(1) - -len([Element, SubElement, XML, tostring, ParseError]) diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index a31bcab1a..9cd8a5531 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -3,6 +3,8 @@ Python 2.4 and such-like """ import sys +# pylint: disable=F0401,E0611 + try: from email.Utils import formatdate except ImportError: diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index f0ee82815..df7db0079 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -151,7 +151,7 @@ class SSLServer (SocketServer.TCPServer, object): def get_request(self): (sock, sockinfo) = self.socket.accept() - sock.settimeout(self.timeout) + sock.settimeout(self.timeout) # pylint: disable=E1101 sslsock = ssl.wrap_socket(sock, server_side=True, certfile=self.certfile, @@ -366,8 +366,10 @@ class XMLRPCServer (SocketServer.ThreadingMixIn, SSLServer, XMLRPCDispatcher.__init__(self, allow_none, encoding) if not RequestHandlerClass: + # pylint: disable=E0102 class RequestHandlerClass (XMLRPCRequestHandler): """A subclassed request handler to prevent class-attribute conflicts.""" + # pylint: enable=E0102 SSLServer.__init__(self, listen_all, diff --git a/src/lib/Bcfg2/Server/Admin/Snapshots.py b/src/lib/Bcfg2/Server/Admin/Snapshots.py index 36e3dfe02..c3f3268d0 100644 --- a/src/lib/Bcfg2/Server/Admin/Snapshots.py +++ b/src/lib/Bcfg2/Server/Admin/Snapshots.py @@ -2,7 +2,7 @@ from datetime import date import sys # Prereq issues can be signaled with ImportError, so no try needed -import sqlalchemy, sqlalchemy.orm +import sqlalchemy, sqlalchemy.orm # pylint: disable=F0401 import Bcfg2.Server.Admin import Bcfg2.Server.Snapshots import Bcfg2.Server.Snapshots.model diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index a90ed1079..5ceee36b7 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -75,7 +75,7 @@ class Core(BaseCore): def _daemonize(self): self.context.open() self.logger.info("%s daemonized" % self.name) - + def _run(self): hostname, port = urlparse(self.setup['location'])[1].split(':') server_address = socket.getaddrinfo(hostname, diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 3fe52c4e9..95d8173c6 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -16,13 +16,13 @@ import Bcfg2.Logger import Bcfg2.Server.FileMonitor from Bcfg2.Cache import Cache from Bcfg2.Statistics import Statistics -from Bcfg2.Compat import xmlrpclib, reduce +from Bcfg2.Compat import xmlrpclib, reduce # pylint: disable=W0622 from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError try: - import psyco + import psyco # pylint: disable=F0401 psyco.full() -except: +except ImportError: pass os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' @@ -503,10 +503,6 @@ class BaseCore(object): until the server is killed """ raise NotImplementedError - def critical_error(self, operation): - """ this should be overridden by child classes """ - self.logger.fatal(operation, exc_info=1) - def GetDecisions(self, metadata, mode): """Get data for the decision list.""" result = [] diff --git a/src/lib/Bcfg2/Server/FileMonitor/Fam.py b/src/lib/Bcfg2/Server/FileMonitor/Fam.py index 1a00fffa0..aef74add4 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Fam.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Fam.py @@ -1,7 +1,7 @@ """ Fam provides FAM support for file alteration events """ import os -import _fam +import _fam # pylint: disable=F0401 import stat import logging from time import time diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 8a72fc0eb..9d4330e89 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -3,8 +3,10 @@ import os import stat import logging +# pylint: disable=F0401 from gamin import WatchMonitor, GAMCreated, GAMExists, GAMEndExist, \ GAMChanged, GAMDeleted +# pylint: enable=F0401 from Bcfg2.Server.FileMonitor import Event, FileMonitor logger = logging.getLogger(__name__) diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index 6a8fd0201..75eff3bc5 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -3,7 +3,7 @@ import os import sys import logging -import pyinotify +import pyinotify # pylint: disable=F0401 from Bcfg2.Compat import reduce from Bcfg2.Server.FileMonitor import Event from Bcfg2.Server.FileMonitor.Pseudo import Pseudo @@ -13,11 +13,13 @@ logger = logging.getLogger(__name__) class Inotify(Pseudo, pyinotify.ProcessEvent): __priority__ = 1 + # pylint: disable=E1101 action_map = {pyinotify.IN_CREATE: 'created', pyinotify.IN_DELETE: 'deleted', pyinotify.IN_MODIFY: 'changed', pyinotify.IN_MOVED_FROM: 'deleted', pyinotify.IN_MOVED_TO: 'created'} + # pylint: enable=E1101 mask = reduce(lambda x, y: x | y, action_map.keys()) def __init__(self, ignore=None, debug=False): diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index f5385cabb..b9d5d79c4 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -38,6 +38,7 @@ def is_device_mode(val): except: return False + class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): """ verify attributes for configuration entries (as defined in doc/server/configurationentries) """ @@ -48,7 +49,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): device=dict(name=is_filename, owner=is_username, group=is_username, dev_type=lambda v: \ - v in Bcfg2.Client.Tools.POSIX.device_map), + v in Bcfg2.Client.Tools.POSIX.base.device_map), directory=dict(name=is_filename, owner=is_username, group=is_username, perms=is_octal_mode), file=dict(name=is_filename, owner=is_username, diff --git a/src/lib/Bcfg2/Server/Plugin/__init__.py b/src/lib/Bcfg2/Server/Plugin/__init__.py index b76dbe7a0..ed1282ba0 100644 --- a/src/lib/Bcfg2/Server/Plugin/__init__.py +++ b/src/lib/Bcfg2/Server/Plugin/__init__.py @@ -16,7 +16,8 @@ import os import sys sys.path.append(os.path.dirname(__file__)) -from base import * -from interfaces import * -from helpers import * -from exceptions import * +# pylint: disable=W0401 +from Bcfg2.Server.Plugin.base import * +from Bcfg2.Server.Plugin.interfaces import * +from Bcfg2.Server.Plugin.helpers import * +from Bcfg2.Server.Plugin.exceptions import * diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index 74ec6df39..8d288f835 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -102,9 +102,8 @@ class Plugin(Debuggable): :type datastore: string :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` - .. autoattribute:: __rmi__ + .. autoattribute:: Bcfg2.Server.Plugin.base.Debuggable.__rmi__ """ - object.__init__(self) self.Entries = {} self.core = core self.data = os.path.join(datastore, self.name) diff --git a/src/lib/Bcfg2/Server/Plugin/exceptions.py b/src/lib/Bcfg2/Server/Plugin/exceptions.py index de561e8e4..24ac96917 100644 --- a/src/lib/Bcfg2/Server/Plugin/exceptions.py +++ b/src/lib/Bcfg2/Server/Plugin/exceptions.py @@ -1,5 +1,6 @@ """ Exceptions for Bcfg2 Server Plugins.""" + class PluginInitError(Exception): """Error raised in cases of :class:`Bcfg2.Server.Plugin.base.Plugin` initialization errors.""" diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index e6b571eea..96d661b57 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -10,34 +10,33 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Options from Bcfg2.Compat import CmpMixin -from base import * -from interfaces import * -from exceptions import * +from Bcfg2.Server.Plugin.base import Debuggable, Plugin +from Bcfg2.Server.Plugin.interfaces import Generator +from Bcfg2.Server.Plugin.exceptions import SpecificityError, PluginInitError, \ + PluginExecutionError try: - import django - has_django = True + import django # pylint: disable=W0611 + HAS_DJANGO = True except ImportError: - has_django = False - -# grab default metadata info from bcfg2.conf -opts = {'owner': Bcfg2.Options.MDATA_OWNER, - 'group': Bcfg2.Options.MDATA_GROUP, - 'perms': Bcfg2.Options.MDATA_PERMS, - 'secontext': Bcfg2.Options.MDATA_SECONTEXT, - 'important': Bcfg2.Options.MDATA_IMPORTANT, - 'paranoid': Bcfg2.Options.MDATA_PARANOID, - 'sensitive': Bcfg2.Options.MDATA_SENSITIVE} - -#: A dict containing default metadata for Path entries -default_file_metadata = Bcfg2.Options.OptionParser(opts) -default_file_metadata.parse([]) -del default_file_metadata['args'] - -logger = logging.getLogger(__name__) + HAS_DJANGO = False + +#: A dict containing default metadata for Path entries from bcfg2.conf +DEFAULT_FILE_METADATA = Bcfg2.Options.OptionParser(dict( + owner=Bcfg2.Options.MDATA_OWNER, + group=Bcfg2.Options.MDATA_GROUP, + perms=Bcfg2.Options.MDATA_PERMS, + secontext=Bcfg2.Options.MDATA_SECONTEXT, + important=Bcfg2.Options.MDATA_IMPORTANT, + paranoid=Bcfg2.Options.MDATA_PARANOID, + sensitive=Bcfg2.Options.MDATA_SENSITIVE)) +DEFAULT_FILE_METADATA.parse([]) +del DEFAULT_FILE_METADATA['args'] + +LOGGER = logging.getLogger(__name__) #: a compiled regular expression for parsing info and :info files -info_regex = re.compile('owner:(\s)*(?P<owner>\S+)|' + +INFO_REGEX = re.compile('owner:(\s)*(?P<owner>\S+)|' + 'group:(\s)*(?P<group>\S+)|' + 'perms:(\s)*(?P<perms>\w+)|' + 'secontext:(\s)*(?P<secontext>\S+)|' + @@ -47,7 +46,8 @@ info_regex = re.compile('owner:(\s)*(?P<owner>\S+)|' + 'important:(\s)*(?P<important>\S+)|' + 'mtime:(\s)*(?P<mtime>\w+)|') -def bind_info(entry, metadata, infoxml=None, default=default_file_metadata): + +def bind_info(entry, metadata, infoxml=None, default=DEFAULT_FILE_METADATA): """ Bind the file metadata in the given :class:`Bcfg2.Server.Plugin.helpers.InfoXML` object to the given entry. @@ -71,7 +71,7 @@ def bind_info(entry, metadata, infoxml=None, default=default_file_metadata): infoxml.pnode.Match(metadata, mdata, entry=entry) if 'Info' not in mdata: msg = "Failed to set metadata for file %s" % entry.get('name') - logger.error(msg) + LOGGER.error(msg) raise PluginExecutionError(msg) for attr, val in list(mdata['Info'][None].items()): entry.set(attr, val) @@ -104,7 +104,7 @@ class DatabaseBacked(Plugin): use_db = self.core.setup.cfp.getboolean(self.section, self.option, default=False) - if use_db and has_django and self.core.database_available: + if use_db and HAS_DJANGO and self.core.database_available: return True elif not use_db: return False @@ -119,7 +119,7 @@ class PluginDatabaseModel(object): inherit from. This is just a mixin; models must also inherit from django.db.models.Model to be valid Django models.""" - class Meta: + class Meta: # pylint: disable=C0111,W0232 app_label = "Server" @@ -155,7 +155,7 @@ class FileBacked(object): self.Index() except IOError: err = sys.exc_info()[1] - logger.error("Failed to read file %s: %s" % (self.name, err)) + LOGGER.error("Failed to read file %s: %s" % (self.name, err)) def Index(self): """ Index() is called by :func:`HandleEvent` every time the @@ -197,7 +197,7 @@ class DirectoryBacked(object): :param fam: The FAM object used to receive notifications of changes :type fam: Bcfg2.Server.FileMonitor.FileMonitor - + .. ----- .. autoattribute:: __child__ """ @@ -241,7 +241,7 @@ class DirectoryBacked(object): dirpathname = os.path.join(self.data, relative) if relative not in self.handles.values(): if not os.path.isdir(dirpathname): - logger.error("%s is not a directory" % dirpathname) + LOGGER.error("%s is not a directory" % dirpathname) return reqid = self.fam.AddMonitor(dirpathname, self) self.handles[reqid] = relative @@ -262,7 +262,7 @@ class DirectoryBacked(object): self.fam) self.entries[relative].HandleEvent(event) - def HandleEvent(self, event): + def HandleEvent(self, event): # pylint: disable=R0912 """ Handle FAM events. This method is invoked by the FAM when it detects a change to @@ -286,7 +286,7 @@ class DirectoryBacked(object): return if event.requestID not in self.handles: - logger.warn("Got %s event with unknown handle (%s) for %s" % + LOGGER.warn("Got %s event with unknown handle (%s) for %s" % (action, event.requestID, event.filename)) return @@ -297,7 +297,7 @@ class DirectoryBacked(object): event.filename = event.filename[len(self.data) + 1:] if self.ignore and self.ignore.search(event.filename): - logger.debug("Ignoring event %s" % event.filename) + LOGGER.debug("Ignoring event %s" % event.filename) return # Calculate the absolute and relative paths this event refers to @@ -332,18 +332,18 @@ class DirectoryBacked(object): # class doesn't support canceling, so at least let # the user know that a restart might be a good # idea. - logger.warn("Directory properties for %s changed, please " + - " consider restarting the server" % (abspath)) + LOGGER.warn("Directory properties for %s changed, please " + " consider restarting the server" % abspath) else: # Got a "changed" event for a directory that we # didn't know about. Go ahead and treat it like a # "created" event, but log a warning, because this # is unexpected. - logger.warn("Got %s event for unexpected dir %s" % + LOGGER.warn("Got %s event for unexpected dir %s" % (action, abspath)) self.add_directory_monitor(relpath) else: - logger.warn("Got unknown dir event %s %s %s" % + LOGGER.warn("Got unknown dir event %s %s %s" % (event.requestID, event.code2str(), abspath)) elif self.patterns.search(event.filename): if action in ['exists', 'created']: @@ -356,15 +356,15 @@ class DirectoryBacked(object): # know about. Go ahead and treat it like a # "created" event, but log a warning, because this # is unexpected. - logger.warn("Got %s event for unexpected file %s" % + LOGGER.warn("Got %s event for unexpected file %s" % (action, abspath)) self.add_entry(relpath, event) else: - logger.warn("Got unknown file event %s %s %s" % + LOGGER.warn("Got unknown file event %s %s %s" % (event.requestID, event.code2str(), abspath)) else: - logger.warn("Could not process filename %s; ignoring" % + LOGGER.warn("Could not process filename %s; ignoring" % event.filename) @@ -399,6 +399,7 @@ class XMLFileBacked(FileBacked): .. autoattribute:: __identifier__ """ FileBacked.__init__(self, filename) + self.xdata = None self.label = "" self.entries = [] self.extras = [] @@ -433,9 +434,9 @@ class XMLFileBacked(FileBacked): else: msg = "%s: %s does not exist, skipping" % (self.name, name) if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): - self.logger.debug(msg) + LOGGER.debug(msg) else: - self.logger.warning(msg) + LOGGER.warning(msg) def Index(self): try: @@ -443,7 +444,7 @@ class XMLFileBacked(FileBacked): parser=Bcfg2.Server.XMLParser) except lxml.etree.XMLSyntaxError: msg = "Failed to parse %s: %s" % (self.name, sys.exc_info()[1]) - logger.error(msg) + LOGGER.error(msg) raise PluginInitError(msg) self._follow_xincludes() @@ -452,7 +453,7 @@ class XMLFileBacked(FileBacked): self.xdata.getroottree().xinclude() except lxml.etree.XIncludeError: err = sys.exc_info()[1] - logger.error("XInclude failed on %s: %s" % (self.name, err)) + LOGGER.error("XInclude failed on %s: %s" % (self.name, err)) self.entries = self.xdata.getchildren() if self.__identifier__ is not None: @@ -491,7 +492,7 @@ class StructFile(XMLFileBacked): def _include_element(self, item, metadata): """ determine if an XML element matches the metadata """ - if isinstance(item, lxml.etree._Comment): + if isinstance(item, lxml.etree._Comment): # pylint: disable=W0212 return False negate = item.get('negate', 'false').lower() == 'true' if item.tag == 'Group': @@ -606,6 +607,7 @@ class INode(object): self._load_children(data, idict) def _load_children(self, data, idict): + """ load children """ for item in data.getchildren(): if item.tag in self.ignore: continue @@ -622,8 +624,8 @@ class INode(object): self.contents[item.tag][item.get('name')]['__text__'] = \ item.text if item.getchildren(): - self.contents[item.tag][item.get('name')]['__children__'] =\ - item.getchildren() + self.contents[item.tag][item.get('name')]['__children__'] \ + = item.getchildren() try: idict[item.tag].append(item.get('name')) except KeyError: @@ -635,7 +637,7 @@ class INode(object): for key in self.contents: try: data[key].update(self.contents[key]) - except: + except: # pylint: disable=W0702 data[key] = {} data[key].update(self.contents[key]) for child in self.children: @@ -647,12 +649,18 @@ class InfoNode (INode): includes ``<Path>`` tags, suitable for use with :file:`info.xml` files.""" - raw = {'Client': "lambda m, e:'%(name)s' == m.hostname and predicate(m, e)", - 'Group': "lambda m, e:'%(name)s' in m.groups and predicate(m, e)", - 'Path': "lambda m, e:('%(name)s' == e.get('name') or '%(name)s' == e.get('realname')) and predicate(m, e)"} - nraw = {'Client': "lambda m, e:'%(name)s' != m.hostname and predicate(m, e)", - 'Group': "lambda m, e:'%(name)s' not in m.groups and predicate(m, e)", - 'Path': "lambda m, e:('%(name)s' != e.get('name') and '%(name)s' != e.get('realname')) and predicate(m, e)"} + raw = dict( + Client="lambda m, e: '%(name)s' == m.hostname and predicate(m, e)", + Group="lambda m, e: '%(name)s' in m.groups and predicate(m, e)", + Path="lambda m, e: ('%(name)s' == e.get('name') or " + + "'%(name)s' == e.get('realname')) and " + + "predicate(m, e)") + nraw = dict( + Client="lambda m, e: '%(name)s' != m.hostname and predicate(m, e)", + Group="lambda m, e: '%(name)s' not in m.groups and predicate(m, e)", + Path="lambda m, e: '%(name)s' != e.get('name') and " + + "'%(name)s' != e.get('realname') and " + + "predicate(m, e)") containers = ['Group', 'Client', 'Path'] @@ -679,14 +687,15 @@ class XMLSrc(XMLFileBacked): data = open(self.name).read() except IOError: msg = "Failed to read file %s: %s" % (self.name, sys.exc_info()[1]) - logger.error(msg) + LOGGER.error(msg) raise PluginExecutionError(msg) self.items = {} try: xdata = lxml.etree.XML(data, parser=Bcfg2.Server.XMLParser) except lxml.etree.XMLSyntaxError: - msg = "Failed to parse file %s" % (self.name, sys.exc_info()[1]) - logger.error(msg) + msg = "Failed to parse file %s: %s" % (self.name, + sys.exc_info()[1]) + LOGGER.error(msg) raise PluginExecutionError(msg) self.pnode = self.__node__(xdata, self.items) self.cache = None @@ -696,7 +705,7 @@ class XMLSrc(XMLFileBacked): if self.__priority_required__: msg = "Got bogus priority %s for file %s" % \ (xdata.get('priority'), self.name) - logger.error(msg) + LOGGER.error(msg) raise PluginExecutionError(msg) del xdata, data @@ -706,7 +715,8 @@ class XMLSrc(XMLFileBacked): if self.cache is None or self.cache[0] != metadata: cache = (metadata, self.__cacheobj__()) if self.pnode is None: - logger.error("Cache method called early for %s; forcing data load" % (self.name)) + LOGGER.error("Cache method called early for %s; " + "forcing data load" % self.name) self.HandleEvent() return self.pnode.Match(metadata, cache[1]) @@ -769,7 +779,20 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): self.Entries[itype] = {child: self.BindEntry} HandleEvent.__doc__ = XMLDirectoryBacked.HandleEvent.__doc__ - def _matches(self, entry, metadata, rules): + def _matches(self, entry, metadata, rules): # pylint: disable=W0613 + """ Whether or not a given entry has a matching entry in this + PrioDir. By default this does strict matching (i.e., the + entry name is in ``rules.keys()``), but this can be overridden + to provide regex matching, etc. + + :param entry: The entry to find a match for + :type entry: lxml.etree._Element + :param metadata: The metadata to get attributes for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :rules: A dict of rules to look in for a matching rule + :type rules: dict + :returns: bool + """ return entry.get('name') in rules def BindEntry(self, entry, metadata): @@ -814,7 +837,9 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): self._matches(entry, metadata, src.cache[1][entry.tag]))] if len(matching) == 0: - raise PluginExecutionError('No matching source for entry when retrieving attributes for %s(%s)' % (entry.tag, entry.attrib.get('name'))) + raise PluginExecutionError("No matching source for entry when " + "retrieving attributes for %s(%s)" % + (entry.tag, entry.attrib.get('name'))) elif len(matching) == 1: index = 0 else: @@ -839,7 +864,8 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): if '__text__' in data: entry.text = data['__text__'] if '__children__' in data: - [entry.append(copy.copy(item)) for item in data['__children__']] + for item in data['__children__']: + entry.append(copy.copy(item)) return dict([(key, data[key]) for key in list(data.keys()) @@ -857,8 +883,8 @@ class Specificity(CmpMixin): apply to a single client are the most specific. Objects that apply to groups are sorted by priority. """ - def __init__(self, all=False, group=False, hostname=False, prio=0, - delta=False): + def __init__(self, all=False, group=False, # pylint: disable=W0622 + hostname=False, prio=0, delta=False): """ :param all: The object applies to all clients. :type all: bool @@ -897,7 +923,7 @@ class Specificity(CmpMixin): self.hostname == metadata.hostname or self.group in metadata.groups) - def __cmp__(self, other): + def __cmp__(self, other): # pylint: disable=R0911 """Sort most to least specific.""" if self.all: if other.all: @@ -936,7 +962,7 @@ class SpecificData(object): """ A file that is specific to certain clients, groups, or all clients. """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific, encoding): # pylint: disable=W0613 """ :param name: The full path to the file :type name: string @@ -948,9 +974,9 @@ class SpecificData(object): :param encoding: The encoding to use for data in this file :type encoding: string """ - self.name = name self.specific = specific + self.data = None def handle_event(self, event): """ Handle a FAM event. Note that the SpecificData object @@ -968,8 +994,8 @@ class SpecificData(object): self.data = open(self.name).read() except UnicodeDecodeError: self.data = open(self.name, mode='rb').read() - except: - logger.error("Failed to read file %s" % self.name) + except: # pylint: disable=W0201 + LOGGER.error("Failed to read file %s" % self.name) class EntrySet(Debuggable): @@ -1026,7 +1052,7 @@ class EntrySet(Debuggable): :type specific: Bcfg2.Server.Plugin.helpers.Specificity :param encoding: The encoding to use for data in this file :type encoding: string - + Additionally, the object returned by ``entry_type`` must have a ``specific`` attribute that is sortable (e.g., a :class:`Bcfg2.Server.Plugin.helpers.Specificity` object). @@ -1038,10 +1064,10 @@ class EntrySet(Debuggable): self.path = path self.entry_type = entry_type self.entries = {} - self.metadata = default_file_metadata.copy() + self.metadata = DEFAULT_FILE_METADATA.copy() self.infoxml = None self.encoding = encoding - + if self.basename_is_regex: base_pat = basename else: @@ -1126,7 +1152,7 @@ class EntrySet(Debuggable): self.entry_init(event) else: if event.filename not in self.entries: - logger.warning("Got %s event for unknown file %s" % + LOGGER.warning("Got %s event for unknown file %s" % (action, event.filename)) if action == 'changed': # received a bogus changed event; warn, but treat @@ -1163,7 +1189,7 @@ class EntrySet(Debuggable): entry_type = self.entry_type if event.filename in self.entries: - logger.warn("Got duplicate add for %s" % event.filename) + LOGGER.warn("Got duplicate add for %s" % event.filename) else: fpath = os.path.join(self.path, event.filename) try: @@ -1171,7 +1197,7 @@ class EntrySet(Debuggable): specific=specific) except SpecificityError: if not self.ignore.match(event.filename): - logger.error("Could not process filename %s; ignoring" % + LOGGER.error("Could not process filename %s; ignoring" % fpath) return self.entries[event.filename] = entry_type(fpath, spec, @@ -1222,7 +1248,7 @@ class EntrySet(Debuggable): def update_metadata(self, event): """ Process changes to or creation of info, :info, and info.xml files for the EntrySet. - + :param event: An event that applies to an info handled by this EntrySet :type event: Bcfg2.Server.FileMonitor.Event @@ -1235,9 +1261,9 @@ class EntrySet(Debuggable): self.infoxml.HandleEvent(event) elif event.filename in [':info', 'info']: for line in open(fpath).readlines(): - match = info_regex.match(line) + match = INFO_REGEX.match(line) if not match: - logger.warning("Failed to match line in %s: %s" % (fpath, + LOGGER.warning("Failed to match line in %s: %s" % (fpath, line)) continue else: @@ -1260,7 +1286,7 @@ class EntrySet(Debuggable): if event.filename == 'info.xml': self.infoxml = None elif event.filename in [':info', 'info']: - self.metadata = default_file_metadata.copy() + self.metadata = DEFAULT_FILE_METADATA.copy() def bind_info_to_entry(self, entry, metadata): """ Shortcut to call :func:`bind_info` with the base diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 87f6ff1bd..f87f6b039 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -7,8 +7,9 @@ import threading import lxml.etree import Bcfg2.Server from Bcfg2.Compat import Queue, Empty, Full, cPickle -from exceptions import * -from base import Plugin +from Bcfg2.Server.Plugin.base import Plugin +from Bcfg2.Server.Plugin.exceptions import PluginInitError, \ + MetadataRuntimeError, MetadataConsistencyError class Generator(object): @@ -31,7 +32,7 @@ class Generator(object): :func:`HandleEntry`. """ - def HandlesEntry(self, entry, metadata): + def HandlesEntry(self, entry, metadata): # pylint: disable=W0613 """ HandlesEntry is the slow path method for routing configuration binding requests. It is called if the ``Entries`` dict does not contain a method for binding the @@ -46,7 +47,7 @@ class Generator(object): """ return False - def HandleEntry(self, entry, metadata): + def HandleEntry(self, entry, metadata): # pylint: disable=W0613 """ HandleEntry is the slow path method for binding configuration binding requests. It is called if the ``Entries`` dict does not contain a method for binding the @@ -104,7 +105,7 @@ class Metadata(object): :type colors: list of strings :return: string """ - return '' + raise NotImplementedError def set_version(self, client, version): """ Set the version for the named client to the specified @@ -136,6 +137,7 @@ class Metadata(object): """ pass + # pylint: disable=W0613 def resolve_client(self, address, cleanup_cache=False): """ Resolve the canonical name of this client. If this method is not implemented, the hostname claimed by the client is @@ -153,6 +155,7 @@ class Metadata(object): :class:`Bcfg2.Server.Plugin.exceptions.MetadataConsistencyError` """ return address[1] + # pylint: enable=W0613 def AuthenticateConnection(self, cert, user, password, address): """ Authenticate the given client. @@ -215,7 +218,7 @@ class Connector(object): """ Connector plugins augment client metadata instances with additional data, additional groups, or both. """ - def get_additional_groups(self, metadata): + def get_additional_groups(self, metadata): # pylint: disable=W0613 """ Return a list of additional groups for the given client. :param metadata: The client metadata @@ -224,7 +227,7 @@ class Connector(object): """ return list() - def get_additional_data(self, metadata): + def get_additional_data(self, metadata): # pylint: disable=W0613 """ Return arbitrary additional data for the given ClientMetadata object. By convention this is usually a dict object, but doesn't need to be. @@ -318,10 +321,12 @@ class ThreadedStatistics(Statistics, threading.Thread): while not self.work_queue.empty(): (metadata, data) = self.work_queue.get_nowait() try: - pending_data.append((metadata.hostname, - lxml.etree.tostring(data, - xml_declaration=False).decode("UTF-8"))) - except: + pending_data.append( + (metadata.hostname, + lxml.etree.tostring( + data, + xml_declaration=False).decode("UTF-8"))) + except Full: err = sys.exc_info()[1] self.logger.warning("Dropping interaction for %s: %s" % (metadata.hostname, err)) @@ -333,7 +338,7 @@ class ThreadedStatistics(Statistics, threading.Thread): cPickle.dump(pending_data, savefile) savefile.close() self.logger.info("Saved pending %s data" % self.name) - except: + except (IOError, TypeError): err = sys.exc_info()[1] self.logger.warning("Failed to save pending data: %s" % err) @@ -346,9 +351,9 @@ class ThreadedStatistics(Statistics, threading.Thread): savefile = open(self.pending_file, 'r') pending_data = cPickle.load(savefile) savefile.close() - except Exception: - e = sys.exc_info()[1] - self.logger.warning("Failed to load pending data: %s" % e) + except (IOError, cPickle.UnpicklingError): + err = sys.exc_info()[1] + self.logger.warning("Failed to load pending data: %s" % err) return False for (pmetadata, pdata) in pending_data: # check that shutdown wasnt called early @@ -367,9 +372,9 @@ class ThreadedStatistics(Statistics, threading.Thread): if self.terminate.isSet(): return False - self.work_queue.put_nowait((metadata, - lxml.etree.XML(pdata, - parser=Bcfg2.Server.XMLParser))) + self.work_queue.put_nowait( + (metadata, + lxml.etree.XML(pdata, parser=Bcfg2.Server.XMLParser))) except Full: self.logger.warning("Queue.Full: Failed to load queue data") break @@ -382,7 +387,7 @@ class ThreadedStatistics(Statistics, threading.Thread): "interaction: %s" % pmetadata) try: os.unlink(self.pending_file) - except: + except OSError: self.logger.error("Failed to unlink save file: %s" % self.pending_file) self.logger.info("Loaded pending %s data" % self.name) @@ -397,8 +402,8 @@ class ThreadedStatistics(Statistics, threading.Thread): except Empty: continue except Exception: - e = sys.exc_info()[1] - self.logger.error("ThreadedStatistics: %s" % e) + err = sys.exc_info()[1] + self.logger.error("ThreadedStatistics: %s" % err) continue self.handle_statistic(client, xdata) if self.work_queue != None and not self.work_queue.empty(): @@ -427,8 +432,11 @@ class ThreadedStatistics(Statistics, threading.Thread): raise NotImplementedError +# pylint: disable=C0111 +# Someone who understands these interfaces better needs to write docs +# for PullSource and PullTarget class PullSource(object): - def GetExtra(self, client): + def GetExtra(self, client): # pylint: disable=W0613 return [] def GetCurrentEntry(self, client, e_type, e_name): @@ -441,6 +449,7 @@ class PullTarget(object): def AcceptPullData(self, specific, new_entry, verbose): raise NotImplementedError +# pylint: enable=C0111 class Decision(object): @@ -505,6 +514,32 @@ class GoalValidator(object): class Version(object): """ Version plugins interact with various version control systems. """ + #: The path to the VCS metadata file or directory, relative to the + #: base of the Bcfg2 repository. E.g., for Subversion this would + #: be ".svn" + __vcs_metadata_path__ = None + + def __init__(self, datastore): + """ + :param datastore: The path to the Bcfg2 repository on the + filesystem + :type datastore: string + :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` + + .. autoattribute:: __vcs_metadata_path__ + """ + + self.datastore = datastore + if self.__vcs_metadata_path__: + self.vcs_path = os.path.join(datastore, self.__vcs_metadata_path__) + + if os.path.exists(self.vcs_path): + self.get_revision() + else: + raise PluginInitError("%s is not present" % self.vcs_path) + else: + self.vcs_path = None + def get_revision(self): """ Return the current revision of the Bcfg2 specification. This will be included in the ``revision`` attribute of the diff --git a/src/lib/Bcfg2/Server/Plugins/Bzr.py b/src/lib/Bcfg2/Server/Plugins/Bzr.py index a71021cb5..4de204468 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bzr.py +++ b/src/lib/Bcfg2/Server/Plugins/Bzr.py @@ -1,35 +1,36 @@ +""" The Bzr plugin provides a revision interface for Bcfg2 repos using +bazaar. """ + import Bcfg2.Server.Plugin +# pylint: disable=F0401 from bzrlib.workingtree import WorkingTree from bzrlib import errors +# pylint: enable=F0401 -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Bzr') class Bzr(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """Bzr is a version plugin for dealing with Bcfg2 repos.""" - name = 'Bzr' + """ The Bzr plugin provides a revision interface for Bcfg2 repos + using bazaar. """ __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # Read revision from bcfg2 repo - revision = self.get_revision() - - logger.debug("Initialized Bazaar plugin with directory = %(dir)s at revision = %(rev)s" % {'dir': datastore, 'rev': revision}) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized Bazaar plugin with directory %s at " + "revision = %s" % (self.datastore, + self.get_revision())) def get_revision(self): """Read Bazaar revision information for the Bcfg2 repository.""" try: working_tree = WorkingTree.open(self.datastore) revision = str(working_tree.branch.revno()) - if working_tree.has_changes(working_tree.basis_tree()) or working_tree.unknowns(): + if (working_tree.has_changes(working_tree.basis_tree()) or + working_tree.unknowns()): revision += "+" except errors.NotBranchError: - logger.error("Failed to read Bazaar branch; disabling Bazaar support") - raise Bcfg2.Server.Plugin.PluginInitError + msg = "Failed to read Bazaar branch" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) return revision diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index a0e999847..7f02d4a05 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -2,18 +2,17 @@ <http://www.cheetahtemplate.org/>`_ templating system to generate :ref:`server-plugins-generators-cfg` files. """ -import copy import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgGenerator -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) try: from Cheetah.Template import Template - have_cheetah = True + HAS_CHEETAH = True except ImportError: - have_cheetah = False + HAS_CHEETAH = False class CfgCheetahGenerator(CfgGenerator): @@ -29,9 +28,9 @@ class CfgCheetahGenerator(CfgGenerator): def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) - if not have_cheetah: - msg = "Cfg: Cheetah is not available: %s" % entry.get("name") - logger.error(msg) + if not HAS_CHEETAH: + msg = "Cfg: Cheetah is not available: %s" % self.name + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) __init__.__doc__ = CfgGenerator.__init__.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py index 409d2cbf6..00b95c970 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py @@ -7,7 +7,8 @@ import Bcfg2.Server.Plugin from subprocess import Popen, PIPE from Bcfg2.Server.Plugins.Cfg import CfgFilter -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class CfgDiffFilter(CfgFilter): """ CfgDiffFilter applies diffs to plaintext @@ -24,14 +25,15 @@ class CfgDiffFilter(CfgFilter): open(basename, 'w').write(data) os.close(basehandle) - cmd = ["patch", "-u", "-f", basefile.name] + cmd = ["patch", "-u", "-f", basename] patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) stderr = patch.communicate(input=self.data)[1] ret = patch.wait() - output = open(basefile.name, 'r').read() - os.unlink(basefile.name) + output = open(basename, 'r').read() + os.unlink(basename) if ret != 0: - logger.error("Error applying diff %s: %s" % (delta.name, stderr)) - raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta) + msg = "Error applying diff %s: %s" % (self.name, stderr) + LOGGER.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) return output modify_data.__doc__ = CfgFilter.modify_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 6fd70e69f..31c3d79b0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -17,7 +17,7 @@ try: from genshi.template import TemplateLoader except ImportError: # CfgGenshiGenerator will raise errors if genshi doesn't exist - TemplateLoader = object + TemplateLoader = object # pylint: disable=C0103 LOGGER = logging.getLogger(__name__) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index 87e11ab6d..fb66ca8bf 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -7,7 +7,8 @@ import Bcfg2.Server.Plugin from subprocess import Popen, PIPE from Bcfg2.Server.Plugins.Cfg import CfgVerifier, CfgVerificationError -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class CfgExternalCommandVerifier(CfgVerifier): """ Invoke an external script to verify @@ -16,6 +17,11 @@ class CfgExternalCommandVerifier(CfgVerifier): #: Handle :file:`:test` files __basenames__ = [':test'] + def __init__(self, name, specific, encoding): + CfgVerifier.__init__(self, name, specific, encoding) + self.cmd = [] + __init__.__doc__ = CfgVerifier.__init__.__doc__ + def verify_entry(self, entry, metadata, data): proc = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) err = proc.communicate(input=data)[1] @@ -34,8 +40,7 @@ class CfgExternalCommandVerifier(CfgVerifier): self.cmd.extend(shlex.split(bangpath[2:].strip())) else: msg = "Cannot execute %s" % self.name - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) self.cmd.append(self.name) handle_event.__doc__ = CfgVerifier.handle_event.__doc__ - diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index dc128bbe9..21662a984 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -9,16 +9,17 @@ import traceback import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgGenerator -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) try: import genshi.core from genshi.template import TemplateLoader, NewTextTemplate from genshi.template.eval import UndefinedError - have_genshi = True + HAS_GENSHI = True except ImportError: - TemplateLoader = None - have_genshi = False + TemplateLoader = None # pylint: disable=C0103 + HAS_GENSHI = False + def removecomment(stream): """ A Genshi filter that removes comments from the stream. This @@ -42,7 +43,7 @@ class CfgGenshiGenerator(CfgGenerator): #: Handle .genshi files __extensions__ = ['genshi'] - + #: ``__loader_cls__`` is the class that will be instantiated to #: load the template files. It must implement one public function, #: ``load()``, as :class:`genshi.template.TemplateLoader`. @@ -61,9 +62,9 @@ class CfgGenshiGenerator(CfgGenerator): def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) - if not have_genshi: + if not HAS_GENSHI: msg = "Cfg: Genshi is not available: %s" % fname - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) self.loader = self.__loader_cls__() self.template = None @@ -87,55 +88,59 @@ class CfgGenshiGenerator(CfgGenerator): stack = traceback.extract_tb(sys.exc_info()[2]) for quad in stack: if quad[0] == self.name: - logger.error("Cfg: Error rendering %s at '%s': %s: %s" % + LOGGER.error("Cfg: Error rendering %s at '%s': %s: %s" % (fname, quad[2], err.__class__.__name__, err)) break raise except: - # a failure in a %{ python ... %} block -- the snippet in - # the traceback is just the beginning of the block. - err = sys.exc_info()[1] - stack = traceback.extract_tb(sys.exc_info()[2]) - (filename, lineno, func, text) = stack[-1] - # this is horrible, and I deeply apologize to whoever gets - # to maintain this after I go to the Great Beer Garden in - # the Sky. genshi is incredibly opaque about what's being - # executed, so the only way I can find to determine which - # {% python %} block is being executed -- if there are - # multiples -- is to iterate through them and match the - # snippet of the first line that's in the traceback with - # the first non-empty line of the block. - execs = [contents - for etype, contents, loc in self.template.stream - if etype == self.template.EXEC] - contents = None - if len(execs) == 1: - contents = execs[0] - elif len(execs) > 1: - match = pyerror_re.match(func) - if match: - firstline = match.group(0) - for pyblock in execs: - if pyblock.startswith(firstline): - contents = pyblock - break - # else, no EXEC blocks -- WTF? - if contents: - # we now have the bogus block, but we need to get the - # offending line. To get there, we do (line number - # given in the exception) - (firstlineno from the - # internal genshi code object of the snippet) + 1 = - # (line number of the line with an error within the - # block, with all multiple line breaks elided to a - # single line break) - real_lineno = lineno - contents.code.co_firstlineno - src = re.sub(r'\n\n+', '\n', contents.source).splitlines() - logger.error("Cfg: Error rendering %s at '%s': %s: %s" % - (fname, src[real_lineno], err.__class__.__name__, - err)) - raise + self._handle_genshi_exception(fname, sys.exc_info()) get_data.__doc__ = CfgGenerator.get_data.__doc__ + def _handle_genshi_exception(self, fname, exc): + """ this is horrible, and I deeply apologize to whoever gets + to maintain this after I go to the Great Beer Garden in the + Sky. genshi is incredibly opaque about what's being executed, + so the only way I can find to determine which {% python %} + block is being executed -- if there are multiples -- is to + iterate through them and match the snippet of the first line + that's in the traceback with the first non-empty line of the + block. """ + + # a failure in a %{ python ... %} block -- the snippet in + # the traceback is just the beginning of the block. + err = [1] + stack = traceback.extract_tb(exc[2]) + lineno, func = stack[-1][1:3] + execs = [contents + for etype, contents in self.template.stream[:2] + if etype == self.template.EXEC] + contents = None + if len(execs) == 1: + contents = execs[0] + elif len(execs) > 1: + match = self.pyerror_re.match(func) + if match: + firstline = match.group(0) + for pyblock in execs: + if pyblock.startswith(firstline): + contents = pyblock + break + # else, no EXEC blocks -- WTF? + if contents: + # we now have the bogus block, but we need to get the + # offending line. To get there, we do (line number + # given in the exception) - (firstlineno from the + # internal genshi code object of the snippet) + 1 = + # (line number of the line with an error within the + # block, with all multiple line breaks elided to a + # single line break) + real_lineno = lineno - contents.code.co_firstlineno + src = re.sub(r'\n\n+', '\n', contents.source).splitlines() + LOGGER.error("Cfg: Error rendering %s at '%s': %s: %s" % + (fname, src[real_lineno], err.__class__.__name__, + err)) + raise + def handle_event(self, event): if event.code2str() == 'deleted': return @@ -146,6 +151,6 @@ class CfgGenshiGenerator(CfgGenerator): except Exception: msg = "Cfg: Could not load template %s: %s" % (self.name, sys.exc_info()[1]) - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) handle_event.__doc__ = CfgGenerator.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py index 472a7dba3..2396d6eb6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py @@ -4,7 +4,8 @@ import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgInfo -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class CfgInfoXML(CfgInfo): """ CfgInfoXML handles :file:`info.xml` files for @@ -22,7 +23,7 @@ class CfgInfoXML(CfgInfo): mdata = dict() self.infoxml.pnode.Match(metadata, mdata, entry=entry) if 'Info' not in mdata: - logger.error("Failed to set metadata for file %s" % + LOGGER.error("Failed to set metadata for file %s" % entry.get('name')) raise Bcfg2.Server.Plugin.PluginExecutionError self._set_info(entry, mdata['Info'][None]) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py index a47663904..8f71c45c8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py @@ -4,7 +4,8 @@ import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgInfo -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class CfgLegacyInfo(CfgInfo): """ CfgLegacyInfo handles :file:`info` and :file:`:info` files for @@ -20,6 +21,9 @@ class CfgLegacyInfo(CfgInfo): def __init__(self, path): CfgInfo.__init__(self, path) self.path = path + + #: The set of info metadata stored in the file + self.metadata = None __init__.__doc__ = CfgInfo.__init__.__doc__ def bind_info_to_entry(self, entry, metadata): @@ -30,9 +34,10 @@ class CfgLegacyInfo(CfgInfo): if event.code2str() == 'deleted': return for line in open(self.path).readlines(): - match = Bcfg2.Server.Plugin.info_regex.match(line) + match = Bcfg2.Server.Plugin.INFO_REGEX.match(line) if not match: - logger.warning("Failed to parse line in %s: %s" % (fpath, line)) + LOGGER.warning("Failed to parse line in %s: %s" % + (event.filename, line)) continue else: self.metadata = \ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index e2832cd26..c34cad30e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -8,10 +8,12 @@ import logging import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages import Bcfg2.Server.Lint +# pylint: disable=W0622 +from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages +# pylint: enable=W0622 -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) #: SETUP contains a reference to the #: :class:`Bcfg2.Options.OptionParser` created by the Bcfg2 core for @@ -24,6 +26,7 @@ logger = logging.getLogger(__name__) #: the EntrySet children. SETUP = None + class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): """ CfgBaseFileMatcher is the parent class for all Cfg handler objects. """ @@ -93,12 +96,13 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): components = ['^(?P<basename>%s)' % basename] if cls.__specific__: - components.append('(|\\.H_(?P<hostname>\S+?)|.G(?P<prio>\d+)_(?P<group>\S+?))') + components.append('(|\\.H_(?P<hostname>\S+?)|' + + '.G(?P<prio>\d+)_(?P<group>\S+?))') if extensions: components.append('\\.(?P<extension>%s)' % '|'.join(extensions)) components.append('$') return re.compile("".join(components)) - + @classmethod def handles(cls, event, basename=None): """ Return True if this handler handles the file described by @@ -129,7 +133,8 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): match = True break return (match and - cls.get_regex(basename=os.path.basename(basename)).match(event.filename)) + cls.get_regex( + basename=os.path.basename(basename)).match(event.filename)) @classmethod def ignore(cls, event, basename=None): @@ -166,13 +171,10 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): return (match and cls.get_regex(basename=os.path.basename(basename), extensions=cls.__ignore__).match(event.filename)) - + def __str__(self): return "%s(%s)" % (self.__class__.__name__, self.name) - def match(self, fname): - return self.regex.match(fname) - class CfgGenerator(CfgBaseFileMatcher): """ CfgGenerators generate the initial content of a file. Every @@ -193,7 +195,7 @@ class CfgGenerator(CfgBaseFileMatcher): CfgBaseFileMatcher.__init__(self, name, specific, encoding) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] - def get_data(self, entry, metadata): + def get_data(self, entry, metadata): # pylint: disable=W0613 """ get_data() returns the initial data of a file. :param entry: The entry to generate data for. ``entry`` should @@ -335,13 +337,17 @@ class CfgDefaultInfo(CfgInfo): bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ #: A :class:`CfgDefaultInfo` object instantiated with -#: :attr:`Bcfg2.Server.Plugin.helper.default_file_metadata` as its +#: :attr:`Bcfg2.Server.Plugin.helper.DEFAULT_FILE_METADATA` as its #: default metadata. This is used to set a default file metadata set #: on an entry before a "real" :class:`CfgInfo` handler applies its #: metadata to the entry. -DEFAULT_INFO = CfgDefaultInfo(Bcfg2.Server.Plugin.default_file_metadata) +DEFAULT_INFO = CfgDefaultInfo(Bcfg2.Server.Plugin.DEFAULT_FILE_METADATA) + class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): + """ Handle a collection of host- and group-specific Cfg files with + multiple different Cfg handlers in a single directory. """ + def __init__(self, basename, path, entry_type, encoding): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) @@ -377,18 +383,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): :returns: None """ action = event.code2str() - + if event.filename not in self.entries: if action not in ['exists', 'created', 'changed']: # process a bogus changed event like a created return - + for hdlr in self.handlers: if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but # handle it like a 'created' - logger.warning("Got %s event for unknown file %s" % + LOGGER.warning("Got %s event for unknown file %s" % (action, event.filename)) self.debug_log("%s handling %s event on %s" % (hdlr.__name__, action, event.filename)) @@ -403,7 +409,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): del self.entries[event.filename] return - logger.error("Could not process event %s for %s; ignoring" % + LOGGER.error("Could not process event %s for %s; ignoring" % (action, event.filename)) def get_matching(self, metadata): @@ -412,7 +418,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): item.specific.matches(metadata))] get_matching.__doc__ = Bcfg2.Server.Plugin.EntrySet.get_matching.__doc__ - def entry_init(self, event, hdlr): + def entry_init(self, event, hdlr): # pylint: disable=W0221 """ Handle the creation of a file on the filesystem and the creation of a Cfg handler object in this CfgEntrySet to track it. @@ -432,13 +438,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): specific=hdlr.get_regex(os.path.basename(self.path))) else: if event.filename in self.entries: - logger.warn("Got duplicate add for %s" % event.filename) + LOGGER.warn("Got duplicate add for %s" % event.filename) else: fpath = os.path.join(self.path, event.filename) self.entries[event.filename] = hdlr(fpath) self.entries[event.filename].handle_event(event) - def bind_entry(self, entry, metadata): + def bind_entry(self, entry, metadata): # pylint: disable=R0912,R0915 info_handlers = [] generators = [] filters = [] @@ -459,12 +465,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): fdesc = "/".join(ent.__basenames__) elif ent.__extensions__: fdesc = "." + "/.".join(ent.__extensions__) - logger.warning("Cfg: %s: Use of %s files is deprecated" % + LOGGER.warning("Cfg: %s: Use of %s files is deprecated" % (ent.name, fdesc)) DEFAULT_INFO.bind_info_to_entry(entry, metadata) if len(info_handlers) > 1: - logger.error("More than one info supplier found for %s: %s" % + LOGGER.error("More than one info supplier found for %s: %s" % (entry.get("name"), info_handlers)) if len(info_handlers): info_handlers[0].bind_info_to_entry(entry, metadata) @@ -482,7 +488,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): except: msg = "Cfg: exception rendering %s with %s: %s" % \ (entry.get("name"), generator, sys.exc_info()[1]) - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) for fltr in filters: @@ -506,9 +512,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): msg = "Data for %s for %s failed to verify: %s" % \ (entry.get('name'), metadata.hostname, sys.exc_info()[1]) - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - + if entry.get('encoding') == 'base64': data = b64encode(data) else: @@ -518,14 +524,14 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) - logger.error(msg) - logger.error("Please verify you are using the proper encoding.") + LOGGER.error(msg) + LOGGER.error("Please verify you are using the proper encoding") raise Bcfg2.Server.Plugin.PluginExecutionError(msg) except ValueError: msg = "Error in specification for %s: %s" % (entry.get('name'), sys.exc_info()[1]) - logger.error(msg) - logger.error("You need to specify base64 encoding for %s." % + LOGGER.error(msg) + LOGGER.error("You need to specify base64 encoding for %s." % entry.get('name')) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) except TypeError: @@ -547,21 +553,23 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): ent.specific.matches(metadata))] if not generators: msg = "No base file found for %s" % entry.get('name') - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - + rv = [] try: best = self.best_matching(metadata, generators) rv.append(best.specific) - except: + except: # pylint: disable=W0702 pass if not rv or not rv[0].hostname: - rv.append(Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)) + rv.append(Bcfg2.Server.Plugin.Specificity( + hostname=metadata.hostname)) return rv def build_filename(self, specific): + """ Create a filename for pulled file data """ bfname = self.path + '/' + self.path.split('/')[-1] if specific.all: return bfname @@ -571,22 +579,23 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): return "%s.H_%s" % (bfname, specific.hostname) def write_update(self, specific, new_entry, log): + """ Write pulled data to the filesystem """ if 'text' in new_entry: name = self.build_filename(specific) if os.path.exists("%s.genshi" % name): msg = "Cfg: Unable to pull data for genshi types" - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) elif os.path.exists("%s.cheetah" % name): msg = "Cfg: Unable to pull data for cheetah types" - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: etext = new_entry['text'].encode(self.encoding) except: - msg = "Cfg: Cannot encode content of %s as %s" % (name, - self.encoding) - logger.error(msg) + msg = "Cfg: Cannot encode content of %s as %s" % \ + (name, self.encoding) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) open(name, 'w').write(etext) self.debug_log("Wrote file %s" % name, flag=log) @@ -597,7 +606,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): for ifile in ['info', ':info']: info = os.path.join(self.path, ifile) if os.path.exists(info): - logger.info("Removing %s and replacing with info.xml" % + LOGGER.info("Removing %s and replacing with info.xml" % info) os.remove(info) metadata_updates = {} @@ -606,8 +615,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): metadata_updates[attr] = new_entry.get(attr) infoxml = lxml.etree.Element('FileInfo') infotag = lxml.etree.SubElement(infoxml, 'Info') - [infotag.attrib.__setitem__(attr, metadata_updates[attr]) - for attr in metadata_updates] + for attr in metadata_updates: + infotag.attrib.__setitem__(attr, metadata_updates[attr]) ofile = open(self.path + "/info.xml", "w") ofile.write(lxml.etree.tostring(infoxml, xml_declaration=False, pretty_print=True).decode('UTF-8')) @@ -629,10 +638,10 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, es_child_cls = Bcfg2.Server.Plugin.SpecificData def __init__(self, core, datastore): - global SETUP + global SETUP # pylint: disable=W0603 Bcfg2.Server.Plugin.GroupSpool.__init__(self, core, datastore) Bcfg2.Server.Plugin.PullTarget.__init__(self) - + SETUP = core.setup if 'validate' not in SETUP: SETUP.add_option('validate', Bcfg2.Options.CFG_VALIDATION) @@ -665,7 +674,8 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, def AcceptChoices(self, entry, metadata): return self.entries[entry.get('name')].list_accept_choices(entry, metadata) - AcceptChoices.__doc__ = Bcfg2.Server.Plugin.PullTarget.AcceptChoices.__doc__ + AcceptChoices.__doc__ = \ + Bcfg2.Server.Plugin.PullTarget.AcceptChoices.__doc__ def AcceptPullData(self, specific, new_entry, log): return self.entries[new_entry.get('name')].write_update(specific, @@ -689,6 +699,7 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): "diff-file-used":"warning"} def check_entry(self, basename, entry): + """ check that no .cat or .diff files are in use """ cfg = self.core.plugins['Cfg'] for basename, entry in list(cfg.entries.items()): for fname, handler in entry.entries.items(): diff --git a/src/lib/Bcfg2/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py index 6ce72acd2..a36a116f5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cvs.py +++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py @@ -1,34 +1,22 @@ -import os +""" The Cvs plugin provides a revision interface for Bcfg2 repos using +cvs. """ + from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Cvs') class Cvs(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """CVS is a version plugin for dealing with Bcfg2 repository.""" - name = 'Cvs' + """ The Cvs plugin provides a revision interface for Bcfg2 repos + using cvs.""" __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True + __vcs_metadata_path__ = "CVSROOT" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to cvs directory for Bcfg2 repo - cvs_dir = "%s/CVSROOT" % datastore - - # Read revision from Bcfg2 repo - if os.path.isdir(cvs_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % cvs_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized cvs plugin with cvs directory = %s" % cvs_dir) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized cvs plugin with cvs directory %s" % + self.vcs_path) def get_revision(self): """Read cvs revision information for the Bcfg2 repository.""" @@ -37,10 +25,9 @@ class Cvs(Bcfg2.Server.Plugin.Plugin, shell=True, cwd=self.datastore, stdout=PIPE).stdout.readlines() - revision = data[3].strip('\n') + return data[3].strip('\n') except IndexError: - logger.error("Failed to read cvs log; disabling cvs support") - logger.error('''Ran command "cvs log %s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError - + msg = "Failed to read cvs log" + self.logger.error(msg) + self.logger.error('Ran command "cvs log %s"' % self.datastore) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index ea3b1b69e..16e9e4a8a 100644 --- a/src/lib/Bcfg2/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -1,6 +1,6 @@ +""" DBstats provides a database-backed statistics handler """ + import difflib -import logging -import lxml.etree import platform import sys import time @@ -15,13 +15,10 @@ from Bcfg2.Server.Reports.importscript import load_stat from Bcfg2.Server.Reports.reports.models import Client from Bcfg2.Compat import b64decode -# for debugging output only -logger = logging.getLogger('Bcfg2.Plugins.DBStats') - class DBStats(Bcfg2.Server.Plugin.ThreadedStatistics, Bcfg2.Server.Plugin.PullSource): - name = 'DBStats' + """ DBstats provides a database-backed statistics handler """ def __init__(self, core, datastore): Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore) @@ -36,29 +33,29 @@ class DBStats(Bcfg2.Server.Plugin.ThreadedStatistics, newstats.set('time', time.asctime(time.localtime())) start = time.time() - for i in [1, 2, 3]: + for try_count in [1, 2, 3]: try: load_stat(metadata, newstats, self.core.encoding, 0, - logger, + self.logger, True, platform.node()) - logger.info("Imported data for %s in %s seconds" \ - % (metadata.hostname, time.time() - start)) + self.logger.info("Imported data for %s in %s seconds" % + (metadata.hostname, time.time() - start)) return except MultipleObjectsReturned: - e = sys.exc_info()[1] - logger.error("DBStats: MultipleObjectsReturned while " - "handling %s: %s" % (metadata.hostname, e)) - logger.error("DBStats: Data is inconsistent") + err = sys.exc_info()[1] + self.logger.error("DBStats: MultipleObjectsReturned while " + "handling %s: %s" % (metadata.hostname, err)) + self.logger.error("DBStats: Data is inconsistent") break except: - logger.error("DBStats: Failed to write to db (lock); retrying", - exc_info=1) - logger.error("DBStats: Retry limit failed for %s; aborting operation" \ - % metadata.hostname) + self.logger.error("DBStats: Failed to write to db (lock); " + "retrying (try %s)" % try_count, exc_info=1) + self.logger.error("DBStats: Retry limit failed for %s; " + "aborting operation" % metadata.hostname) def GetExtra(self, client): c_inst = Client.objects.filter(name=client)[0] @@ -78,11 +75,11 @@ class DBStats(Bcfg2.Server.Plugin.ThreadedStatistics, entry = result[0] ret = [] data = ('owner', 'group', 'perms') - for t in data: - if getattr(entry.reason, "current_%s" % t) == '': - ret.append(getattr(entry.reason, t)) + for dtype in data: + if getattr(entry.reason, "current_%s" % dtype) == '': + ret.append(getattr(entry.reason, dtype)) else: - ret.append(getattr(entry.reason, "current_%s" % t)) + ret.append(getattr(entry.reason, "current_%s" % dtype)) if entry.reason.is_sensitive: raise Bcfg2.Server.Plugin.PluginExecutionError elif len(entry.reason.unpruned) != 0: diff --git a/src/lib/Bcfg2/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py index 9fb9ff4f1..9ec8e2df3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Darcs.py +++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py @@ -1,35 +1,22 @@ -import os +""" Darcs is a version plugin for dealing with Bcfg2 repos stored in the +Darcs VCS. """ + from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Darcs') class Darcs(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """Darcs is a version plugin for dealing with Bcfg2 repos.""" - name = 'Darcs' + """ Darcs is a version plugin for dealing with Bcfg2 repos stored + in the Darcs VCS. """ __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True + __vcs_metadata_path__ = "_darcs" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to darcs directory for bcfg2 repo - darcs_dir = "%s/_darcs" % datastore - - # Read changeset from bcfg2 repo - if os.path.isdir(darcs_dir): - self.get_revision() - else: - logger.error("%s is not present." % darcs_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized Darcs plugin with darcs directory = %s" % darcs_dir) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized Darcs plugin with darcs directory %s" % + self.vcs_path) def get_revision(self): """Read Darcs changeset information for the Bcfg2 repository.""" @@ -40,9 +27,9 @@ class Darcs(Bcfg2.Server.Plugin.Plugin, stdout=PIPE).stdout.readlines() revision = data[0].strip('\n') except: - logger.error("Failed to read darcs repository; disabling Darcs support") - logger.error('''Ran command "darcs changes" from directory "%s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError + msg = "Failed to read darcs repository" + self.logger.error(msg) + self.logger.error('Ran command "darcs changes" from directory "%s"' + % self.datastore) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) return revision - diff --git a/src/lib/Bcfg2/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py index 718192e2a..53eed3798 100644 --- a/src/lib/Bcfg2/Server/Plugins/Defaults.py +++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py @@ -1,9 +1,9 @@ """This generator provides rule-based entry mappings.""" -import re import Bcfg2.Server.Plugin import Bcfg2.Server.Plugins.Rules + class Defaults(Bcfg2.Server.Plugins.Rules.Rules, Bcfg2.Server.Plugin.StructureValidator): """Set default attributes on bound entries""" @@ -21,7 +21,7 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, return False def HandleEntry(self, entry, metadata): - raise PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError def HandleEvent(self, event): Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index d6e4aadab..59ac9f85e 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -52,10 +52,11 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Probing.__init__(self) - self.config = Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, - 'config.xml'), - fam=core.fam, - should_monitor=True) + self.config = \ + Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, + 'config.xml'), + fam=core.fam, + should_monitor=True) self.entries = dict() self.probes = dict() @@ -89,8 +90,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, for data in datalist: if data.text is None: - self.logger.error("Got null response to %s file probe from %s" % - (data.get('name'), metadata.hostname)) + self.logger.error("Got null response to %s file probe from %s" + % (data.get('name'), metadata.hostname)) else: try: self.write_data( @@ -145,6 +146,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, return def write_file(self, fileloc, contents): + """ Write the probed file to disk """ try: os.makedirs(os.path.dirname(fileloc)) except OSError: @@ -152,8 +154,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, if err.errno == errno.EEXIST: pass else: - self.logger.error("Could not create parent directories for %s: " - "%s" % (fileloc, err)) + self.logger.error("Could not create parent directories for " + "%s: %s" % (fileloc, err)) return try: @@ -164,14 +166,14 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, return def verify_file(self, filename, contents, metadata): - # Service the FAM events queued up by the key generation so - # the data structure entries will be available for binding. - # - # NOTE: We wait for up to ten seconds. There is some potential - # for race condition, because if the file monitor doesn't get - # notified about the new key files in time, those entries - # won't be available for binding. In practice, this seems - # "good enough". + """ Service the FAM events queued up by the key generation so + the data structure entries will be available for binding. + + NOTE: We wait for up to ten seconds. There is some potential + for race condition, because if the file monitor doesn't get + notified about the new key files in time, those entries won't + be available for binding. In practice, this seems "good + enough".""" entry = self.entries[metadata.hostname][filename] cfg = self.core.plugins['Cfg'] tries = 0 @@ -202,16 +204,12 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, return self.logger.info("Writing %s for %s" % (infoxml, data.get("name"))) - info = \ - lxml.etree.Element("Info", - owner=data.get("owner", - Bcfg2.Options.MDATA_OWNER.value), - group=data.get("group", - Bcfg2.Options.MDATA_GROUP.value), - perms=data.get("perms", - Bcfg2.Options.MDATA_PERMS.value), - encoding=entry.get("encoding", - Bcfg2.Options.ENCODING.value)) + info = lxml.etree.Element( + "Info", + owner=data.get("owner", Bcfg2.Options.MDATA_OWNER.value), + group=data.get("group", Bcfg2.Options.MDATA_GROUP.value), + perms=data.get("perms", Bcfg2.Options.MDATA_PERMS.value), + encoding=entry.get("encoding", Bcfg2.Options.ENCODING.value)) root = lxml.etree.Element("FileInfo") root.append(info) diff --git a/src/lib/Bcfg2/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index 1b1627688..85d0f38f5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -1,37 +1,22 @@ -import os +""" The Fossil plugin provides a revision interface for Bcfg2 repos +using fossil.""" + from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Fossil') class Fossil(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """Fossil is a version plugin for dealing with Bcfg2 repos.""" - name = 'Fossil' + """ The Fossil plugin provides a revision interface for Bcfg2 + repos using fossil. """ __author__ = 'bcfg-dev@mcs.anl.gov' + __vcs_metadata_path__ = "_FOSSIL_" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to fossil file for bcfg2 repo - fossil_file = "%s/_FOSSIL_" % datastore - - # Read revision from bcfg2 repo - if os.path.isfile(fossil_file): - revision = self.get_revision() - elif not os.path.isdir(datastore): - logger.error("%s is not a directory" % datastore) - raise Bcfg2.Server.Plugin.PluginInitError - else: - logger.error("%s is not a file" % fossil_file) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized Fossil.py plugin with %(ffile)s at revision %(frev)s" \ - % {'ffile': fossil_file, 'frev': revision}) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized Fossil plugin with fossil directory %s" + % self.vcs_path) def get_revision(self): """Read fossil revision information for the Bcfg2 repository.""" @@ -42,10 +27,10 @@ class Fossil(Bcfg2.Server.Plugin.Plugin, stdout=PIPE).stdout.readlines() revline = [line.split(': ')[1].strip() for line in data if \ line.split(': ')[0].strip() == 'checkout'][-1] - revision = revline.split(' ')[0] + return revline.split(' ')[0] except IndexError: - logger.error("Failed to read fossil info; disabling fossil support") - logger.error('''Ran command "fossil info" from directory "%s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError - return revision + msg = "Failed to read fossil info" + self.logger.error(msg) + self.logger.error('Ran command "fossil info" from directory "%s"' % + self.datastore) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 8f8ea87f1..30416a147 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -1,44 +1,28 @@ -"""The Git plugin provides a revision interface for Bcfg2 repos using git.""" +""" The Git plugin provides a revision interface for Bcfg2 repos using +git. """ -import os -from dulwich.repo import Repo +from dulwich.repo import Repo # pylint: disable=F0401 import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Git') - class Git(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """Git is a version plugin for dealing with Bcfg2 repos.""" - name = 'Git' + """ The Git plugin provides a revision interface for Bcfg2 repos + using git. """ __author__ = 'bcfg-dev@mcs.anl.gov' + __vcs_metadata_path__ = ".git" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to git directory for bcfg2 repo - git_dir = "%s/.git" % datastore - - # Read revision from bcfg2 repo - if os.path.isdir(git_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % git_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized git plugin with git directory %s" % git_dir) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized git plugin with git directory %s" % + self.vcs_path) def get_revision(self): """Read git revision information for the Bcfg2 repository.""" try: - repo = Repo(self.datastore) - revision = repo.head() + return Repo(self.datastore).head() except: - logger.error("Failed to read git repository; disabling git support") - raise Bcfg2.Server.Plugin.PluginInitError - return revision + msg = "Failed to read git repository" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Guppy.py b/src/lib/Bcfg2/Server/Plugins/Guppy.py index eea92f30f..d13e3f061 100644 --- a/src/lib/Bcfg2/Server/Plugins/Guppy.py +++ b/src/lib/Bcfg2/Server/Plugins/Guppy.py @@ -26,16 +26,16 @@ Remote interactive console. To return to Annex, type '-'. """ -import re import Bcfg2.Server.Plugin + class Guppy(Bcfg2.Server.Plugin.Plugin): """Guppy is a debugging plugin to help trace memory leaks""" name = 'Guppy' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = True - __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable','Disable'] + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable', 'Disable'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -59,4 +59,3 @@ class Guppy(Bcfg2.Server.Plugin.Plugin): except: self.logger.error("Failed to disable Heapy") raise Bcfg2.Server.Plugin.PluginInitError - diff --git a/src/lib/Bcfg2/Server/Plugins/Hg.py b/src/lib/Bcfg2/Server/Plugins/Hg.py index 0c3537613..b5dec2e3f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Hg.py +++ b/src/lib/Bcfg2/Server/Plugins/Hg.py @@ -1,45 +1,32 @@ -import os +""" The Hg plugin provides a revision interface for Bcfg2 repos using +mercurial. """ + from mercurial import ui, hg import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Mercurial') class Hg(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Mercurial is a version plugin for dealing with Bcfg2 repository.""" - name = 'Mercurial' + Bcfg2.Server.Plugin.Version): + """ The Hg plugin provides a revision interface for Bcfg2 repos + using mercurial. """ + __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True + __vcs_metadata_path__ = ".hg" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to hg directory for Bcfg2 repo - hg_dir = "%s/.hg" % datastore - - # Read changeset from bcfg2 repo - if os.path.isdir(hg_dir): - self.get_revision() - else: - logger.error("%s is not present." % hg_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized hg plugin with hg directory = %s" % hg_dir) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized hg plugin with hg directory %s" % + self.vcs_path) def get_revision(self): """Read hg revision information for the Bcfg2 repository.""" try: - repo_path = "%s/" % self.datastore + repo_path = self.datastore + "/" repo = hg.repository(ui.ui(), repo_path) tip = repo.changelog.tip() - revision = repo.changelog.rev(tip) + return repo.changelog.rev(tip) except: - logger.error("Failed to read hg repository; disabling mercurial support") - raise Bcfg2.Server.Plugin.PluginInitError - return revision - + msg = "Failed to read hg repository" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index 9883085db..8e5ce2624 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -93,7 +93,7 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): return data except Exception: if hasattr(query, "name"): - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + logger.error("LdapPlugin error: " + "Exception during processing of query named '" + str(query.name) + "', query results will be empty" + @@ -101,7 +101,7 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): for line in traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]): - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + logger.error("LdapPlugin error: " + line.replace("\n", "")) return {} @@ -130,7 +130,7 @@ class LdapConnection(object): result = None for attempt in range(RETRY_COUNT + 1): if attempt >= 1: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + logger.error("LdapPlugin error: " + "LDAP server down (retry " + str(attempt) + "/" + str(RETRY_COUNT) + ")") try: @@ -207,7 +207,7 @@ class LdapQuery(object): self.result = self.process_result(metadata) return self.result else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + logger.error("LdapPlugin error: " + "No valid connection defined for query " + str(self)) return None @@ -240,6 +240,6 @@ class LdapSubQuery(LdapQuery): self.result = self.connection.run_query(self) return self.process_result(metadata, **kwargs) else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + logger.error("LdapPlugin error: " + "No valid connection defined for query " + str(self)) return None diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 5d0b35835..e0904339f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -105,7 +105,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): Bcfg2.Server.FileMonitor.Pseudo) @property - def xdata(self): + def xdata(self): # pylint: disable=E0202 if not self.data: raise Bcfg2.Server.Plugin.MetadataRuntimeError("%s has no data" % self.basefile) @@ -334,7 +334,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" __author__ = 'bcfg-dev@mcs.anl.gov' - name = "Metadata" sort_order = 500 def __init__(self, core, datastore, watch_clients=True): @@ -345,14 +344,14 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.states = dict() self.extra = dict() self.handlers = [] - self._handle_file("groups.xml") + self.groups_xml = self._handle_file("groups.xml") if (self._use_db and os.path.exists(os.path.join(self.data, "clients.xml"))): self.logger.warning("Metadata: database enabled but clients.xml" "found, parsing in compatibility mode") - self._handle_file("clients.xml") + self.clients_xml = self._handle_file("clients.xml") elif not self._use_db: - self._handle_file("clients.xml") + self.clients_xml = self._handle_file("clients.xml") # mapping of clientname -> authtype self.auth = dict() @@ -415,9 +414,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.states[fname] = False aname = re.sub(r'[^A-z0-9_]', '_', fname) xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname) - setattr(self, aname, xmlcfg) self.handlers.append(xmlcfg.HandleEvent) self.extra[fname] = [] + return xmlcfg def _search_xdata(self, tag, name, tree, alias=False): for node in tree.findall("//%s" % tag): @@ -456,9 +455,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def add_group(self, group_name, attribs): """Add group to groups.xml.""" if self._use_db: - msg = "Metadata does not support adding groups with use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support adding groups with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._add_xdata(self.groups_xml, "Group", group_name, attribs=attribs) @@ -466,9 +466,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def add_bundle(self, bundle_name): """Add bundle to groups.xml.""" if self._use_db: - msg = "Metadata does not support adding bundles with use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support adding bundles with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._add_xdata(self.groups_xml, "Bundle", bundle_name) @@ -503,9 +504,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def update_group(self, group_name, attribs): """Update a groups attributes.""" if self._use_db: - msg = "Metadata does not support updating groups with use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support updating groups with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._update_xdata(self.groups_xml, "Group", group_name, attribs) @@ -513,9 +515,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def update_client(self, client_name, attribs): """Update a clients attributes.""" if self._use_db: - msg = "Metadata does not support updating clients with use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support updating clients with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._update_xdata(self.clients_xml, "Client", client_name, attribs, alias=True) @@ -544,16 +547,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def remove_group(self, group_name): """Remove a group.""" if self._use_db: - msg = "Metadata does not support removing groups with use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support removing groups with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._remove_xdata(self.groups_xml, "Group", group_name) def remove_bundle(self, bundle_name): """Remove a bundle.""" if self._use_db: - msg = "Metadata does not support removing bundles with use_database enabled" + msg = "Metadata does not support removing bundles with " + \ + "use_database enabled" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: @@ -652,8 +657,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.warning("%s: Group %s suppressed by " "category %s; %s already a member " "of %s" % - (self.name, gname, category, client, - categories[category])) + (self.name, gname, category, + client, categories[category])) if gname in self.groups: self.groups[gname].warned.append(client) return False @@ -690,7 +695,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership = dict() self.negated_groups = dict() - self.options = dict() + # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is # infinitely recursive, so "all" elements really means _all_ @@ -727,7 +732,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def HandleEvent(self, event): """Handle update events for data files.""" for hdlr in self.handlers: - aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(event.filename)) + aname = re.sub(r'[^A-z0-9_]', '_', + os.path.basename(event.filename)) if hdlr(event): # clear the entire cache when we get an event for any # metadata file @@ -735,7 +741,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, try: proc = getattr(self, "_handle_%s_event" % aname) except AttributeError: - proc = self._handle_default_event + proc = self._handle_default_event # pylint: disable=E1101 proc(event) if False not in list(self.states.values()) and self.debug_flag: @@ -745,8 +751,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, for client, groups in list(self.clientgroups.items()): for group in groups: if group not in self.groups: - self.debug_log("Client %s set as nonexistent group %s" % - (client, group)) + self.debug_log("Client %s set as nonexistent group %s" + % (client, group)) for gname, ginfo in list(self.groups.items()): for group in ginfo.groups: if group not in self.groups: @@ -765,7 +771,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] if not force and not group.is_public: - msg = "Cannot set client %s to private group %s" % (client, profile) + msg = "Cannot set client %s to private group %s" % (client, + profile) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) @@ -831,13 +838,13 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # be faster? curtime = time.time() for addrpair in list(self.session_cache.keys()): - if addresspair[0] == addrpair[0]: - (stamp, _) = self.session_cache[addrpair] - if curtime - stamp > cache_ttl: - del self.session_cache[addrpair] + if addresspair[0] == addrpair[0]: + (stamp, _) = self.session_cache[addrpair] + if curtime - stamp > cache_ttl: + del self.session_cache[addrpair] # return the cached data try: - (stamp, uuid) = self.session_cache[addresspair] + stamp = self.session_cache[addresspair][0] if time.time() - stamp < cache_ttl: return self.session_cache[addresspair][1] except KeyError: @@ -846,7 +853,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, address = addresspair[0] if address in self.addresses: if len(self.addresses[address]) != 1: - err = "Address %s has multiple reverse assignments; a uuid must be used" % address + err = "Address %s has multiple reverse assignments; a " + \ + "uuid must be used" % address self.logger.error(err) raise Bcfg2.Server.Plugin.MetadataConsistencyError(err) return self.addresses[address][0] @@ -1165,8 +1173,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, try: groups_tree.xinclude() except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for file %s: %s" % - (dest, sys.exc_info()[1])) + self.logger.error("Failed to process XInclude for groups.xml: %s" % + sys.exc_info()[1]) groups = groups_tree.getroot() categories = {'default': 'grey83'} viz_str = [] @@ -1253,7 +1261,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): def deprecated_options(self): clientdata = self.metadata.clients_xml.xdata - for el in groupdata.xpath("//Client"): + for el in clientdata.xpath("//Client"): loc = el.get("location") if loc: if loc == "floating": diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index d0e4a3665..bb98fe1e5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -78,8 +78,7 @@ import copy import logging import lxml.etree import Bcfg2.Server.Plugin -from Bcfg2.Compat import any, md5 -from Bcfg2.Server.Plugins.Packages.Source import Source +from Bcfg2.Compat import any, md5 # pylint: disable=W0622 LOGGER = logging.getLogger(__name__) @@ -180,15 +179,18 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): return "\n".join(srcs) def get_relevant_groups(self): - groups = [] - for source in self: - groups.extend(source.get_relevant_groups(self.metadata)) - return sorted(list(set(groups))) - get_relevant_groups.__doc__ = Source.get_relevant_groups.__doc__ + """ + """ Get all groups that might be relevant to determining which + sources apply to this collection's client. The base implementation simply aggregates the results of :func:`Bcfg2.Server.Plugins.Packages.Source.Source.get_relevant_groups` + + :return: list of strings - group names """ + groups = [] + for source in self: + groups.extend(source.get_relevant_groups(self.metadata)) + return sorted(list(set(groups))) @property def basegroups(self): @@ -253,8 +255,8 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): :ref:`pkg-objects` """ if not self.__package_groups__: - self.logger.error("Packages: Package groups are not supported by %s" - % self.__class__.__name__) + self.logger.error("Packages: Package groups are not supported by " + "%s" % self.__class__.__name__) return [] for source in self: @@ -484,7 +486,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ return list(complete.difference(initial)) - def complete(self, packagelist): + def complete(self, packagelist): # pylint: disable=R0912,R0914 """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the @@ -562,7 +564,8 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): if len(vpkg_cache[current]) == 1: self.debug_log("Packages: requirement %s satisfied by %s" % (current, vpkg_cache[current])) - unclassified.update(vpkg_cache[current].difference(examined)) + unclassified.update( + vpkg_cache[current].difference(examined)) satisfied_vpkgs.add(current) else: satisfiers = [item for item in vpkg_cache[current] @@ -623,7 +626,8 @@ def get_collection_class(source_type): try: cclass = getattr(module, source_type.title() + "Collection") except AttributeError: - msg = "Packages: No collection class found for %s sources" % source_type + msg = "Packages: No collection class found for %s sources" % \ + source_type LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) return cclass diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 329dfc394..2519ddff6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -1,3 +1,6 @@ +""" PackagesSources handles the +:ref:`server-plugins-generators-packages` ``sources.xml`` file""" + import os import sys import Bcfg2.Server.Plugin diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 26f3ab92f..22d488121 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -92,7 +92,7 @@ class SourceInitError(Exception): REPO_RE = re.compile(r'(?:pulp/repos/|/RPMS\.|/)([^/]+)/?$') -class Source(Bcfg2.Server.Plugin.Debuggable): +class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 """ ``Source`` objects represent a single <Source> tag in ``sources.xml``. Note that a single Source tag can itself describe multiple repositories (if it uses the "url" attribute @@ -116,7 +116,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`. ptype = None - def __init__(self, basepath, xsource, setup): + def __init__(self, basepath, xsource, setup): # pylint: disable=R0912 """ :param basepath: The base filesystem path under which cache data for this source should be stored @@ -301,7 +301,8 @@ class Source(Bcfg2.Server.Plugin.Debuggable): """ Get all groups that might be relevant to determining which sources apply to this collection's client. - :return: list of strings - group names""" + :return: list of strings - group names + """ return sorted(list(set([g for g in metadata.groups if (g in self.basegroups or g in self.groups or @@ -349,6 +350,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): upstream repository. :type force_update: bool """ + # pylint: disable=W0702 if not force_update: if os.path.exists(self.cachefile): try: @@ -379,6 +381,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): self.logger.error("Packages: Failed to load data for %s: %s" % (self, err)) self.logger.error("Some Packages will be missing") + # pylint: enable=W0702 def get_repo_name(self, url_map): """ Try to find a sensible name for a repository. Since @@ -485,7 +488,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): vdict[key].update(value) return vdict - def is_virtual_package(self, metadata, package): + def is_virtual_package(self, metadata, package): # pylint: disable=W0613 """ Return True if a name is a virtual package (i.e., is a symbol provided by a real package), False otherwise. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 1d7eca808..4224798a8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -59,15 +59,17 @@ import logging import lxml.etree from subprocess import Popen, PIPE import Bcfg2.Server.Plugin +# pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, json, any +# pylint: enable=W0622 from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) -# pylint: disable=E0611 +# pylint: disable=E0611,F0401 try: from pulp.client.consumer.config import ConsumerConfig from pulp.client.api.repository import RepositoryAPI @@ -82,9 +84,9 @@ try: HAS_YUM = True except ImportError: HAS_YUM = False - logger.info("Packages: No yum libraries found; forcing use of internal " + LOGGER.info("Packages: No yum libraries found; forcing use of internal " "dependency resolver") -# pylint: enable=E0611 +# pylint: enable=E0611,F0401 XP = '{http://linux.duke.edu/metadata/common}' RP = '{http://linux.duke.edu/metadata/rpm}' @@ -108,7 +110,7 @@ def _setup_pulp(setup): if not HAS_PULP: msg = "Packages: Cannot create Pulp collection: Pulp libraries " + \ "not found" - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) if PULPSERVER is None: @@ -117,12 +119,12 @@ def _setup_pulp(setup): password = setup.cfp.get("packages:pulp", "password") except ConfigParser.NoSectionError: msg = "Packages: No [pulp] section found in bcfg2.conf" - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) except ConfigParser.NoOptionError: msg = "Packages: Required option not found in bcfg2.conf: %s" % \ sys.exc_info()[1] - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) PULPCONFIG = ConsumerConfig() @@ -265,7 +267,7 @@ class YumCollection(Collection): yumconf.write(open(self.cfgfile, 'w')) - def get_config(self, raw=False): + def get_config(self, raw=False): # pylint: disable=W0221 """ Get the yum configuration for this collection. :param raw: Return a :class:`ConfigParser.SafeConfigParser` @@ -528,7 +530,7 @@ class YumCollection(Collection): ptype = "default" gdicts.append(dict(group=group, type=ptype)) - return self.call_helper("get_groups", gdicts) + return self.call_helper("get_groups", inputdata=gdicts) def packages_from_entry(self, entry): """ When using the Python yum libraries, convert a Package @@ -543,6 +545,7 @@ class YumCollection(Collection): name = entry.get("name") def _tag_to_pkg(tag): + """ Convert a Package or Instance tag to a package tuple """ rv = (name, tag.get("arch"), tag.get("epoch"), tag.get("version"), tag.get("release")) if rv[3] in ['any', 'auto']: @@ -584,6 +587,9 @@ class YumCollection(Collection): :returns: None """ def _get_entry_attrs(pkgtup): + """ Given a package tuple, return a dict of attributes + suitable for applying to either a Package or an Instance + tag """ attrs = dict(version=self.setup.cfp.get("packages", "version", default="auto")) @@ -685,7 +691,7 @@ class YumCollection(Collection): else: return set(), set() - def call_helper(self, command, input=None): + def call_helper(self, command, inputdata=None): """ Make a call to :ref:`bcfg2-yum-helper`. The yum libs have horrific memory leaks, so apparently the right way to get around that in long-running processes it to have a short-lived @@ -694,10 +700,10 @@ class YumCollection(Collection): :param command: The :ref:`bcfg2-yum-helper` command to call. :type command: string - :param input: The input to pass to ``bcfg2-yum-helper`` on - stdin. If this is None, no input will be given - at all. - :type input: Any JSON-encodable data structure. + :param inputdata: The input to pass to ``bcfg2-yum-helper`` on + stdin. If this is None, no input will be + given at all. + :type inputdata: Any JSON-encodable data structure. :returns: Varies depending on the return value of the ``bcfg2-yum-helper`` command. """ @@ -715,8 +721,8 @@ class YumCollection(Collection): (" ".join(cmd), err)) return None - if input: - idata = json.dumps(input) + if inputdata: + idata = json.dumps(inputdata) (stdout, stderr) = helper.communicate(idata) else: (stdout, stderr) = helper.communicate() @@ -938,7 +944,7 @@ class YumSource(Source): try: self.packages['global'] = copy.deepcopy(sdata.pop()) except IndexError: - logger.error("Packages: No packages in repo") + self.logger.error("Packages: No packages in repo") while sdata: self.packages['global'] = \ self.packages['global'].intersection(sdata.pop()) @@ -951,6 +957,7 @@ class YumSource(Source): self.save_state() def parse_filelist(self, data, arch): + """ parse filelists.xml.gz data """ if arch not in self.filemap: self.filemap[arch] = dict() for pkg in data.findall(FL + 'package'): @@ -963,6 +970,7 @@ class YumSource(Source): set([pkg.get('name')]) def parse_primary(self, data, arch): + """ parse primary.xml.gz data """ if arch not in self.packages: self.packages[arch] = set() if arch not in self.deps: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index fd6369619..6f8e3ecad 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -1,3 +1,7 @@ +""" Packages resolves Package entries on the Bcfg2 server in order to +present a complete list of Package entries to the client in order to +determine the completeness of the client configuration. """ + import os import sys import glob @@ -68,6 +72,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.logger.warning("You can disable magic groups by setting " "magic_groups=0 in [packages] in bcfg2.conf") + # pylint: disable=C0301 #: The #: :class:`Bcfg2.Server.Plugins.Packages.PackagesSources.PackagesSources` #: object used to generate @@ -98,12 +103,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.collections = dict() #: clients is a cache mapping of hostname -> - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`. + #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` #: Unlike :attr:`collections`, this _is_ used to return a #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = dict() + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 68cdce6e8..9e6b43d7b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -1,3 +1,5 @@ +""" A plugin to gather information from a client machine """ + import re import os import sys @@ -7,29 +9,15 @@ import operator import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin -from Bcfg2.Compat import any, json +from Bcfg2.Compat import json +# pylint: disable=F0401 try: from django.db import models - has_django = True -except ImportError: - has_django = False -try: - import syck as yaml - has_yaml = True - yaml_error = yaml.error -except ImportError: - try: - import yaml - yaml_error = yaml.YAMLError - has_yaml = True - except ImportError: - has_yaml = False - -if has_django: class ProbesDataModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): + """ The database model for storing probe data """ hostname = models.CharField(max_length=255) probe = models.CharField(max_length=255) timestamp = models.DateTimeField(auto_now=True) @@ -37,8 +25,24 @@ if has_django: class ProbesGroupsModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): + """ The database model for storing probe groups """ hostname = models.CharField(max_length=255) group = models.CharField(max_length=255) +except ImportError: + pass + +try: + import syck as yaml + import syck.error as YAMLError + HAS_YAML = True +except ImportError: + try: + import yaml + from yaml import YAMLError + HAS_YAML = True + except ImportError: + HAS_YAML = False +# pylint: enable=F0401 class ClientProbeDataSet(dict): @@ -58,8 +62,8 @@ class ProbeData(str): ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): return str.__new__(cls, data) - - def __init__(self, data): + + def __init__(self): str.__init__(self) self._xdata = None self._json = None @@ -70,9 +74,10 @@ class ProbeData(str): """ provide backwards compatibility with broken ProbeData object in bcfg2 1.2.0 thru 1.2.2 """ return str(self) - + @property def xdata(self): + """ The probe data as a lxml.etree._Element document """ if self._xdata is None: try: self._xdata = lxml.etree.XML(self.data, @@ -83,6 +88,7 @@ class ProbeData(str): @property def json(self): + """ The probe data as a decoded JSON data structure """ if self._json is None: try: self._json = json.loads(self.data) @@ -92,17 +98,20 @@ class ProbeData(str): @property def yaml(self): - if self._yaml is None and has_yaml: + """ The probe data as a decoded YAML data structure """ + if self._yaml is None and HAS_YAML: try: self._yaml = yaml.load(self.data) - except yaml_error: + except YAMLError: pass return self._yaml class ProbeSet(Bcfg2.Server.Plugin.EntrySet): + """ Handle universal and group- and host-specific probe files """ ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") - probename = re.compile("(.*/)?(?P<basename>\S+?)(\.(?P<mode>(?:G\d\d)|H)_\S+)?$") + probename = \ + re.compile("(.*/)?(?P<basename>\S+?)(\.(?P<mode>(?:G\d\d)|H)_\S+)?$") bangline = re.compile('^#!\s*(?P<interpreter>.*)$') basename_is_regex = True @@ -113,12 +122,15 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): encoding) fam.AddMonitor(path, self) - def HandleEvent(self, event): - if (event.filename != self.path and - not event.filename.endswith("probed.xml")): - return self.handle_event(event) - def get_probe_data(self, metadata): + """ Get an XML description of all probes for a client suitable + for sending to that client. + + :params metadata: The client metadata to get probes for. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: list of lxml.etree._Element objects, each of which + represents one probe. + """ ret = [] build = dict() candidates = self.get_matching(metadata) @@ -146,8 +158,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): - """A plugin to gather information from a client machine.""" - name = 'Probes' + """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): @@ -165,33 +176,37 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.probedata = dict() self.cgroups = dict() self.load_data() + __init__.__doc__ = Bcfg2.Server.Plugin.DatabaseBacked.__init__.__doc__ def write_data(self, client): - """Write probe data out for use with bcfg2-info.""" + """ Write probe data out for use with bcfg2-info """ if self._use_db: return self._write_data_db(client) else: return self._write_data_xml(client) def _write_data_xml(self, _): + """ Write received probe data to probed.xml """ top = lxml.etree.Element("Probed") for client, probed in sorted(self.probedata.items()): - cx = lxml.etree.SubElement(top, 'Client', name=client, - timestamp=str(int(probed.timestamp))) + ctag = lxml.etree.SubElement(top, 'Client', name=client, + timestamp=str(int(probed.timestamp))) for probe in sorted(probed): - lxml.etree.SubElement(cx, 'Probe', name=probe, + lxml.etree.SubElement(ctag, 'Probe', name=probe, value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): - lxml.etree.SubElement(cx, "Group", name=group) + lxml.etree.SubElement(ctag, "Group", name=group) try: datafile = open(os.path.join(self.data, 'probed.xml'), 'w') - datafile.write(lxml.etree.tostring(top, xml_declaration=False, - pretty_print='true').decode('UTF-8')) + datafile.write(lxml.etree.tostring( + top, xml_declaration=False, + pretty_print='true').decode('UTF-8')) except IOError: err = sys.exc_info()[1] self.logger.error("Failed to write probed.xml: %s" % err) def _write_data_db(self, client): + """ Write received probe data to the database """ for probe, data in self.probedata[client.hostname].items(): pdata = \ ProbesDataModel.objects.get_or_create(hostname=client.hostname, @@ -199,7 +214,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, if pdata.data != data: pdata.data = data pdata.save() - ProbesDataModel.objects.filter(hostname=client.hostname).exclude(probe__in=self.probedata[client.hostname]).delete() + ProbesDataModel.objects.filter( + hostname=client.hostname).exclude( + probe__in=self.probedata[client.hostname]).delete() for group in self.cgroups[client.hostname]: try: @@ -209,19 +226,24 @@ class Probes(Bcfg2.Server.Plugin.Probing, grp = ProbesGroupsModel(hostname=client.hostname, group=group) grp.save() - ProbesGroupsModel.objects.filter(hostname=client.hostname).exclude(group__in=self.cgroups[client.hostname]).delete() + ProbesGroupsModel.objects.filter( + hostname=client.hostname).exclude( + group__in=self.cgroups[client.hostname]).delete() def load_data(self): + """ Load probe data from the appropriate backend (probed.xml + or the database) """ if self._use_db: return self._load_data_db() else: return self._load_data_xml() - + def _load_data_xml(self): + """ Load probe data from probed.xml """ try: data = lxml.etree.parse(os.path.join(self.data, 'probed.xml'), parser=Bcfg2.Server.XMLParser).getroot() - except: + except (IOError, lxml.etree.XMLSyntaxError): err = sys.exc_info()[1] self.logger.error("Failed to read file probed.xml: %s" % err) return @@ -239,21 +261,22 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[client.get('name')].append(pdata.get('name')) def _load_data_db(self): + """ Load probe data from the database """ self.probedata = {} self.cgroups = {} for pdata in ProbesDataModel.objects.all(): if pdata.hostname not in self.probedata: - self.probedata[pdata.hostname] = \ - ClientProbeDataSet(timestamp=time.mktime(pdata.timestamp.timetuple())) + self.probedata[pdata.hostname] = ClientProbeDataSet( + timestamp=time.mktime(pdata.timestamp.timetuple())) self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) for pgroup in ProbesGroupsModel.objects.all(): if pgroup.hostname not in self.cgroups: self.cgroups[pgroup.hostname] = [] self.cgroups[pgroup.hostname].append(pgroup.group) - def GetProbes(self, meta, force=False): - """Return a set of probes for execution on client.""" + def GetProbes(self, meta): return self.probes.get_probe_data(meta) + GetProbes.__doc__ = Bcfg2.Server.Plugin.Probing.GetProbes.__doc__ def ReceiveData(self, client, datalist): if self.core.metadata_cache_mode in ['cautious', 'aggressive']: @@ -271,6 +294,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, olddata != self.cgroups[client.hostname]): self.core.metadata_cache.expire(client.hostname) self.write_data(client) + ReceiveData.__doc__ = Bcfg2.Server.Plugin.Probing.ReceiveData.__doc__ def ReceiveDataItem(self, client, data): """Receive probe results pertaining to client.""" @@ -299,6 +323,10 @@ class Probes(Bcfg2.Server.Plugin.Probing, def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) + get_additional_groups.__doc__ = \ + Bcfg2.Server.Plugin.Connector.get_additional_groups.__doc__ def get_additional_data(self, meta): return self.probedata.get(meta.hostname, ClientProbeDataSet()) + get_additional_data.__doc__ = \ + Bcfg2.Server.Plugin.Connector.get_additional_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 590d536a9..1d5bdbcfc 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -1,3 +1,6 @@ +""" The properties plugin maps property files into client metadata +instances. """ + import os import re import sys @@ -89,6 +92,7 @@ class PropertyFile(Bcfg2.Server.Plugin.StructFile): raise PluginExecutionError(msg) def _decrypt(self, element): + """ Decrypt a single encrypted properties file element """ if not element.text.strip(): return passes = get_passphrases(SETUP) @@ -108,6 +112,7 @@ class PropertyFile(Bcfg2.Server.Plugin.StructFile): class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): + """ A collection of properties files """ __child__ = PropertyFile patterns = re.compile(r'.*\.xml$') ignore = re.compile(r'.*\.xsd$') @@ -115,22 +120,20 @@ class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): class Properties(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): - """ - The properties plugin maps property - files into client metadata instances. - """ + """ The properties plugin maps property files into client metadata + instances. """ name = 'Properties' def __init__(self, core, datastore): - global SETUP + global SETUP # pylint: disable=W0603 Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) try: self.store = PropDirectoryBacked(self.data, core.fam) except OSError: - e = sys.exc_info()[1] - self.logger.error("Error while creating Properties store: %s %s" % - (e.strerror, e.filename)) + err = sys.exc_info()[1] + self.logger.error("Error while creating Properties store: %s" % + err) raise Bcfg2.Server.Plugin.PluginInitError SETUP = core.setup diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 341d63118..ebcbf3bcc 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -1,8 +1,12 @@ +""" A plugin to run Puppet external node classifiers """ + import os +import sys import Bcfg2.Server import Bcfg2.Server.Plugin from subprocess import Popen, PIPE +# pylint: disable=F0401 try: from syck import load as yaml_load, error as yaml_error except ImportError: @@ -10,8 +14,12 @@ except ImportError: from yaml import load as yaml_load, YAMLError as yaml_error except ImportError: raise ImportError("No yaml library could be found") +# pylint: enable=F0401 + class PuppetENCFile(Bcfg2.Server.Plugin.FileBacked): + """ A representation of a Puppet external node classifier script """ + def HandleEvent(self, event=None): return @@ -22,7 +30,6 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): """ A plugin to run Puppet external node classifiers (http://docs.puppetlabs.com/guides/external_nodes.html) """ - name = 'PuppetENC' experimental = True __child__ = PuppetENCFile @@ -35,6 +42,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, self.cache = dict() def _run_encs(self, metadata): + """ Run all Puppet ENCs """ cache = dict(groups=[], params=dict()) for enc in self.entries.keys(): epath = os.path.join(self.data, enc) @@ -46,8 +54,8 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, rv = proc.wait() if rv != 0: msg = "PuppetENC: Error running ENC %s for %s (%s): %s" % \ - (enc, metadata.hostname, rv) - self.logger.error("%s: %s" % (msg, err)) + (enc, metadata.hostname, rv, err) + self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) if err: self.debug_log("ENC Error: %s" % err) @@ -62,8 +70,8 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, (enc, metadata.hostname, err) self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - groups = [] + + groups = dict() if "classes" in yaml: # stock Puppet ENC output format groups = yaml['classes'] @@ -87,7 +95,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, if "environment" in yaml: self.logger.info("Ignoring unsupported environment section of " "ENC %s for %s" % (enc, metadata.hostname)) - + self.cache[metadata.hostname] = cache def get_additional_groups(self, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index e77d08653..904876794 100644 --- a/src/lib/Bcfg2/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -3,6 +3,7 @@ import re import Bcfg2.Server.Plugin + class Rules(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles service assignments.""" name = 'Rules' @@ -48,4 +49,5 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): return False def _regex_enabled(self): + """ Return True if rules regexes are enabled, False otherwise """ return self.core.setup.cfp.getboolean("rules", "regex", default=False) diff --git a/src/lib/Bcfg2/Server/Plugins/SEModules.py b/src/lib/Bcfg2/Server/Plugins/SEModules.py index b04e2d359..3edfb72a3 100644 --- a/src/lib/Bcfg2/Server/Plugins/SEModules.py +++ b/src/lib/Bcfg2/Server/Plugins/SEModules.py @@ -10,12 +10,9 @@ See :ref:`server-selinux` for more information. """ import os -import logging import Bcfg2.Server.Plugin from Bcfg2.Compat import b64encode -logger = logging.getLogger(__name__) - class SEModuleData(Bcfg2.Server.Plugin.SpecificData): """ Representation of a single SELinux module file. Encodes the diff --git a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py index f1309412a..0aea439f9 100644 --- a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py @@ -1,5 +1,8 @@ +""" Use old-style service modes for older clients """ + import Bcfg2.Server.Plugin + class ServiceCompat(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.GoalValidator): """ Use old-style service modes for older clients """ diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index ae43388ea..5bc9d6bbd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -1,35 +1,23 @@ -import os +""" The Svn plugin provides a revision interface for Bcfg2 repos using +svn. """ + import pipes from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Svn') - class Svn(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """Svn is a version plugin for dealing with Bcfg2 repos.""" - name = 'Svn' + """ The Svn plugin provides a revision interface for Bcfg2 repos + using svn. """ __author__ = 'bcfg-dev@mcs.anl.gov' + __vcs_metadata_path__ = ".svn" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to svn directory for bcfg2 repo - svn_dir = "%s/.svn" % datastore - - # Read revision from bcfg2 repo - if os.path.isdir(svn_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % svn_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized svn plugin with svn directory = %s" % svn_dir) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized svn plugin with svn directory %s" % + self.vcs_path) def get_revision(self): """Read svn revision information for the Bcfg2 repository.""" @@ -40,7 +28,7 @@ class Svn(Bcfg2.Server.Plugin.Plugin, return [line.split(': ')[1] for line in data \ if line[:9] == 'Revision:'][-1] except IndexError: - logger.error("Failed to read svn info; disabling svn support") - logger.error('''Ran command "svn info %s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError + msg = "Failed to read svn info" + self.logger.error(msg) + self.logger.error('Ran command "svn info %s"' % self.datastore) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Svn2.py b/src/lib/Bcfg2/Server/Plugins/Svn2.py index e4df9574f..110f2c2eb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn2.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn2.py @@ -1,41 +1,47 @@ +""" The Svn2 plugin provides a revision interface for Bcfg2 repos using +Subversion. It uses a pure python backend and exposes two additional +XML-RPC methods. """ + +import sys +import Bcfg2.Server.Plugin +# pylint: disable=F0401 try: import pysvn - missing = False -except: - missing = True -import Bcfg2.Server.Plugin + HAS_SVN = False +except ImportError: + HAS_SVN = True +# pylint: enable=F0401 + class Svn2(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" - name = 'Svn2' __author__ = 'bcfg-dev@mcs.anl.gov' conflicts = ['Svn'] - experimental = True - __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Update','Commit'] + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Update', 'Commit'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) - if missing: - self.logger.error("Svn2: Missing PySvn") - raise Bcfg2.Server.Plugin.PluginInitError + if HAS_SVN: + msg = "Missing PySvn" + self.logger.error("Svn2: " + msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) self.client = pysvn.Client() - self.core = core - self.datastore = datastore self.svn_root = None self.revision = None # Read revision from bcfg2 repo revision = self.get_revision() if not self.revision: - raise Bcfg2.Server.Plugin.PluginInitError + raise Bcfg2.Server.Plugin.PluginInitError("Failed to get revision") - self.logger.debug("Initialized svn plugin with svn root %s at revision %s" - % (self.svn_root, revision)) + self.logger.debug("Initialized svn plugin with svn root %s at " + "revision %s" % (self.svn_root, revision)) def get_revision(self): """Read svn revision information for the Bcfg2 repository.""" @@ -44,55 +50,18 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, self.revision = info.revision self.svn_root = info.url return str(self.revision.number) - except: + except pysvn.ClientError: # pylint: disable=E1101 self.logger.error("Svn2: Failed to get revision", exc_info=1) self.revision = None return str(-1) - def commit_data(self, file_list, comment=None): - """Commit changes into the repository""" - if not comment: - comment = 'Svn2: autocommit' - - # First try to update - if not self.Update(): - self.logger.error("Failed to update svn repository, refusing to commit changes") - return - - #FIXME - look for conflicts? - - for fname in file_list: - stat = self.client.status(fname) - self.client.add([f.path for f in stat \ - if f.text_status == pysvn.wc_status_kind.unversioned]) - try: - self.revision = self.client.checkin([self.datastore], comment, - recurse=True) - self.revision = self.client.update(self.datastore, recurse=True)[0] - self.logger.info("Svn2: Commited changes. At %s" % - self.revision.number) - except Exception: - err = sys.exc_info()[1] - # try to be smart about the error we got back - details = None - if "callback_ssl_server_trust_prompt" in str(err): - details = "SVN server certificate is not trusted" - elif "callback_get_login" in str(err): - details = "SVN credentials not cached" - - if details is None: - self.logger.error("Svn2: Failed to commit changes", - exc_info=1) - else: - self.logger.error("Svn2: Failed to commit changes: %s" % - details) - def Update(self): '''Svn2.Update() => True|False\nUpdate svn working copy\n''' try: old_revision = self.revision.number self.revision = self.client.update(self.datastore, recurse=True)[0] - except Exception, err: + except pysvn.ClientError: # pylint: disable=E1101 + err = sys.exc_info()[1] # try to be smart about the error we got back details = None if "callback_ssl_server_trust_prompt" in str(err): @@ -104,8 +73,8 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, self.logger.error("Svn2: Failed to update server repository", exc_info=1) else: - self.logger.error("Svn2: Failed to update server repository: %s" % - details) + self.logger.error("Svn2: Failed to update server repository: " + "%s" % details) return False if old_revision == self.revision.number: @@ -117,10 +86,33 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, def Commit(self): """Svn2.Commit() => True|False\nCommit svn repository\n""" - try: - self.commit_changes([]) - return True - except: + # First try to update + if not self.Update(): + self.logger.error("Failed to update svn repository, refusing to " + "commit changes") return False + try: + self.revision = self.client.checkin([self.datastore], + 'Svn2: autocommit', + recurse=True) + self.revision = self.client.update(self.datastore, recurse=True)[0] + self.logger.info("Svn2: Commited changes. At %s" % + self.revision.number) + return True + except pysvn.ClientError: # pylint: disable=E1101 + err = sys.exc_info()[1] + # try to be smart about the error we got back + details = None + if "callback_ssl_server_trust_prompt" in str(err): + details = "SVN server certificate is not trusted" + elif "callback_get_login" in str(err): + details = "SVN credentials not cached" + if details is None: + self.logger.error("Svn2: Failed to commit changes", + exc_info=1) + else: + self.logger.error("Svn2: Failed to commit changes: %s" % + details) + return False diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 6d92bb530..627c82f25 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -1,47 +1,51 @@ +""" A plugin to provide helper classes and functions to templates """ + +import os import re import imp import sys -import glob import logging import Bcfg2.Server.Lint import Bcfg2.Server.Plugin -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + +MODULE_RE = re.compile(r'(?P<filename>(?P<module>[^\/]+)\.py)$') -module_pattern = r'(?P<filename>(?P<module>[^\/]+)\.py)$' -module_re = re.compile(module_pattern) class HelperModule(Bcfg2.Server.Plugin.FileBacked): + """ Representation of a TemplateHelper module """ + def __init__(self, name, fam=None): Bcfg2.Server.Plugin.FileBacked.__init__(self, name, fam=fam) - self._module_name = module_re.search(self.name).group('module') + self._module_name = MODULE_RE.search(self.name).group('module') self._attrs = [] def Index(self): try: module = imp.load_source(self._module_name, self.name) - except: + except: # pylint: disable=W0702 err = sys.exc_info()[1] - logger.error("TemplateHelper: Failed to import %s: %s" % + LOGGER.error("TemplateHelper: Failed to import %s: %s" % (self.name, err)) return if not hasattr(module, "__export__"): - logger.error("TemplateHelper: %s has no __export__ list" % + LOGGER.error("TemplateHelper: %s has no __export__ list" % self.name) return newattrs = [] for sym in module.__export__: if sym not in self._attrs and hasattr(self, sym): - logger.warning("TemplateHelper: %s: %s is a reserved keyword, " + LOGGER.warning("TemplateHelper: %s: %s is a reserved keyword, " "skipping export" % (self.name, sym)) continue try: setattr(self, sym, getattr(module, sym)) newattrs.append(sym) except AttributeError: - logger.warning("TemplateHelper: %s: %s exports %s, but has no " + LOGGER.warning("TemplateHelper: %s exports %s, but has no " "such attribute" % (self.name, sym)) # remove old exports for sym in set(self._attrs) - set(newattrs): @@ -51,8 +55,9 @@ class HelperModule(Bcfg2.Server.Plugin.FileBacked): class HelperSet(Bcfg2.Server.Plugin.DirectoryBacked): + """ A set of template helper modules """ ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") - patterns = module_re + patterns = MODULE_RE __child__ = HelperModule @@ -68,7 +73,7 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, self.helpers = HelperSet(self.data, core.fam) def get_additional_data(self, _): - return dict([(h._module_name, h) + return dict([(h._module_name, h) # pylint: disable=W0212 for h in self.helpers.entries.values()]) @@ -76,24 +81,24 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerlessPlugin): """ find duplicate Pkgmgr entries with the same priority """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) - hm = HelperModule("foo.py") - self.reserved_keywords = dir(hm) + self.reserved_keywords = dir(HelperModule("foo.py")) def Run(self): for fname in os.listdir(os.path.join(self.config['repo'], "TemplateHelper")): helper = os.path.join(self.config['repo'], "TemplateHelper", fname) - if not module_re.search(helper) or not self.HandlesFile(helper): + if not MODULE_RE.search(helper) or not self.HandlesFile(helper): continue self.check_helper(helper) def check_helper(self, helper): - module_name = module_re.search(helper).group(1) + """ check a helper module for export errors """ + module_name = MODULE_RE.search(helper).group(1) try: module = imp.load_source(module_name, helper) - except: + except: # pylint: disable=W0702 err = sys.exc_info()[1] self.LintError("templatehelper-import-error", "Failed to import %s: %s" % @@ -120,8 +125,8 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerlessPlugin): (helper, sym)) elif sym.startswith("_"): self.LintError("templatehelper-underscore-export", - "%s: exported symbol %s starts with underscore" % - (helper, sym)) + "%s: exported symbol %s starts with underscore" + % (helper, sym)) @classmethod def Errors(cls): diff --git a/src/lib/Bcfg2/Server/Plugins/Trigger.py b/src/lib/Bcfg2/Server/Plugins/Trigger.py index 313a1bf03..7279edfed 100644 --- a/src/lib/Bcfg2/Server/Plugins/Trigger.py +++ b/src/lib/Bcfg2/Server/Plugins/Trigger.py @@ -1,9 +1,14 @@ +""" Trigger is a plugin that calls external scripts (on the server) """ + import os import pipes import Bcfg2.Server.Plugin from subprocess import Popen, PIPE + class TriggerFile(Bcfg2.Server.Plugin.FileBacked): + """ Representation of a trigger script file """ + def HandleEvent(self, event=None): return @@ -25,6 +30,8 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, self.core.fam) def async_run(self, args): + """ Run the trigger script asynchronously in a forked process + """ pid = os.fork() if pid: os.waitpid(pid, 0) @@ -34,15 +41,14 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, self.debug_log("Running %s" % " ".join(pipes.quote(a) for a in args)) proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) - (out, err) = proc.communicate() + err = proc.communicate()[1] rv = proc.wait() if rv != 0: self.logger.error("Trigger: Error running %s (%s): %s" % (args[0], rv, err)) elif err: self.debug_log("Trigger: Error: %s" % err) - os._exit(0) - + os._exit(0) # pylint: disable=W0212 def end_client_run(self, metadata): args = [metadata.hostname, '-p', metadata.profile, '-g', diff --git a/src/lib/Bcfg2/Server/Reports/backends.py b/src/lib/Bcfg2/Server/Reports/backends.py index 85241932f..9f07c104f 100644 --- a/src/lib/Bcfg2/Server/Reports/backends.py +++ b/src/lib/Bcfg2/Server/Reports/backends.py @@ -1,3 +1,4 @@ +import sys from django.contrib.auth.models import User from nisauth import * diff --git a/src/lib/Bcfg2/Server/Reports/manage.py b/src/lib/Bcfg2/Server/Reports/manage.py index 858bddeca..1c8fb03f6 100755 --- a/src/lib/Bcfg2/Server/Reports/manage.py +++ b/src/lib/Bcfg2/Server/Reports/manage.py @@ -1,11 +1,11 @@ #!/usr/bin/env python from django.core.management import execute_manager try: - import settings # Assumed to be in the same directory. + import Bcfg2.settings except ImportError: import sys - sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.stderr.write("Error: Can't find the Bcfg2.settings module. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) sys.exit(1) if __name__ == "__main__": - execute_manager(settings) + execute_manager(Bcfg2.settings) diff --git a/src/lib/Bcfg2/Server/Reports/nisauth.py b/src/lib/Bcfg2/Server/Reports/nisauth.py index b3e37113b..dd1f2f742 100644 --- a/src/lib/Bcfg2/Server/Reports/nisauth.py +++ b/src/lib/Bcfg2/Server/Reports/nisauth.py @@ -1,8 +1,8 @@ +"""Checks with NIS to see if the current user is in the support group""" + import crypt import nis -from Bcfg2.Server.Reports.settings import AUTHORIZED_GROUP - -"""Checks with NIS to see if the current user is in the support group""" +from Bcfg2.settings import AUTHORIZED_GROUP # pylint: disable=E0611 class NISAUTHError(Exception): diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py index 894353bba..736d6448a 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py @@ -330,7 +330,7 @@ def do_qs(parser, token): try: tag, name, value = token.split_contents() except ValueError: - raise TemplateSyntaxError, "%r tag requires exactly two arguments" \ + raise template.TemplateSyntaxError, "%r tag requires exactly two arguments" \ % token.contents.split()[0] return QsNode(name, value) @@ -367,7 +367,7 @@ def sort_link(parser, token): try: tag, sort_key, text = token.split_contents() except ValueError: - raise TemplateSyntaxError("%r tag requires at least four arguments" \ + raise template.TemplateSyntaxError("%r tag requires at least four arguments" \ % token.split_contents()[0]) return SortLinkNode(sort_key, text) diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py index b2814b445..70ef49bc1 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py @@ -8,6 +8,7 @@ from Bcfg2.Compat import u_str register = template.Library() +# pylint: disable=E0611,F0401 try: from pygments import highlight from pygments.lexers import get_lexer_by_name @@ -16,6 +17,7 @@ try: except: colorize = False +# pylint: enable=E0611,F0401 @register.filter diff --git a/src/lib/Bcfg2/Server/Reports/reports/views.py b/src/lib/Bcfg2/Server/Reports/reports/views.py index e4c38363f..ca9e5f1f9 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/views.py +++ b/src/lib/Bcfg2/Server/Reports/reports/views.py @@ -574,7 +574,7 @@ def prepare_paginated_list(request, context, paged_list, page=1, max_results=25) view, args, kwargs = resolve(request.META['PATH_INFO']) kwargs['page_number'] = total_pages raise PaginationError(HttpResponseRedirect(reverse(view, - kwards=kwargs))) + kwargs=kwargs))) except (Resolver404, NoReverseMatch, ValueError): raise "Accessing beyond last page. Unable to resolve redirect." diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 95a18b608..44379ee89 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -98,29 +98,6 @@ class FileNotBuilt(Exception): return repr(self.value) -def getClientList(hostglobs): - """ given a host glob, get a list of clients that match it """ - # special cases to speed things up: - if '*' in hostglobs: - return self.metadata.clients - has_wildcards = False - for glob in hostglobs: - # check if any wildcard characters are in the string - if set('*?[]') & set(glob): - has_wildcards = True - break - if not has_wildcards: - return hostglobs - - rv = set() - clist = set(self.metadata.clients) - for glob in hostglobs: - for client in clist: - if fnmatch.fnmatch(client, glob): - rv.update(client) - clist.difference_update(rv) - return list(rv) - def printTabular(rows): """Print data in tabular format.""" cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 \ @@ -153,6 +130,7 @@ def load_interpreters(): # before --interpreter was added, so we call IPython # better import IPython + # pylint: disable=E1101 if hasattr(IPython, "Shell"): interpreters["ipython"] = lambda v: \ IPython.Shell.IPShell(argv=[], user_ns=v).mainloop() @@ -162,6 +140,7 @@ def load_interpreters(): best = "ipython" else: print("Unknown IPython API version") + # pylint: enable=E1101 except ImportError: pass @@ -178,6 +157,29 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): self.cont = True self.fam.handle_events_in_interval(4) + def _get_client_list(self, hostglobs): + """ given a host glob, get a list of clients that match it """ + # special cases to speed things up: + if '*' in hostglobs: + return self.metadata.clients + has_wildcards = False + for glob in hostglobs: + # check if any wildcard characters are in the string + if set('*?[]') & set(glob): + has_wildcards = True + break + if not has_wildcards: + return hostglobs + + rv = set() + clist = set(self.metadata.clients) + for glob in hostglobs: + for client in clist: + if fnmatch.fnmatch(client, glob): + rv.update(client) + clist.difference_update(rv) + return list(rv) + def do_loop(self): """Looping.""" self.cont = True @@ -227,7 +229,7 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): else: logger.error("Invalid interpreter %s" % setup['interpreter']) logger.error("Valid interpreters are: %s" % - ", ".join(interpeters.keys())) + ", ".join(interpreters.keys())) def do_quit(self, _): """ @@ -318,7 +320,7 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): if err.errno != 17: print("Could not create %s: %s" % (destdir, err)) if len(alist) > 1: - clients = getClientList(alist[1:]) + clients = self._get_client_list(alist[1:]) else: clients = self.metadata.clients for client in clients: @@ -350,7 +352,7 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): if err.errno != 17: print("Could not create %s: %s" % (destdir, err)) if len(args) > 2: - clients = getClientList(args[1:]) + clients = self._get_client_list(args[1:]) else: clients = self.metadata.clients if altsrc: diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 0e9a32e49..f1f91b7f4 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -122,11 +122,11 @@ if __name__ == '__main__': obj_name=plugin + "Lint") except (ImportError, AttributeError): err = sys.exc_info()[1] - logger.error("Failed to load plugin %s: %s" % (plugin + "Lint", - err)) + logger.error("Failed to load plugin %s: %s" % + (plugin + "Lint", err)) except AttributeError: err = sys.exc_info()[1] - logger.error("Failed to load plugin %s: %s" % (obj_name, err)) + logger.error("Failed to load plugin %s: %s" % (plugin, err)) serverplugins = dict() serverlessplugins = dict() diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index a9c620496..07f9b81b0 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -61,12 +61,14 @@ class DepSolver(object): def __init__(self, cfgfile, verbose=1): self.cfgfile = cfgfile self.yumbase = yum.YumBase() + # pylint: disable=E1121 try: self.yumbase.preconf.debuglevel = verbose self.yumbase.preconf.fn = cfgfile self.yumbase._getConfig() except AttributeError: self.yumbase._getConfig(cfgfile, debuglevel=verbose) + # pylint: enable=E1121 self.logger = get_logger(verbose) def get_groups(self): |