From dd28e90f183972cc2a395094ce3e3f72e861953f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 21 Sep 2012 13:55:05 -0400 Subject: run pylint for errors on almost everything, full runs on some selected stuff --- src/lib/Bcfg2/Server/Plugins/Bzr.py | 31 +++--- .../Server/Plugins/Cfg/CfgCheetahGenerator.py | 13 ++- src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py | 14 +-- .../Plugins/Cfg/CfgEncryptedGenshiGenerator.py | 2 +- .../Plugins/Cfg/CfgExternalCommandVerifier.py | 11 +- .../Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 107 ++++++++++--------- src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py | 5 +- src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py | 11 +- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 97 +++++++++-------- src/lib/Bcfg2/Server/Plugins/Cvs.py | 41 +++---- src/lib/Bcfg2/Server/Plugins/DBStats.py | 41 ++++--- src/lib/Bcfg2/Server/Plugins/Darcs.py | 41 +++---- src/lib/Bcfg2/Server/Plugins/Defaults.py | 4 +- src/lib/Bcfg2/Server/Plugins/FileProbes.py | 50 +++++---- src/lib/Bcfg2/Server/Plugins/Fossil.py | 45 +++----- src/lib/Bcfg2/Server/Plugins/Git.py | 42 +++----- src/lib/Bcfg2/Server/Plugins/Guppy.py | 5 +- src/lib/Bcfg2/Server/Plugins/Hg.py | 45 +++----- src/lib/Bcfg2/Server/Plugins/Ldap.py | 10 +- src/lib/Bcfg2/Server/Plugins/Metadata.py | 86 ++++++++------- .../Bcfg2/Server/Plugins/Packages/Collection.py | 28 ++--- .../Server/Plugins/Packages/PackagesSources.py | 3 + src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 11 +- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 42 +++++--- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 8 +- src/lib/Bcfg2/Server/Plugins/Probes.py | 118 +++++++++++++-------- src/lib/Bcfg2/Server/Plugins/Properties.py | 19 ++-- src/lib/Bcfg2/Server/Plugins/PuppetENC.py | 20 ++-- src/lib/Bcfg2/Server/Plugins/Rules.py | 2 + src/lib/Bcfg2/Server/Plugins/SEModules.py | 3 - src/lib/Bcfg2/Server/Plugins/ServiceCompat.py | 3 + src/lib/Bcfg2/Server/Plugins/Svn.py | 38 +++---- src/lib/Bcfg2/Server/Plugins/Svn2.py | 114 +++++++++----------- src/lib/Bcfg2/Server/Plugins/TemplateHelper.py | 43 ++++---- src/lib/Bcfg2/Server/Plugins/Trigger.py | 12 ++- 35 files changed, 591 insertions(+), 574 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Bzr.py b/src/lib/Bcfg2/Server/Plugins/Bzr.py index a71021cb5..4de204468 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bzr.py +++ b/src/lib/Bcfg2/Server/Plugins/Bzr.py @@ -1,35 +1,36 @@ +""" The Bzr plugin provides a revision interface for Bcfg2 repos using +bazaar. """ + import Bcfg2.Server.Plugin +# pylint: disable=F0401 from bzrlib.workingtree import WorkingTree from bzrlib import errors +# pylint: enable=F0401 -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Bzr') class Bzr(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """Bzr is a version plugin for dealing with Bcfg2 repos.""" - name = 'Bzr' + """ The Bzr plugin provides a revision interface for Bcfg2 repos + using bazaar. """ __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # Read revision from bcfg2 repo - revision = self.get_revision() - - logger.debug("Initialized Bazaar plugin with directory = %(dir)s at revision = %(rev)s" % {'dir': datastore, 'rev': revision}) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized Bazaar plugin with directory %s at " + "revision = %s" % (self.datastore, + self.get_revision())) def get_revision(self): """Read Bazaar revision information for the Bcfg2 repository.""" try: working_tree = WorkingTree.open(self.datastore) revision = str(working_tree.branch.revno()) - if working_tree.has_changes(working_tree.basis_tree()) or working_tree.unknowns(): + if (working_tree.has_changes(working_tree.basis_tree()) or + working_tree.unknowns()): revision += "+" except errors.NotBranchError: - logger.error("Failed to read Bazaar branch; disabling Bazaar support") - raise Bcfg2.Server.Plugin.PluginInitError + msg = "Failed to read Bazaar branch" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) return revision diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index a0e999847..7f02d4a05 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -2,18 +2,17 @@ `_ templating system to generate :ref:`server-plugins-generators-cfg` files. """ -import copy import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgGenerator -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) try: from Cheetah.Template import Template - have_cheetah = True + HAS_CHEETAH = True except ImportError: - have_cheetah = False + HAS_CHEETAH = False class CfgCheetahGenerator(CfgGenerator): @@ -29,9 +28,9 @@ class CfgCheetahGenerator(CfgGenerator): def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) - if not have_cheetah: - msg = "Cfg: Cheetah is not available: %s" % entry.get("name") - logger.error(msg) + if not HAS_CHEETAH: + msg = "Cfg: Cheetah is not available: %s" % self.name + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) __init__.__doc__ = CfgGenerator.__init__.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py index 409d2cbf6..00b95c970 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py @@ -7,7 +7,8 @@ import Bcfg2.Server.Plugin from subprocess import Popen, PIPE from Bcfg2.Server.Plugins.Cfg import CfgFilter -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class CfgDiffFilter(CfgFilter): """ CfgDiffFilter applies diffs to plaintext @@ -24,14 +25,15 @@ class CfgDiffFilter(CfgFilter): open(basename, 'w').write(data) os.close(basehandle) - cmd = ["patch", "-u", "-f", basefile.name] + cmd = ["patch", "-u", "-f", basename] patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) stderr = patch.communicate(input=self.data)[1] ret = patch.wait() - output = open(basefile.name, 'r').read() - os.unlink(basefile.name) + output = open(basename, 'r').read() + os.unlink(basename) if ret != 0: - logger.error("Error applying diff %s: %s" % (delta.name, stderr)) - raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta) + msg = "Error applying diff %s: %s" % (self.name, stderr) + LOGGER.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) return output modify_data.__doc__ = CfgFilter.modify_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 6fd70e69f..31c3d79b0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -17,7 +17,7 @@ try: from genshi.template import TemplateLoader except ImportError: # CfgGenshiGenerator will raise errors if genshi doesn't exist - TemplateLoader = object + TemplateLoader = object # pylint: disable=C0103 LOGGER = logging.getLogger(__name__) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index 87e11ab6d..fb66ca8bf 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -7,7 +7,8 @@ import Bcfg2.Server.Plugin from subprocess import Popen, PIPE from Bcfg2.Server.Plugins.Cfg import CfgVerifier, CfgVerificationError -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class CfgExternalCommandVerifier(CfgVerifier): """ Invoke an external script to verify @@ -16,6 +17,11 @@ class CfgExternalCommandVerifier(CfgVerifier): #: Handle :file:`:test` files __basenames__ = [':test'] + def __init__(self, name, specific, encoding): + CfgVerifier.__init__(self, name, specific, encoding) + self.cmd = [] + __init__.__doc__ = CfgVerifier.__init__.__doc__ + def verify_entry(self, entry, metadata, data): proc = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) err = proc.communicate(input=data)[1] @@ -34,8 +40,7 @@ class CfgExternalCommandVerifier(CfgVerifier): self.cmd.extend(shlex.split(bangpath[2:].strip())) else: msg = "Cannot execute %s" % self.name - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) self.cmd.append(self.name) handle_event.__doc__ = CfgVerifier.handle_event.__doc__ - diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index dc128bbe9..21662a984 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -9,16 +9,17 @@ import traceback import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgGenerator -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) try: import genshi.core from genshi.template import TemplateLoader, NewTextTemplate from genshi.template.eval import UndefinedError - have_genshi = True + HAS_GENSHI = True except ImportError: - TemplateLoader = None - have_genshi = False + TemplateLoader = None # pylint: disable=C0103 + HAS_GENSHI = False + def removecomment(stream): """ A Genshi filter that removes comments from the stream. This @@ -42,7 +43,7 @@ class CfgGenshiGenerator(CfgGenerator): #: Handle .genshi files __extensions__ = ['genshi'] - + #: ``__loader_cls__`` is the class that will be instantiated to #: load the template files. It must implement one public function, #: ``load()``, as :class:`genshi.template.TemplateLoader`. @@ -61,9 +62,9 @@ class CfgGenshiGenerator(CfgGenerator): def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) - if not have_genshi: + if not HAS_GENSHI: msg = "Cfg: Genshi is not available: %s" % fname - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) self.loader = self.__loader_cls__() self.template = None @@ -87,55 +88,59 @@ class CfgGenshiGenerator(CfgGenerator): stack = traceback.extract_tb(sys.exc_info()[2]) for quad in stack: if quad[0] == self.name: - logger.error("Cfg: Error rendering %s at '%s': %s: %s" % + LOGGER.error("Cfg: Error rendering %s at '%s': %s: %s" % (fname, quad[2], err.__class__.__name__, err)) break raise except: - # a failure in a %{ python ... %} block -- the snippet in - # the traceback is just the beginning of the block. - err = sys.exc_info()[1] - stack = traceback.extract_tb(sys.exc_info()[2]) - (filename, lineno, func, text) = stack[-1] - # this is horrible, and I deeply apologize to whoever gets - # to maintain this after I go to the Great Beer Garden in - # the Sky. genshi is incredibly opaque about what's being - # executed, so the only way I can find to determine which - # {% python %} block is being executed -- if there are - # multiples -- is to iterate through them and match the - # snippet of the first line that's in the traceback with - # the first non-empty line of the block. - execs = [contents - for etype, contents, loc in self.template.stream - if etype == self.template.EXEC] - contents = None - if len(execs) == 1: - contents = execs[0] - elif len(execs) > 1: - match = pyerror_re.match(func) - if match: - firstline = match.group(0) - for pyblock in execs: - if pyblock.startswith(firstline): - contents = pyblock - break - # else, no EXEC blocks -- WTF? - if contents: - # we now have the bogus block, but we need to get the - # offending line. To get there, we do (line number - # given in the exception) - (firstlineno from the - # internal genshi code object of the snippet) + 1 = - # (line number of the line with an error within the - # block, with all multiple line breaks elided to a - # single line break) - real_lineno = lineno - contents.code.co_firstlineno - src = re.sub(r'\n\n+', '\n', contents.source).splitlines() - logger.error("Cfg: Error rendering %s at '%s': %s: %s" % - (fname, src[real_lineno], err.__class__.__name__, - err)) - raise + self._handle_genshi_exception(fname, sys.exc_info()) get_data.__doc__ = CfgGenerator.get_data.__doc__ + def _handle_genshi_exception(self, fname, exc): + """ this is horrible, and I deeply apologize to whoever gets + to maintain this after I go to the Great Beer Garden in the + Sky. genshi is incredibly opaque about what's being executed, + so the only way I can find to determine which {% python %} + block is being executed -- if there are multiples -- is to + iterate through them and match the snippet of the first line + that's in the traceback with the first non-empty line of the + block. """ + + # a failure in a %{ python ... %} block -- the snippet in + # the traceback is just the beginning of the block. + err = [1] + stack = traceback.extract_tb(exc[2]) + lineno, func = stack[-1][1:3] + execs = [contents + for etype, contents in self.template.stream[:2] + if etype == self.template.EXEC] + contents = None + if len(execs) == 1: + contents = execs[0] + elif len(execs) > 1: + match = self.pyerror_re.match(func) + if match: + firstline = match.group(0) + for pyblock in execs: + if pyblock.startswith(firstline): + contents = pyblock + break + # else, no EXEC blocks -- WTF? + if contents: + # we now have the bogus block, but we need to get the + # offending line. To get there, we do (line number + # given in the exception) - (firstlineno from the + # internal genshi code object of the snippet) + 1 = + # (line number of the line with an error within the + # block, with all multiple line breaks elided to a + # single line break) + real_lineno = lineno - contents.code.co_firstlineno + src = re.sub(r'\n\n+', '\n', contents.source).splitlines() + LOGGER.error("Cfg: Error rendering %s at '%s': %s: %s" % + (fname, src[real_lineno], err.__class__.__name__, + err)) + raise + def handle_event(self, event): if event.code2str() == 'deleted': return @@ -146,6 +151,6 @@ class CfgGenshiGenerator(CfgGenerator): except Exception: msg = "Cfg: Could not load template %s: %s" % (self.name, sys.exc_info()[1]) - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) handle_event.__doc__ = CfgGenerator.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py index 472a7dba3..2396d6eb6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py @@ -4,7 +4,8 @@ import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgInfo -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class CfgInfoXML(CfgInfo): """ CfgInfoXML handles :file:`info.xml` files for @@ -22,7 +23,7 @@ class CfgInfoXML(CfgInfo): mdata = dict() self.infoxml.pnode.Match(metadata, mdata, entry=entry) if 'Info' not in mdata: - logger.error("Failed to set metadata for file %s" % + LOGGER.error("Failed to set metadata for file %s" % entry.get('name')) raise Bcfg2.Server.Plugin.PluginExecutionError self._set_info(entry, mdata['Info'][None]) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py index a47663904..8f71c45c8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py @@ -4,7 +4,8 @@ import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgInfo -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + class CfgLegacyInfo(CfgInfo): """ CfgLegacyInfo handles :file:`info` and :file:`:info` files for @@ -20,6 +21,9 @@ class CfgLegacyInfo(CfgInfo): def __init__(self, path): CfgInfo.__init__(self, path) self.path = path + + #: The set of info metadata stored in the file + self.metadata = None __init__.__doc__ = CfgInfo.__init__.__doc__ def bind_info_to_entry(self, entry, metadata): @@ -30,9 +34,10 @@ class CfgLegacyInfo(CfgInfo): if event.code2str() == 'deleted': return for line in open(self.path).readlines(): - match = Bcfg2.Server.Plugin.info_regex.match(line) + match = Bcfg2.Server.Plugin.INFO_REGEX.match(line) if not match: - logger.warning("Failed to parse line in %s: %s" % (fpath, line)) + LOGGER.warning("Failed to parse line in %s: %s" % + (event.filename, line)) continue else: self.metadata = \ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index e2832cd26..c34cad30e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -8,10 +8,12 @@ import logging import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages import Bcfg2.Server.Lint +# pylint: disable=W0622 +from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages +# pylint: enable=W0622 -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) #: SETUP contains a reference to the #: :class:`Bcfg2.Options.OptionParser` created by the Bcfg2 core for @@ -24,6 +26,7 @@ logger = logging.getLogger(__name__) #: the EntrySet children. SETUP = None + class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): """ CfgBaseFileMatcher is the parent class for all Cfg handler objects. """ @@ -93,12 +96,13 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): components = ['^(?P%s)' % basename] if cls.__specific__: - components.append('(|\\.H_(?P\S+?)|.G(?P\d+)_(?P\S+?))') + components.append('(|\\.H_(?P\S+?)|' + + '.G(?P\d+)_(?P\S+?))') if extensions: components.append('\\.(?P%s)' % '|'.join(extensions)) components.append('$') return re.compile("".join(components)) - + @classmethod def handles(cls, event, basename=None): """ Return True if this handler handles the file described by @@ -129,7 +133,8 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): match = True break return (match and - cls.get_regex(basename=os.path.basename(basename)).match(event.filename)) + cls.get_regex( + basename=os.path.basename(basename)).match(event.filename)) @classmethod def ignore(cls, event, basename=None): @@ -166,13 +171,10 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): return (match and cls.get_regex(basename=os.path.basename(basename), extensions=cls.__ignore__).match(event.filename)) - + def __str__(self): return "%s(%s)" % (self.__class__.__name__, self.name) - def match(self, fname): - return self.regex.match(fname) - class CfgGenerator(CfgBaseFileMatcher): """ CfgGenerators generate the initial content of a file. Every @@ -193,7 +195,7 @@ class CfgGenerator(CfgBaseFileMatcher): CfgBaseFileMatcher.__init__(self, name, specific, encoding) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] - def get_data(self, entry, metadata): + def get_data(self, entry, metadata): # pylint: disable=W0613 """ get_data() returns the initial data of a file. :param entry: The entry to generate data for. ``entry`` should @@ -335,13 +337,17 @@ class CfgDefaultInfo(CfgInfo): bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ #: A :class:`CfgDefaultInfo` object instantiated with -#: :attr:`Bcfg2.Server.Plugin.helper.default_file_metadata` as its +#: :attr:`Bcfg2.Server.Plugin.helper.DEFAULT_FILE_METADATA` as its #: default metadata. This is used to set a default file metadata set #: on an entry before a "real" :class:`CfgInfo` handler applies its #: metadata to the entry. -DEFAULT_INFO = CfgDefaultInfo(Bcfg2.Server.Plugin.default_file_metadata) +DEFAULT_INFO = CfgDefaultInfo(Bcfg2.Server.Plugin.DEFAULT_FILE_METADATA) + class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): + """ Handle a collection of host- and group-specific Cfg files with + multiple different Cfg handlers in a single directory. """ + def __init__(self, basename, path, entry_type, encoding): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) @@ -377,18 +383,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): :returns: None """ action = event.code2str() - + if event.filename not in self.entries: if action not in ['exists', 'created', 'changed']: # process a bogus changed event like a created return - + for hdlr in self.handlers: if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but # handle it like a 'created' - logger.warning("Got %s event for unknown file %s" % + LOGGER.warning("Got %s event for unknown file %s" % (action, event.filename)) self.debug_log("%s handling %s event on %s" % (hdlr.__name__, action, event.filename)) @@ -403,7 +409,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): del self.entries[event.filename] return - logger.error("Could not process event %s for %s; ignoring" % + LOGGER.error("Could not process event %s for %s; ignoring" % (action, event.filename)) def get_matching(self, metadata): @@ -412,7 +418,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): item.specific.matches(metadata))] get_matching.__doc__ = Bcfg2.Server.Plugin.EntrySet.get_matching.__doc__ - def entry_init(self, event, hdlr): + def entry_init(self, event, hdlr): # pylint: disable=W0221 """ Handle the creation of a file on the filesystem and the creation of a Cfg handler object in this CfgEntrySet to track it. @@ -432,13 +438,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): specific=hdlr.get_regex(os.path.basename(self.path))) else: if event.filename in self.entries: - logger.warn("Got duplicate add for %s" % event.filename) + LOGGER.warn("Got duplicate add for %s" % event.filename) else: fpath = os.path.join(self.path, event.filename) self.entries[event.filename] = hdlr(fpath) self.entries[event.filename].handle_event(event) - def bind_entry(self, entry, metadata): + def bind_entry(self, entry, metadata): # pylint: disable=R0912,R0915 info_handlers = [] generators = [] filters = [] @@ -459,12 +465,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): fdesc = "/".join(ent.__basenames__) elif ent.__extensions__: fdesc = "." + "/.".join(ent.__extensions__) - logger.warning("Cfg: %s: Use of %s files is deprecated" % + LOGGER.warning("Cfg: %s: Use of %s files is deprecated" % (ent.name, fdesc)) DEFAULT_INFO.bind_info_to_entry(entry, metadata) if len(info_handlers) > 1: - logger.error("More than one info supplier found for %s: %s" % + LOGGER.error("More than one info supplier found for %s: %s" % (entry.get("name"), info_handlers)) if len(info_handlers): info_handlers[0].bind_info_to_entry(entry, metadata) @@ -482,7 +488,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): except: msg = "Cfg: exception rendering %s with %s: %s" % \ (entry.get("name"), generator, sys.exc_info()[1]) - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) for fltr in filters: @@ -506,9 +512,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): msg = "Data for %s for %s failed to verify: %s" % \ (entry.get('name'), metadata.hostname, sys.exc_info()[1]) - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - + if entry.get('encoding') == 'base64': data = b64encode(data) else: @@ -518,14 +524,14 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) - logger.error(msg) - logger.error("Please verify you are using the proper encoding.") + LOGGER.error(msg) + LOGGER.error("Please verify you are using the proper encoding") raise Bcfg2.Server.Plugin.PluginExecutionError(msg) except ValueError: msg = "Error in specification for %s: %s" % (entry.get('name'), sys.exc_info()[1]) - logger.error(msg) - logger.error("You need to specify base64 encoding for %s." % + LOGGER.error(msg) + LOGGER.error("You need to specify base64 encoding for %s." % entry.get('name')) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) except TypeError: @@ -547,21 +553,23 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): ent.specific.matches(metadata))] if not generators: msg = "No base file found for %s" % entry.get('name') - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - + rv = [] try: best = self.best_matching(metadata, generators) rv.append(best.specific) - except: + except: # pylint: disable=W0702 pass if not rv or not rv[0].hostname: - rv.append(Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)) + rv.append(Bcfg2.Server.Plugin.Specificity( + hostname=metadata.hostname)) return rv def build_filename(self, specific): + """ Create a filename for pulled file data """ bfname = self.path + '/' + self.path.split('/')[-1] if specific.all: return bfname @@ -571,22 +579,23 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): return "%s.H_%s" % (bfname, specific.hostname) def write_update(self, specific, new_entry, log): + """ Write pulled data to the filesystem """ if 'text' in new_entry: name = self.build_filename(specific) if os.path.exists("%s.genshi" % name): msg = "Cfg: Unable to pull data for genshi types" - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) elif os.path.exists("%s.cheetah" % name): msg = "Cfg: Unable to pull data for cheetah types" - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: etext = new_entry['text'].encode(self.encoding) except: - msg = "Cfg: Cannot encode content of %s as %s" % (name, - self.encoding) - logger.error(msg) + msg = "Cfg: Cannot encode content of %s as %s" % \ + (name, self.encoding) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) open(name, 'w').write(etext) self.debug_log("Wrote file %s" % name, flag=log) @@ -597,7 +606,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): for ifile in ['info', ':info']: info = os.path.join(self.path, ifile) if os.path.exists(info): - logger.info("Removing %s and replacing with info.xml" % + LOGGER.info("Removing %s and replacing with info.xml" % info) os.remove(info) metadata_updates = {} @@ -606,8 +615,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): metadata_updates[attr] = new_entry.get(attr) infoxml = lxml.etree.Element('FileInfo') infotag = lxml.etree.SubElement(infoxml, 'Info') - [infotag.attrib.__setitem__(attr, metadata_updates[attr]) - for attr in metadata_updates] + for attr in metadata_updates: + infotag.attrib.__setitem__(attr, metadata_updates[attr]) ofile = open(self.path + "/info.xml", "w") ofile.write(lxml.etree.tostring(infoxml, xml_declaration=False, pretty_print=True).decode('UTF-8')) @@ -629,10 +638,10 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, es_child_cls = Bcfg2.Server.Plugin.SpecificData def __init__(self, core, datastore): - global SETUP + global SETUP # pylint: disable=W0603 Bcfg2.Server.Plugin.GroupSpool.__init__(self, core, datastore) Bcfg2.Server.Plugin.PullTarget.__init__(self) - + SETUP = core.setup if 'validate' not in SETUP: SETUP.add_option('validate', Bcfg2.Options.CFG_VALIDATION) @@ -665,7 +674,8 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, def AcceptChoices(self, entry, metadata): return self.entries[entry.get('name')].list_accept_choices(entry, metadata) - AcceptChoices.__doc__ = Bcfg2.Server.Plugin.PullTarget.AcceptChoices.__doc__ + AcceptChoices.__doc__ = \ + Bcfg2.Server.Plugin.PullTarget.AcceptChoices.__doc__ def AcceptPullData(self, specific, new_entry, log): return self.entries[new_entry.get('name')].write_update(specific, @@ -689,6 +699,7 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): "diff-file-used":"warning"} def check_entry(self, basename, entry): + """ check that no .cat or .diff files are in use """ cfg = self.core.plugins['Cfg'] for basename, entry in list(cfg.entries.items()): for fname, handler in entry.entries.items(): diff --git a/src/lib/Bcfg2/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py index 6ce72acd2..a36a116f5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cvs.py +++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py @@ -1,34 +1,22 @@ -import os +""" The Cvs plugin provides a revision interface for Bcfg2 repos using +cvs. """ + from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Cvs') class Cvs(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """CVS is a version plugin for dealing with Bcfg2 repository.""" - name = 'Cvs' + """ The Cvs plugin provides a revision interface for Bcfg2 repos + using cvs.""" __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True + __vcs_metadata_path__ = "CVSROOT" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to cvs directory for Bcfg2 repo - cvs_dir = "%s/CVSROOT" % datastore - - # Read revision from Bcfg2 repo - if os.path.isdir(cvs_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % cvs_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized cvs plugin with cvs directory = %s" % cvs_dir) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized cvs plugin with cvs directory %s" % + self.vcs_path) def get_revision(self): """Read cvs revision information for the Bcfg2 repository.""" @@ -37,10 +25,9 @@ class Cvs(Bcfg2.Server.Plugin.Plugin, shell=True, cwd=self.datastore, stdout=PIPE).stdout.readlines() - revision = data[3].strip('\n') + return data[3].strip('\n') except IndexError: - logger.error("Failed to read cvs log; disabling cvs support") - logger.error('''Ran command "cvs log %s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError - + msg = "Failed to read cvs log" + self.logger.error(msg) + self.logger.error('Ran command "cvs log %s"' % self.datastore) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index ea3b1b69e..16e9e4a8a 100644 --- a/src/lib/Bcfg2/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -1,6 +1,6 @@ +""" DBstats provides a database-backed statistics handler """ + import difflib -import logging -import lxml.etree import platform import sys import time @@ -15,13 +15,10 @@ from Bcfg2.Server.Reports.importscript import load_stat from Bcfg2.Server.Reports.reports.models import Client from Bcfg2.Compat import b64decode -# for debugging output only -logger = logging.getLogger('Bcfg2.Plugins.DBStats') - class DBStats(Bcfg2.Server.Plugin.ThreadedStatistics, Bcfg2.Server.Plugin.PullSource): - name = 'DBStats' + """ DBstats provides a database-backed statistics handler """ def __init__(self, core, datastore): Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore) @@ -36,29 +33,29 @@ class DBStats(Bcfg2.Server.Plugin.ThreadedStatistics, newstats.set('time', time.asctime(time.localtime())) start = time.time() - for i in [1, 2, 3]: + for try_count in [1, 2, 3]: try: load_stat(metadata, newstats, self.core.encoding, 0, - logger, + self.logger, True, platform.node()) - logger.info("Imported data for %s in %s seconds" \ - % (metadata.hostname, time.time() - start)) + self.logger.info("Imported data for %s in %s seconds" % + (metadata.hostname, time.time() - start)) return except MultipleObjectsReturned: - e = sys.exc_info()[1] - logger.error("DBStats: MultipleObjectsReturned while " - "handling %s: %s" % (metadata.hostname, e)) - logger.error("DBStats: Data is inconsistent") + err = sys.exc_info()[1] + self.logger.error("DBStats: MultipleObjectsReturned while " + "handling %s: %s" % (metadata.hostname, err)) + self.logger.error("DBStats: Data is inconsistent") break except: - logger.error("DBStats: Failed to write to db (lock); retrying", - exc_info=1) - logger.error("DBStats: Retry limit failed for %s; aborting operation" \ - % metadata.hostname) + self.logger.error("DBStats: Failed to write to db (lock); " + "retrying (try %s)" % try_count, exc_info=1) + self.logger.error("DBStats: Retry limit failed for %s; " + "aborting operation" % metadata.hostname) def GetExtra(self, client): c_inst = Client.objects.filter(name=client)[0] @@ -78,11 +75,11 @@ class DBStats(Bcfg2.Server.Plugin.ThreadedStatistics, entry = result[0] ret = [] data = ('owner', 'group', 'perms') - for t in data: - if getattr(entry.reason, "current_%s" % t) == '': - ret.append(getattr(entry.reason, t)) + for dtype in data: + if getattr(entry.reason, "current_%s" % dtype) == '': + ret.append(getattr(entry.reason, dtype)) else: - ret.append(getattr(entry.reason, "current_%s" % t)) + ret.append(getattr(entry.reason, "current_%s" % dtype)) if entry.reason.is_sensitive: raise Bcfg2.Server.Plugin.PluginExecutionError elif len(entry.reason.unpruned) != 0: diff --git a/src/lib/Bcfg2/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py index 9fb9ff4f1..9ec8e2df3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Darcs.py +++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py @@ -1,35 +1,22 @@ -import os +""" Darcs is a version plugin for dealing with Bcfg2 repos stored in the +Darcs VCS. """ + from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Darcs') class Darcs(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """Darcs is a version plugin for dealing with Bcfg2 repos.""" - name = 'Darcs' + """ Darcs is a version plugin for dealing with Bcfg2 repos stored + in the Darcs VCS. """ __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True + __vcs_metadata_path__ = "_darcs" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to darcs directory for bcfg2 repo - darcs_dir = "%s/_darcs" % datastore - - # Read changeset from bcfg2 repo - if os.path.isdir(darcs_dir): - self.get_revision() - else: - logger.error("%s is not present." % darcs_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized Darcs plugin with darcs directory = %s" % darcs_dir) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized Darcs plugin with darcs directory %s" % + self.vcs_path) def get_revision(self): """Read Darcs changeset information for the Bcfg2 repository.""" @@ -40,9 +27,9 @@ class Darcs(Bcfg2.Server.Plugin.Plugin, stdout=PIPE).stdout.readlines() revision = data[0].strip('\n') except: - logger.error("Failed to read darcs repository; disabling Darcs support") - logger.error('''Ran command "darcs changes" from directory "%s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError + msg = "Failed to read darcs repository" + self.logger.error(msg) + self.logger.error('Ran command "darcs changes" from directory "%s"' + % self.datastore) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) return revision - diff --git a/src/lib/Bcfg2/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py index 718192e2a..53eed3798 100644 --- a/src/lib/Bcfg2/Server/Plugins/Defaults.py +++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py @@ -1,9 +1,9 @@ """This generator provides rule-based entry mappings.""" -import re import Bcfg2.Server.Plugin import Bcfg2.Server.Plugins.Rules + class Defaults(Bcfg2.Server.Plugins.Rules.Rules, Bcfg2.Server.Plugin.StructureValidator): """Set default attributes on bound entries""" @@ -21,7 +21,7 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, return False def HandleEntry(self, entry, metadata): - raise PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError def HandleEvent(self, event): Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index d6e4aadab..59ac9f85e 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -52,10 +52,11 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Probing.__init__(self) - self.config = Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, - 'config.xml'), - fam=core.fam, - should_monitor=True) + self.config = \ + Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, + 'config.xml'), + fam=core.fam, + should_monitor=True) self.entries = dict() self.probes = dict() @@ -89,8 +90,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, for data in datalist: if data.text is None: - self.logger.error("Got null response to %s file probe from %s" % - (data.get('name'), metadata.hostname)) + self.logger.error("Got null response to %s file probe from %s" + % (data.get('name'), metadata.hostname)) else: try: self.write_data( @@ -145,6 +146,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, return def write_file(self, fileloc, contents): + """ Write the probed file to disk """ try: os.makedirs(os.path.dirname(fileloc)) except OSError: @@ -152,8 +154,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, if err.errno == errno.EEXIST: pass else: - self.logger.error("Could not create parent directories for %s: " - "%s" % (fileloc, err)) + self.logger.error("Could not create parent directories for " + "%s: %s" % (fileloc, err)) return try: @@ -164,14 +166,14 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, return def verify_file(self, filename, contents, metadata): - # Service the FAM events queued up by the key generation so - # the data structure entries will be available for binding. - # - # NOTE: We wait for up to ten seconds. There is some potential - # for race condition, because if the file monitor doesn't get - # notified about the new key files in time, those entries - # won't be available for binding. In practice, this seems - # "good enough". + """ Service the FAM events queued up by the key generation so + the data structure entries will be available for binding. + + NOTE: We wait for up to ten seconds. There is some potential + for race condition, because if the file monitor doesn't get + notified about the new key files in time, those entries won't + be available for binding. In practice, this seems "good + enough".""" entry = self.entries[metadata.hostname][filename] cfg = self.core.plugins['Cfg'] tries = 0 @@ -202,16 +204,12 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, return self.logger.info("Writing %s for %s" % (infoxml, data.get("name"))) - info = \ - lxml.etree.Element("Info", - owner=data.get("owner", - Bcfg2.Options.MDATA_OWNER.value), - group=data.get("group", - Bcfg2.Options.MDATA_GROUP.value), - perms=data.get("perms", - Bcfg2.Options.MDATA_PERMS.value), - encoding=entry.get("encoding", - Bcfg2.Options.ENCODING.value)) + info = lxml.etree.Element( + "Info", + owner=data.get("owner", Bcfg2.Options.MDATA_OWNER.value), + group=data.get("group", Bcfg2.Options.MDATA_GROUP.value), + perms=data.get("perms", Bcfg2.Options.MDATA_PERMS.value), + encoding=entry.get("encoding", Bcfg2.Options.ENCODING.value)) root = lxml.etree.Element("FileInfo") root.append(info) diff --git a/src/lib/Bcfg2/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index 1b1627688..85d0f38f5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -1,37 +1,22 @@ -import os +""" The Fossil plugin provides a revision interface for Bcfg2 repos +using fossil.""" + from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Fossil') class Fossil(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """Fossil is a version plugin for dealing with Bcfg2 repos.""" - name = 'Fossil' + """ The Fossil plugin provides a revision interface for Bcfg2 + repos using fossil. """ __author__ = 'bcfg-dev@mcs.anl.gov' + __vcs_metadata_path__ = "_FOSSIL_" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to fossil file for bcfg2 repo - fossil_file = "%s/_FOSSIL_" % datastore - - # Read revision from bcfg2 repo - if os.path.isfile(fossil_file): - revision = self.get_revision() - elif not os.path.isdir(datastore): - logger.error("%s is not a directory" % datastore) - raise Bcfg2.Server.Plugin.PluginInitError - else: - logger.error("%s is not a file" % fossil_file) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized Fossil.py plugin with %(ffile)s at revision %(frev)s" \ - % {'ffile': fossil_file, 'frev': revision}) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized Fossil plugin with fossil directory %s" + % self.vcs_path) def get_revision(self): """Read fossil revision information for the Bcfg2 repository.""" @@ -42,10 +27,10 @@ class Fossil(Bcfg2.Server.Plugin.Plugin, stdout=PIPE).stdout.readlines() revline = [line.split(': ')[1].strip() for line in data if \ line.split(': ')[0].strip() == 'checkout'][-1] - revision = revline.split(' ')[0] + return revline.split(' ')[0] except IndexError: - logger.error("Failed to read fossil info; disabling fossil support") - logger.error('''Ran command "fossil info" from directory "%s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError - return revision + msg = "Failed to read fossil info" + self.logger.error(msg) + self.logger.error('Ran command "fossil info" from directory "%s"' % + self.datastore) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 8f8ea87f1..30416a147 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -1,44 +1,28 @@ -"""The Git plugin provides a revision interface for Bcfg2 repos using git.""" +""" The Git plugin provides a revision interface for Bcfg2 repos using +git. """ -import os -from dulwich.repo import Repo +from dulwich.repo import Repo # pylint: disable=F0401 import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Git') - class Git(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """Git is a version plugin for dealing with Bcfg2 repos.""" - name = 'Git' + """ The Git plugin provides a revision interface for Bcfg2 repos + using git. """ __author__ = 'bcfg-dev@mcs.anl.gov' + __vcs_metadata_path__ = ".git" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to git directory for bcfg2 repo - git_dir = "%s/.git" % datastore - - # Read revision from bcfg2 repo - if os.path.isdir(git_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % git_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized git plugin with git directory %s" % git_dir) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized git plugin with git directory %s" % + self.vcs_path) def get_revision(self): """Read git revision information for the Bcfg2 repository.""" try: - repo = Repo(self.datastore) - revision = repo.head() + return Repo(self.datastore).head() except: - logger.error("Failed to read git repository; disabling git support") - raise Bcfg2.Server.Plugin.PluginInitError - return revision + msg = "Failed to read git repository" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Guppy.py b/src/lib/Bcfg2/Server/Plugins/Guppy.py index eea92f30f..d13e3f061 100644 --- a/src/lib/Bcfg2/Server/Plugins/Guppy.py +++ b/src/lib/Bcfg2/Server/Plugins/Guppy.py @@ -26,16 +26,16 @@ Remote interactive console. To return to Annex, type '-'. """ -import re import Bcfg2.Server.Plugin + class Guppy(Bcfg2.Server.Plugin.Plugin): """Guppy is a debugging plugin to help trace memory leaks""" name = 'Guppy' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = True - __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable','Disable'] + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable', 'Disable'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -59,4 +59,3 @@ class Guppy(Bcfg2.Server.Plugin.Plugin): except: self.logger.error("Failed to disable Heapy") raise Bcfg2.Server.Plugin.PluginInitError - diff --git a/src/lib/Bcfg2/Server/Plugins/Hg.py b/src/lib/Bcfg2/Server/Plugins/Hg.py index 0c3537613..b5dec2e3f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Hg.py +++ b/src/lib/Bcfg2/Server/Plugins/Hg.py @@ -1,45 +1,32 @@ -import os +""" The Hg plugin provides a revision interface for Bcfg2 repos using +mercurial. """ + from mercurial import ui, hg import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Mercurial') class Hg(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Mercurial is a version plugin for dealing with Bcfg2 repository.""" - name = 'Mercurial' + Bcfg2.Server.Plugin.Version): + """ The Hg plugin provides a revision interface for Bcfg2 repos + using mercurial. """ + __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True + __vcs_metadata_path__ = ".hg" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to hg directory for Bcfg2 repo - hg_dir = "%s/.hg" % datastore - - # Read changeset from bcfg2 repo - if os.path.isdir(hg_dir): - self.get_revision() - else: - logger.error("%s is not present." % hg_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized hg plugin with hg directory = %s" % hg_dir) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized hg plugin with hg directory %s" % + self.vcs_path) def get_revision(self): """Read hg revision information for the Bcfg2 repository.""" try: - repo_path = "%s/" % self.datastore + repo_path = self.datastore + "/" repo = hg.repository(ui.ui(), repo_path) tip = repo.changelog.tip() - revision = repo.changelog.rev(tip) + return repo.changelog.rev(tip) except: - logger.error("Failed to read hg repository; disabling mercurial support") - raise Bcfg2.Server.Plugin.PluginInitError - return revision - + msg = "Failed to read hg repository" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index 9883085db..8e5ce2624 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -93,7 +93,7 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): return data except Exception: if hasattr(query, "name"): - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + logger.error("LdapPlugin error: " + "Exception during processing of query named '" + str(query.name) + "', query results will be empty" + @@ -101,7 +101,7 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): for line in traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]): - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + logger.error("LdapPlugin error: " + line.replace("\n", "")) return {} @@ -130,7 +130,7 @@ class LdapConnection(object): result = None for attempt in range(RETRY_COUNT + 1): if attempt >= 1: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + logger.error("LdapPlugin error: " + "LDAP server down (retry " + str(attempt) + "/" + str(RETRY_COUNT) + ")") try: @@ -207,7 +207,7 @@ class LdapQuery(object): self.result = self.process_result(metadata) return self.result else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + logger.error("LdapPlugin error: " + "No valid connection defined for query " + str(self)) return None @@ -240,6 +240,6 @@ class LdapSubQuery(LdapQuery): self.result = self.connection.run_query(self) return self.process_result(metadata, **kwargs) else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + logger.error("LdapPlugin error: " + "No valid connection defined for query " + str(self)) return None diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 5d0b35835..e0904339f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -105,7 +105,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): Bcfg2.Server.FileMonitor.Pseudo) @property - def xdata(self): + def xdata(self): # pylint: disable=E0202 if not self.data: raise Bcfg2.Server.Plugin.MetadataRuntimeError("%s has no data" % self.basefile) @@ -334,7 +334,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" __author__ = 'bcfg-dev@mcs.anl.gov' - name = "Metadata" sort_order = 500 def __init__(self, core, datastore, watch_clients=True): @@ -345,14 +344,14 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.states = dict() self.extra = dict() self.handlers = [] - self._handle_file("groups.xml") + self.groups_xml = self._handle_file("groups.xml") if (self._use_db and os.path.exists(os.path.join(self.data, "clients.xml"))): self.logger.warning("Metadata: database enabled but clients.xml" "found, parsing in compatibility mode") - self._handle_file("clients.xml") + self.clients_xml = self._handle_file("clients.xml") elif not self._use_db: - self._handle_file("clients.xml") + self.clients_xml = self._handle_file("clients.xml") # mapping of clientname -> authtype self.auth = dict() @@ -415,9 +414,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.states[fname] = False aname = re.sub(r'[^A-z0-9_]', '_', fname) xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname) - setattr(self, aname, xmlcfg) self.handlers.append(xmlcfg.HandleEvent) self.extra[fname] = [] + return xmlcfg def _search_xdata(self, tag, name, tree, alias=False): for node in tree.findall("//%s" % tag): @@ -456,9 +455,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def add_group(self, group_name, attribs): """Add group to groups.xml.""" if self._use_db: - msg = "Metadata does not support adding groups with use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support adding groups with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._add_xdata(self.groups_xml, "Group", group_name, attribs=attribs) @@ -466,9 +466,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def add_bundle(self, bundle_name): """Add bundle to groups.xml.""" if self._use_db: - msg = "Metadata does not support adding bundles with use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support adding bundles with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._add_xdata(self.groups_xml, "Bundle", bundle_name) @@ -503,9 +504,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def update_group(self, group_name, attribs): """Update a groups attributes.""" if self._use_db: - msg = "Metadata does not support updating groups with use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support updating groups with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._update_xdata(self.groups_xml, "Group", group_name, attribs) @@ -513,9 +515,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def update_client(self, client_name, attribs): """Update a clients attributes.""" if self._use_db: - msg = "Metadata does not support updating clients with use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support updating clients with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._update_xdata(self.clients_xml, "Client", client_name, attribs, alias=True) @@ -544,16 +547,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def remove_group(self, group_name): """Remove a group.""" if self._use_db: - msg = "Metadata does not support removing groups with use_database enabled" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + msg = "Metadata does not support removing groups with " + \ + "use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._remove_xdata(self.groups_xml, "Group", group_name) def remove_bundle(self, bundle_name): """Remove a bundle.""" if self._use_db: - msg = "Metadata does not support removing bundles with use_database enabled" + msg = "Metadata does not support removing bundles with " + \ + "use_database enabled" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: @@ -652,8 +657,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.warning("%s: Group %s suppressed by " "category %s; %s already a member " "of %s" % - (self.name, gname, category, client, - categories[category])) + (self.name, gname, category, + client, categories[category])) if gname in self.groups: self.groups[gname].warned.append(client) return False @@ -690,7 +695,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership = dict() self.negated_groups = dict() - self.options = dict() + # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is # infinitely recursive, so "all" elements really means _all_ @@ -727,7 +732,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def HandleEvent(self, event): """Handle update events for data files.""" for hdlr in self.handlers: - aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(event.filename)) + aname = re.sub(r'[^A-z0-9_]', '_', + os.path.basename(event.filename)) if hdlr(event): # clear the entire cache when we get an event for any # metadata file @@ -735,7 +741,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, try: proc = getattr(self, "_handle_%s_event" % aname) except AttributeError: - proc = self._handle_default_event + proc = self._handle_default_event # pylint: disable=E1101 proc(event) if False not in list(self.states.values()) and self.debug_flag: @@ -745,8 +751,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, for client, groups in list(self.clientgroups.items()): for group in groups: if group not in self.groups: - self.debug_log("Client %s set as nonexistent group %s" % - (client, group)) + self.debug_log("Client %s set as nonexistent group %s" + % (client, group)) for gname, ginfo in list(self.groups.items()): for group in ginfo.groups: if group not in self.groups: @@ -765,7 +771,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] if not force and not group.is_public: - msg = "Cannot set client %s to private group %s" % (client, profile) + msg = "Cannot set client %s to private group %s" % (client, + profile) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) @@ -831,13 +838,13 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # be faster? curtime = time.time() for addrpair in list(self.session_cache.keys()): - if addresspair[0] == addrpair[0]: - (stamp, _) = self.session_cache[addrpair] - if curtime - stamp > cache_ttl: - del self.session_cache[addrpair] + if addresspair[0] == addrpair[0]: + (stamp, _) = self.session_cache[addrpair] + if curtime - stamp > cache_ttl: + del self.session_cache[addrpair] # return the cached data try: - (stamp, uuid) = self.session_cache[addresspair] + stamp = self.session_cache[addresspair][0] if time.time() - stamp < cache_ttl: return self.session_cache[addresspair][1] except KeyError: @@ -846,7 +853,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, address = addresspair[0] if address in self.addresses: if len(self.addresses[address]) != 1: - err = "Address %s has multiple reverse assignments; a uuid must be used" % address + err = "Address %s has multiple reverse assignments; a " + \ + "uuid must be used" % address self.logger.error(err) raise Bcfg2.Server.Plugin.MetadataConsistencyError(err) return self.addresses[address][0] @@ -1165,8 +1173,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, try: groups_tree.xinclude() except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for file %s: %s" % - (dest, sys.exc_info()[1])) + self.logger.error("Failed to process XInclude for groups.xml: %s" % + sys.exc_info()[1]) groups = groups_tree.getroot() categories = {'default': 'grey83'} viz_str = [] @@ -1253,7 +1261,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): def deprecated_options(self): clientdata = self.metadata.clients_xml.xdata - for el in groupdata.xpath("//Client"): + for el in clientdata.xpath("//Client"): loc = el.get("location") if loc: if loc == "floating": diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index d0e4a3665..bb98fe1e5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -78,8 +78,7 @@ import copy import logging import lxml.etree import Bcfg2.Server.Plugin -from Bcfg2.Compat import any, md5 -from Bcfg2.Server.Plugins.Packages.Source import Source +from Bcfg2.Compat import any, md5 # pylint: disable=W0622 LOGGER = logging.getLogger(__name__) @@ -180,15 +179,18 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): return "\n".join(srcs) def get_relevant_groups(self): - groups = [] - for source in self: - groups.extend(source.get_relevant_groups(self.metadata)) - return sorted(list(set(groups))) - get_relevant_groups.__doc__ = Source.get_relevant_groups.__doc__ + """ + """ Get all groups that might be relevant to determining which + sources apply to this collection's client. The base implementation simply aggregates the results of :func:`Bcfg2.Server.Plugins.Packages.Source.Source.get_relevant_groups` + + :return: list of strings - group names """ + groups = [] + for source in self: + groups.extend(source.get_relevant_groups(self.metadata)) + return sorted(list(set(groups))) @property def basegroups(self): @@ -253,8 +255,8 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): :ref:`pkg-objects` """ if not self.__package_groups__: - self.logger.error("Packages: Package groups are not supported by %s" - % self.__class__.__name__) + self.logger.error("Packages: Package groups are not supported by " + "%s" % self.__class__.__name__) return [] for source in self: @@ -484,7 +486,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ return list(complete.difference(initial)) - def complete(self, packagelist): + def complete(self, packagelist): # pylint: disable=R0912,R0914 """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the @@ -562,7 +564,8 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): if len(vpkg_cache[current]) == 1: self.debug_log("Packages: requirement %s satisfied by %s" % (current, vpkg_cache[current])) - unclassified.update(vpkg_cache[current].difference(examined)) + unclassified.update( + vpkg_cache[current].difference(examined)) satisfied_vpkgs.add(current) else: satisfiers = [item for item in vpkg_cache[current] @@ -623,7 +626,8 @@ def get_collection_class(source_type): try: cclass = getattr(module, source_type.title() + "Collection") except AttributeError: - msg = "Packages: No collection class found for %s sources" % source_type + msg = "Packages: No collection class found for %s sources" % \ + source_type LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) return cclass diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 329dfc394..2519ddff6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -1,3 +1,6 @@ +""" PackagesSources handles the +:ref:`server-plugins-generators-packages` ``sources.xml`` file""" + import os import sys import Bcfg2.Server.Plugin diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 26f3ab92f..22d488121 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -92,7 +92,7 @@ class SourceInitError(Exception): REPO_RE = re.compile(r'(?:pulp/repos/|/RPMS\.|/)([^/]+)/?$') -class Source(Bcfg2.Server.Plugin.Debuggable): +class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 """ ``Source`` objects represent a single tag in ``sources.xml``. Note that a single Source tag can itself describe multiple repositories (if it uses the "url" attribute @@ -116,7 +116,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`. ptype = None - def __init__(self, basepath, xsource, setup): + def __init__(self, basepath, xsource, setup): # pylint: disable=R0912 """ :param basepath: The base filesystem path under which cache data for this source should be stored @@ -301,7 +301,8 @@ class Source(Bcfg2.Server.Plugin.Debuggable): """ Get all groups that might be relevant to determining which sources apply to this collection's client. - :return: list of strings - group names""" + :return: list of strings - group names + """ return sorted(list(set([g for g in metadata.groups if (g in self.basegroups or g in self.groups or @@ -349,6 +350,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): upstream repository. :type force_update: bool """ + # pylint: disable=W0702 if not force_update: if os.path.exists(self.cachefile): try: @@ -379,6 +381,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): self.logger.error("Packages: Failed to load data for %s: %s" % (self, err)) self.logger.error("Some Packages will be missing") + # pylint: enable=W0702 def get_repo_name(self, url_map): """ Try to find a sensible name for a repository. Since @@ -485,7 +488,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): vdict[key].update(value) return vdict - def is_virtual_package(self, metadata, package): + def is_virtual_package(self, metadata, package): # pylint: disable=W0613 """ Return True if a name is a virtual package (i.e., is a symbol provided by a real package), False otherwise. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 1d7eca808..4224798a8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -59,15 +59,17 @@ import logging import lxml.etree from subprocess import Popen, PIPE import Bcfg2.Server.Plugin +# pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, json, any +# pylint: enable=W0622 from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) -# pylint: disable=E0611 +# pylint: disable=E0611,F0401 try: from pulp.client.consumer.config import ConsumerConfig from pulp.client.api.repository import RepositoryAPI @@ -82,9 +84,9 @@ try: HAS_YUM = True except ImportError: HAS_YUM = False - logger.info("Packages: No yum libraries found; forcing use of internal " + LOGGER.info("Packages: No yum libraries found; forcing use of internal " "dependency resolver") -# pylint: enable=E0611 +# pylint: enable=E0611,F0401 XP = '{http://linux.duke.edu/metadata/common}' RP = '{http://linux.duke.edu/metadata/rpm}' @@ -108,7 +110,7 @@ def _setup_pulp(setup): if not HAS_PULP: msg = "Packages: Cannot create Pulp collection: Pulp libraries " + \ "not found" - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) if PULPSERVER is None: @@ -117,12 +119,12 @@ def _setup_pulp(setup): password = setup.cfp.get("packages:pulp", "password") except ConfigParser.NoSectionError: msg = "Packages: No [pulp] section found in bcfg2.conf" - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) except ConfigParser.NoOptionError: msg = "Packages: Required option not found in bcfg2.conf: %s" % \ sys.exc_info()[1] - logger.error(msg) + LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) PULPCONFIG = ConsumerConfig() @@ -265,7 +267,7 @@ class YumCollection(Collection): yumconf.write(open(self.cfgfile, 'w')) - def get_config(self, raw=False): + def get_config(self, raw=False): # pylint: disable=W0221 """ Get the yum configuration for this collection. :param raw: Return a :class:`ConfigParser.SafeConfigParser` @@ -528,7 +530,7 @@ class YumCollection(Collection): ptype = "default" gdicts.append(dict(group=group, type=ptype)) - return self.call_helper("get_groups", gdicts) + return self.call_helper("get_groups", inputdata=gdicts) def packages_from_entry(self, entry): """ When using the Python yum libraries, convert a Package @@ -543,6 +545,7 @@ class YumCollection(Collection): name = entry.get("name") def _tag_to_pkg(tag): + """ Convert a Package or Instance tag to a package tuple """ rv = (name, tag.get("arch"), tag.get("epoch"), tag.get("version"), tag.get("release")) if rv[3] in ['any', 'auto']: @@ -584,6 +587,9 @@ class YumCollection(Collection): :returns: None """ def _get_entry_attrs(pkgtup): + """ Given a package tuple, return a dict of attributes + suitable for applying to either a Package or an Instance + tag """ attrs = dict(version=self.setup.cfp.get("packages", "version", default="auto")) @@ -685,7 +691,7 @@ class YumCollection(Collection): else: return set(), set() - def call_helper(self, command, input=None): + def call_helper(self, command, inputdata=None): """ Make a call to :ref:`bcfg2-yum-helper`. The yum libs have horrific memory leaks, so apparently the right way to get around that in long-running processes it to have a short-lived @@ -694,10 +700,10 @@ class YumCollection(Collection): :param command: The :ref:`bcfg2-yum-helper` command to call. :type command: string - :param input: The input to pass to ``bcfg2-yum-helper`` on - stdin. If this is None, no input will be given - at all. - :type input: Any JSON-encodable data structure. + :param inputdata: The input to pass to ``bcfg2-yum-helper`` on + stdin. If this is None, no input will be + given at all. + :type inputdata: Any JSON-encodable data structure. :returns: Varies depending on the return value of the ``bcfg2-yum-helper`` command. """ @@ -715,8 +721,8 @@ class YumCollection(Collection): (" ".join(cmd), err)) return None - if input: - idata = json.dumps(input) + if inputdata: + idata = json.dumps(inputdata) (stdout, stderr) = helper.communicate(idata) else: (stdout, stderr) = helper.communicate() @@ -938,7 +944,7 @@ class YumSource(Source): try: self.packages['global'] = copy.deepcopy(sdata.pop()) except IndexError: - logger.error("Packages: No packages in repo") + self.logger.error("Packages: No packages in repo") while sdata: self.packages['global'] = \ self.packages['global'].intersection(sdata.pop()) @@ -951,6 +957,7 @@ class YumSource(Source): self.save_state() def parse_filelist(self, data, arch): + """ parse filelists.xml.gz data """ if arch not in self.filemap: self.filemap[arch] = dict() for pkg in data.findall(FL + 'package'): @@ -963,6 +970,7 @@ class YumSource(Source): set([pkg.get('name')]) def parse_primary(self, data, arch): + """ parse primary.xml.gz data """ if arch not in self.packages: self.packages[arch] = set() if arch not in self.deps: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index fd6369619..6f8e3ecad 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -1,3 +1,7 @@ +""" Packages resolves Package entries on the Bcfg2 server in order to +present a complete list of Package entries to the client in order to +determine the completeness of the client configuration. """ + import os import sys import glob @@ -68,6 +72,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.logger.warning("You can disable magic groups by setting " "magic_groups=0 in [packages] in bcfg2.conf") + # pylint: disable=C0301 #: The #: :class:`Bcfg2.Server.Plugins.Packages.PackagesSources.PackagesSources` #: object used to generate @@ -98,12 +103,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.collections = dict() #: clients is a cache mapping of hostname -> - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`. + #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` #: Unlike :attr:`collections`, this _is_ used to return a #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = dict() + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 68cdce6e8..9e6b43d7b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -1,3 +1,5 @@ +""" A plugin to gather information from a client machine """ + import re import os import sys @@ -7,29 +9,15 @@ import operator import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin -from Bcfg2.Compat import any, json +from Bcfg2.Compat import json +# pylint: disable=F0401 try: from django.db import models - has_django = True -except ImportError: - has_django = False -try: - import syck as yaml - has_yaml = True - yaml_error = yaml.error -except ImportError: - try: - import yaml - yaml_error = yaml.YAMLError - has_yaml = True - except ImportError: - has_yaml = False - -if has_django: class ProbesDataModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): + """ The database model for storing probe data """ hostname = models.CharField(max_length=255) probe = models.CharField(max_length=255) timestamp = models.DateTimeField(auto_now=True) @@ -37,8 +25,24 @@ if has_django: class ProbesGroupsModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): + """ The database model for storing probe groups """ hostname = models.CharField(max_length=255) group = models.CharField(max_length=255) +except ImportError: + pass + +try: + import syck as yaml + import syck.error as YAMLError + HAS_YAML = True +except ImportError: + try: + import yaml + from yaml import YAMLError + HAS_YAML = True + except ImportError: + HAS_YAML = False +# pylint: enable=F0401 class ClientProbeDataSet(dict): @@ -58,8 +62,8 @@ class ProbeData(str): ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): return str.__new__(cls, data) - - def __init__(self, data): + + def __init__(self): str.__init__(self) self._xdata = None self._json = None @@ -70,9 +74,10 @@ class ProbeData(str): """ provide backwards compatibility with broken ProbeData object in bcfg2 1.2.0 thru 1.2.2 """ return str(self) - + @property def xdata(self): + """ The probe data as a lxml.etree._Element document """ if self._xdata is None: try: self._xdata = lxml.etree.XML(self.data, @@ -83,6 +88,7 @@ class ProbeData(str): @property def json(self): + """ The probe data as a decoded JSON data structure """ if self._json is None: try: self._json = json.loads(self.data) @@ -92,17 +98,20 @@ class ProbeData(str): @property def yaml(self): - if self._yaml is None and has_yaml: + """ The probe data as a decoded YAML data structure """ + if self._yaml is None and HAS_YAML: try: self._yaml = yaml.load(self.data) - except yaml_error: + except YAMLError: pass return self._yaml class ProbeSet(Bcfg2.Server.Plugin.EntrySet): + """ Handle universal and group- and host-specific probe files """ ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") - probename = re.compile("(.*/)?(?P\S+?)(\.(?P(?:G\d\d)|H)_\S+)?$") + probename = \ + re.compile("(.*/)?(?P\S+?)(\.(?P(?:G\d\d)|H)_\S+)?$") bangline = re.compile('^#!\s*(?P.*)$') basename_is_regex = True @@ -113,12 +122,15 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): encoding) fam.AddMonitor(path, self) - def HandleEvent(self, event): - if (event.filename != self.path and - not event.filename.endswith("probed.xml")): - return self.handle_event(event) - def get_probe_data(self, metadata): + """ Get an XML description of all probes for a client suitable + for sending to that client. + + :params metadata: The client metadata to get probes for. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: list of lxml.etree._Element objects, each of which + represents one probe. + """ ret = [] build = dict() candidates = self.get_matching(metadata) @@ -146,8 +158,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): - """A plugin to gather information from a client machine.""" - name = 'Probes' + """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): @@ -165,33 +176,37 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.probedata = dict() self.cgroups = dict() self.load_data() + __init__.__doc__ = Bcfg2.Server.Plugin.DatabaseBacked.__init__.__doc__ def write_data(self, client): - """Write probe data out for use with bcfg2-info.""" + """ Write probe data out for use with bcfg2-info """ if self._use_db: return self._write_data_db(client) else: return self._write_data_xml(client) def _write_data_xml(self, _): + """ Write received probe data to probed.xml """ top = lxml.etree.Element("Probed") for client, probed in sorted(self.probedata.items()): - cx = lxml.etree.SubElement(top, 'Client', name=client, - timestamp=str(int(probed.timestamp))) + ctag = lxml.etree.SubElement(top, 'Client', name=client, + timestamp=str(int(probed.timestamp))) for probe in sorted(probed): - lxml.etree.SubElement(cx, 'Probe', name=probe, + lxml.etree.SubElement(ctag, 'Probe', name=probe, value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): - lxml.etree.SubElement(cx, "Group", name=group) + lxml.etree.SubElement(ctag, "Group", name=group) try: datafile = open(os.path.join(self.data, 'probed.xml'), 'w') - datafile.write(lxml.etree.tostring(top, xml_declaration=False, - pretty_print='true').decode('UTF-8')) + datafile.write(lxml.etree.tostring( + top, xml_declaration=False, + pretty_print='true').decode('UTF-8')) except IOError: err = sys.exc_info()[1] self.logger.error("Failed to write probed.xml: %s" % err) def _write_data_db(self, client): + """ Write received probe data to the database """ for probe, data in self.probedata[client.hostname].items(): pdata = \ ProbesDataModel.objects.get_or_create(hostname=client.hostname, @@ -199,7 +214,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, if pdata.data != data: pdata.data = data pdata.save() - ProbesDataModel.objects.filter(hostname=client.hostname).exclude(probe__in=self.probedata[client.hostname]).delete() + ProbesDataModel.objects.filter( + hostname=client.hostname).exclude( + probe__in=self.probedata[client.hostname]).delete() for group in self.cgroups[client.hostname]: try: @@ -209,19 +226,24 @@ class Probes(Bcfg2.Server.Plugin.Probing, grp = ProbesGroupsModel(hostname=client.hostname, group=group) grp.save() - ProbesGroupsModel.objects.filter(hostname=client.hostname).exclude(group__in=self.cgroups[client.hostname]).delete() + ProbesGroupsModel.objects.filter( + hostname=client.hostname).exclude( + group__in=self.cgroups[client.hostname]).delete() def load_data(self): + """ Load probe data from the appropriate backend (probed.xml + or the database) """ if self._use_db: return self._load_data_db() else: return self._load_data_xml() - + def _load_data_xml(self): + """ Load probe data from probed.xml """ try: data = lxml.etree.parse(os.path.join(self.data, 'probed.xml'), parser=Bcfg2.Server.XMLParser).getroot() - except: + except (IOError, lxml.etree.XMLSyntaxError): err = sys.exc_info()[1] self.logger.error("Failed to read file probed.xml: %s" % err) return @@ -239,21 +261,22 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[client.get('name')].append(pdata.get('name')) def _load_data_db(self): + """ Load probe data from the database """ self.probedata = {} self.cgroups = {} for pdata in ProbesDataModel.objects.all(): if pdata.hostname not in self.probedata: - self.probedata[pdata.hostname] = \ - ClientProbeDataSet(timestamp=time.mktime(pdata.timestamp.timetuple())) + self.probedata[pdata.hostname] = ClientProbeDataSet( + timestamp=time.mktime(pdata.timestamp.timetuple())) self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) for pgroup in ProbesGroupsModel.objects.all(): if pgroup.hostname not in self.cgroups: self.cgroups[pgroup.hostname] = [] self.cgroups[pgroup.hostname].append(pgroup.group) - def GetProbes(self, meta, force=False): - """Return a set of probes for execution on client.""" + def GetProbes(self, meta): return self.probes.get_probe_data(meta) + GetProbes.__doc__ = Bcfg2.Server.Plugin.Probing.GetProbes.__doc__ def ReceiveData(self, client, datalist): if self.core.metadata_cache_mode in ['cautious', 'aggressive']: @@ -271,6 +294,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, olddata != self.cgroups[client.hostname]): self.core.metadata_cache.expire(client.hostname) self.write_data(client) + ReceiveData.__doc__ = Bcfg2.Server.Plugin.Probing.ReceiveData.__doc__ def ReceiveDataItem(self, client, data): """Receive probe results pertaining to client.""" @@ -299,6 +323,10 @@ class Probes(Bcfg2.Server.Plugin.Probing, def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) + get_additional_groups.__doc__ = \ + Bcfg2.Server.Plugin.Connector.get_additional_groups.__doc__ def get_additional_data(self, meta): return self.probedata.get(meta.hostname, ClientProbeDataSet()) + get_additional_data.__doc__ = \ + Bcfg2.Server.Plugin.Connector.get_additional_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 590d536a9..1d5bdbcfc 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -1,3 +1,6 @@ +""" The properties plugin maps property files into client metadata +instances. """ + import os import re import sys @@ -89,6 +92,7 @@ class PropertyFile(Bcfg2.Server.Plugin.StructFile): raise PluginExecutionError(msg) def _decrypt(self, element): + """ Decrypt a single encrypted properties file element """ if not element.text.strip(): return passes = get_passphrases(SETUP) @@ -108,6 +112,7 @@ class PropertyFile(Bcfg2.Server.Plugin.StructFile): class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): + """ A collection of properties files """ __child__ = PropertyFile patterns = re.compile(r'.*\.xml$') ignore = re.compile(r'.*\.xsd$') @@ -115,22 +120,20 @@ class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): class Properties(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): - """ - The properties plugin maps property - files into client metadata instances. - """ + """ The properties plugin maps property files into client metadata + instances. """ name = 'Properties' def __init__(self, core, datastore): - global SETUP + global SETUP # pylint: disable=W0603 Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) try: self.store = PropDirectoryBacked(self.data, core.fam) except OSError: - e = sys.exc_info()[1] - self.logger.error("Error while creating Properties store: %s %s" % - (e.strerror, e.filename)) + err = sys.exc_info()[1] + self.logger.error("Error while creating Properties store: %s" % + err) raise Bcfg2.Server.Plugin.PluginInitError SETUP = core.setup diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 341d63118..ebcbf3bcc 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -1,8 +1,12 @@ +""" A plugin to run Puppet external node classifiers """ + import os +import sys import Bcfg2.Server import Bcfg2.Server.Plugin from subprocess import Popen, PIPE +# pylint: disable=F0401 try: from syck import load as yaml_load, error as yaml_error except ImportError: @@ -10,8 +14,12 @@ except ImportError: from yaml import load as yaml_load, YAMLError as yaml_error except ImportError: raise ImportError("No yaml library could be found") +# pylint: enable=F0401 + class PuppetENCFile(Bcfg2.Server.Plugin.FileBacked): + """ A representation of a Puppet external node classifier script """ + def HandleEvent(self, event=None): return @@ -22,7 +30,6 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): """ A plugin to run Puppet external node classifiers (http://docs.puppetlabs.com/guides/external_nodes.html) """ - name = 'PuppetENC' experimental = True __child__ = PuppetENCFile @@ -35,6 +42,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, self.cache = dict() def _run_encs(self, metadata): + """ Run all Puppet ENCs """ cache = dict(groups=[], params=dict()) for enc in self.entries.keys(): epath = os.path.join(self.data, enc) @@ -46,8 +54,8 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, rv = proc.wait() if rv != 0: msg = "PuppetENC: Error running ENC %s for %s (%s): %s" % \ - (enc, metadata.hostname, rv) - self.logger.error("%s: %s" % (msg, err)) + (enc, metadata.hostname, rv, err) + self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) if err: self.debug_log("ENC Error: %s" % err) @@ -62,8 +70,8 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, (enc, metadata.hostname, err) self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - groups = [] + + groups = dict() if "classes" in yaml: # stock Puppet ENC output format groups = yaml['classes'] @@ -87,7 +95,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, if "environment" in yaml: self.logger.info("Ignoring unsupported environment section of " "ENC %s for %s" % (enc, metadata.hostname)) - + self.cache[metadata.hostname] = cache def get_additional_groups(self, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index e77d08653..904876794 100644 --- a/src/lib/Bcfg2/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -3,6 +3,7 @@ import re import Bcfg2.Server.Plugin + class Rules(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles service assignments.""" name = 'Rules' @@ -48,4 +49,5 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): return False def _regex_enabled(self): + """ Return True if rules regexes are enabled, False otherwise """ return self.core.setup.cfp.getboolean("rules", "regex", default=False) diff --git a/src/lib/Bcfg2/Server/Plugins/SEModules.py b/src/lib/Bcfg2/Server/Plugins/SEModules.py index b04e2d359..3edfb72a3 100644 --- a/src/lib/Bcfg2/Server/Plugins/SEModules.py +++ b/src/lib/Bcfg2/Server/Plugins/SEModules.py @@ -10,12 +10,9 @@ See :ref:`server-selinux` for more information. """ import os -import logging import Bcfg2.Server.Plugin from Bcfg2.Compat import b64encode -logger = logging.getLogger(__name__) - class SEModuleData(Bcfg2.Server.Plugin.SpecificData): """ Representation of a single SELinux module file. Encodes the diff --git a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py index f1309412a..0aea439f9 100644 --- a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py @@ -1,5 +1,8 @@ +""" Use old-style service modes for older clients """ + import Bcfg2.Server.Plugin + class ServiceCompat(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.GoalValidator): """ Use old-style service modes for older clients """ diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index ae43388ea..5bc9d6bbd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -1,35 +1,23 @@ -import os +""" The Svn plugin provides a revision interface for Bcfg2 repos using +svn. """ + import pipes from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Svn') - class Svn(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): - """Svn is a version plugin for dealing with Bcfg2 repos.""" - name = 'Svn' + """ The Svn plugin provides a revision interface for Bcfg2 repos + using svn. """ __author__ = 'bcfg-dev@mcs.anl.gov' + __vcs_metadata_path__ = ".svn" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to svn directory for bcfg2 repo - svn_dir = "%s/.svn" % datastore - - # Read revision from bcfg2 repo - if os.path.isdir(svn_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % svn_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized svn plugin with svn directory = %s" % svn_dir) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) + self.logger.debug("Initialized svn plugin with svn directory %s" % + self.vcs_path) def get_revision(self): """Read svn revision information for the Bcfg2 repository.""" @@ -40,7 +28,7 @@ class Svn(Bcfg2.Server.Plugin.Plugin, return [line.split(': ')[1] for line in data \ if line[:9] == 'Revision:'][-1] except IndexError: - logger.error("Failed to read svn info; disabling svn support") - logger.error('''Ran command "svn info %s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError + msg = "Failed to read svn info" + self.logger.error(msg) + self.logger.error('Ran command "svn info %s"' % self.datastore) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Svn2.py b/src/lib/Bcfg2/Server/Plugins/Svn2.py index e4df9574f..110f2c2eb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn2.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn2.py @@ -1,41 +1,47 @@ +""" The Svn2 plugin provides a revision interface for Bcfg2 repos using +Subversion. It uses a pure python backend and exposes two additional +XML-RPC methods. """ + +import sys +import Bcfg2.Server.Plugin +# pylint: disable=F0401 try: import pysvn - missing = False -except: - missing = True -import Bcfg2.Server.Plugin + HAS_SVN = False +except ImportError: + HAS_SVN = True +# pylint: enable=F0401 + class Svn2(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" - name = 'Svn2' __author__ = 'bcfg-dev@mcs.anl.gov' conflicts = ['Svn'] - experimental = True - __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Update','Commit'] + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Update', 'Commit'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Version.__init__(self, datastore) - if missing: - self.logger.error("Svn2: Missing PySvn") - raise Bcfg2.Server.Plugin.PluginInitError + if HAS_SVN: + msg = "Missing PySvn" + self.logger.error("Svn2: " + msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) self.client = pysvn.Client() - self.core = core - self.datastore = datastore self.svn_root = None self.revision = None # Read revision from bcfg2 repo revision = self.get_revision() if not self.revision: - raise Bcfg2.Server.Plugin.PluginInitError + raise Bcfg2.Server.Plugin.PluginInitError("Failed to get revision") - self.logger.debug("Initialized svn plugin with svn root %s at revision %s" - % (self.svn_root, revision)) + self.logger.debug("Initialized svn plugin with svn root %s at " + "revision %s" % (self.svn_root, revision)) def get_revision(self): """Read svn revision information for the Bcfg2 repository.""" @@ -44,55 +50,18 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, self.revision = info.revision self.svn_root = info.url return str(self.revision.number) - except: + except pysvn.ClientError: # pylint: disable=E1101 self.logger.error("Svn2: Failed to get revision", exc_info=1) self.revision = None return str(-1) - def commit_data(self, file_list, comment=None): - """Commit changes into the repository""" - if not comment: - comment = 'Svn2: autocommit' - - # First try to update - if not self.Update(): - self.logger.error("Failed to update svn repository, refusing to commit changes") - return - - #FIXME - look for conflicts? - - for fname in file_list: - stat = self.client.status(fname) - self.client.add([f.path for f in stat \ - if f.text_status == pysvn.wc_status_kind.unversioned]) - try: - self.revision = self.client.checkin([self.datastore], comment, - recurse=True) - self.revision = self.client.update(self.datastore, recurse=True)[0] - self.logger.info("Svn2: Commited changes. At %s" % - self.revision.number) - except Exception: - err = sys.exc_info()[1] - # try to be smart about the error we got back - details = None - if "callback_ssl_server_trust_prompt" in str(err): - details = "SVN server certificate is not trusted" - elif "callback_get_login" in str(err): - details = "SVN credentials not cached" - - if details is None: - self.logger.error("Svn2: Failed to commit changes", - exc_info=1) - else: - self.logger.error("Svn2: Failed to commit changes: %s" % - details) - def Update(self): '''Svn2.Update() => True|False\nUpdate svn working copy\n''' try: old_revision = self.revision.number self.revision = self.client.update(self.datastore, recurse=True)[0] - except Exception, err: + except pysvn.ClientError: # pylint: disable=E1101 + err = sys.exc_info()[1] # try to be smart about the error we got back details = None if "callback_ssl_server_trust_prompt" in str(err): @@ -104,8 +73,8 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, self.logger.error("Svn2: Failed to update server repository", exc_info=1) else: - self.logger.error("Svn2: Failed to update server repository: %s" % - details) + self.logger.error("Svn2: Failed to update server repository: " + "%s" % details) return False if old_revision == self.revision.number: @@ -117,10 +86,33 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, def Commit(self): """Svn2.Commit() => True|False\nCommit svn repository\n""" - try: - self.commit_changes([]) - return True - except: + # First try to update + if not self.Update(): + self.logger.error("Failed to update svn repository, refusing to " + "commit changes") return False + try: + self.revision = self.client.checkin([self.datastore], + 'Svn2: autocommit', + recurse=True) + self.revision = self.client.update(self.datastore, recurse=True)[0] + self.logger.info("Svn2: Commited changes. At %s" % + self.revision.number) + return True + except pysvn.ClientError: # pylint: disable=E1101 + err = sys.exc_info()[1] + # try to be smart about the error we got back + details = None + if "callback_ssl_server_trust_prompt" in str(err): + details = "SVN server certificate is not trusted" + elif "callback_get_login" in str(err): + details = "SVN credentials not cached" + if details is None: + self.logger.error("Svn2: Failed to commit changes", + exc_info=1) + else: + self.logger.error("Svn2: Failed to commit changes: %s" % + details) + return False diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 6d92bb530..627c82f25 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -1,47 +1,51 @@ +""" A plugin to provide helper classes and functions to templates """ + +import os import re import imp import sys -import glob import logging import Bcfg2.Server.Lint import Bcfg2.Server.Plugin -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) + +MODULE_RE = re.compile(r'(?P(?P[^\/]+)\.py)$') -module_pattern = r'(?P(?P[^\/]+)\.py)$' -module_re = re.compile(module_pattern) class HelperModule(Bcfg2.Server.Plugin.FileBacked): + """ Representation of a TemplateHelper module """ + def __init__(self, name, fam=None): Bcfg2.Server.Plugin.FileBacked.__init__(self, name, fam=fam) - self._module_name = module_re.search(self.name).group('module') + self._module_name = MODULE_RE.search(self.name).group('module') self._attrs = [] def Index(self): try: module = imp.load_source(self._module_name, self.name) - except: + except: # pylint: disable=W0702 err = sys.exc_info()[1] - logger.error("TemplateHelper: Failed to import %s: %s" % + LOGGER.error("TemplateHelper: Failed to import %s: %s" % (self.name, err)) return if not hasattr(module, "__export__"): - logger.error("TemplateHelper: %s has no __export__ list" % + LOGGER.error("TemplateHelper: %s has no __export__ list" % self.name) return newattrs = [] for sym in module.__export__: if sym not in self._attrs and hasattr(self, sym): - logger.warning("TemplateHelper: %s: %s is a reserved keyword, " + LOGGER.warning("TemplateHelper: %s: %s is a reserved keyword, " "skipping export" % (self.name, sym)) continue try: setattr(self, sym, getattr(module, sym)) newattrs.append(sym) except AttributeError: - logger.warning("TemplateHelper: %s: %s exports %s, but has no " + LOGGER.warning("TemplateHelper: %s exports %s, but has no " "such attribute" % (self.name, sym)) # remove old exports for sym in set(self._attrs) - set(newattrs): @@ -51,8 +55,9 @@ class HelperModule(Bcfg2.Server.Plugin.FileBacked): class HelperSet(Bcfg2.Server.Plugin.DirectoryBacked): + """ A set of template helper modules """ ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") - patterns = module_re + patterns = MODULE_RE __child__ = HelperModule @@ -68,7 +73,7 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, self.helpers = HelperSet(self.data, core.fam) def get_additional_data(self, _): - return dict([(h._module_name, h) + return dict([(h._module_name, h) # pylint: disable=W0212 for h in self.helpers.entries.values()]) @@ -76,24 +81,24 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerlessPlugin): """ find duplicate Pkgmgr entries with the same priority """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) - hm = HelperModule("foo.py") - self.reserved_keywords = dir(hm) + self.reserved_keywords = dir(HelperModule("foo.py")) def Run(self): for fname in os.listdir(os.path.join(self.config['repo'], "TemplateHelper")): helper = os.path.join(self.config['repo'], "TemplateHelper", fname) - if not module_re.search(helper) or not self.HandlesFile(helper): + if not MODULE_RE.search(helper) or not self.HandlesFile(helper): continue self.check_helper(helper) def check_helper(self, helper): - module_name = module_re.search(helper).group(1) + """ check a helper module for export errors """ + module_name = MODULE_RE.search(helper).group(1) try: module = imp.load_source(module_name, helper) - except: + except: # pylint: disable=W0702 err = sys.exc_info()[1] self.LintError("templatehelper-import-error", "Failed to import %s: %s" % @@ -120,8 +125,8 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerlessPlugin): (helper, sym)) elif sym.startswith("_"): self.LintError("templatehelper-underscore-export", - "%s: exported symbol %s starts with underscore" % - (helper, sym)) + "%s: exported symbol %s starts with underscore" + % (helper, sym)) @classmethod def Errors(cls): diff --git a/src/lib/Bcfg2/Server/Plugins/Trigger.py b/src/lib/Bcfg2/Server/Plugins/Trigger.py index 313a1bf03..7279edfed 100644 --- a/src/lib/Bcfg2/Server/Plugins/Trigger.py +++ b/src/lib/Bcfg2/Server/Plugins/Trigger.py @@ -1,9 +1,14 @@ +""" Trigger is a plugin that calls external scripts (on the server) """ + import os import pipes import Bcfg2.Server.Plugin from subprocess import Popen, PIPE + class TriggerFile(Bcfg2.Server.Plugin.FileBacked): + """ Representation of a trigger script file """ + def HandleEvent(self, event=None): return @@ -25,6 +30,8 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, self.core.fam) def async_run(self, args): + """ Run the trigger script asynchronously in a forked process + """ pid = os.fork() if pid: os.waitpid(pid, 0) @@ -34,15 +41,14 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, self.debug_log("Running %s" % " ".join(pipes.quote(a) for a in args)) proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) - (out, err) = proc.communicate() + err = proc.communicate()[1] rv = proc.wait() if rv != 0: self.logger.error("Trigger: Error running %s (%s): %s" % (args[0], rv, err)) elif err: self.debug_log("Trigger: Error: %s" % err) - os._exit(0) - + os._exit(0) # pylint: disable=W0212 def end_client_run(self, metadata): args = [metadata.hostname, '-p', metadata.profile, '-g', -- cgit v1.2.3-1-g7c22