From 71c679e1a0105490bd5845a15de5e8f1a32e2166 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 11 Sep 2012 10:32:30 -0400 Subject: Cfg: documented all Cfg modules, added development docs --- src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py | 14 +- .../Server/Plugins/Cfg/CfgCheetahGenerator.py | 13 + src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py | 9 + .../Plugins/Cfg/CfgEncryptedCheetahGenerator.py | 12 +- .../Server/Plugins/Cfg/CfgEncryptedGenerator.py | 34 +- .../Plugins/Cfg/CfgEncryptedGenshiGenerator.py | 25 +- .../Plugins/Cfg/CfgExternalCommandVerifier.py | 8 + .../Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 42 ++- src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py | 12 +- src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py | 12 + .../Server/Plugins/Cfg/CfgPlaintextGenerator.py | 14 +- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 345 ++++++++++++++++++--- 12 files changed, 444 insertions(+), 96 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Cfg') diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py index c25cf85f1..a2e86b3db 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py @@ -1,11 +1,16 @@ -import logging -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugins.Cfg import CfgFilter +""" Handle .cat files, which append lines to and remove lines from +plaintext files """ -logger = logging.getLogger(__name__) +from Bcfg2.Server.Plugins.Cfg import CfgFilter class CfgCatFilter(CfgFilter): + """ CfgCatFilter appends lines to and remove lines from plaintext + :ref:`server-plugins-generators-Cfg` files""" + + #: Handle .cat files __extensions__ = ['cat'] + + #: .cat files are deprecated deprecated = True def modify_data(self, entry, metadata, data): @@ -19,3 +24,4 @@ class CfgCatFilter(CfgFilter): if line[1:] in datalines: datalines.remove(line[1:]) return "\n".join(datalines) + "\n" + modify_data.__doc__ = CfgFilter.modify_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index f02461673..a0e999847 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -1,3 +1,7 @@ +""" The CfgCheetahGenerator allows you to use the `Cheetah +`_ templating system to generate +:ref:`server-plugins-generators-cfg` files. """ + import copy import logging import Bcfg2.Server.Plugin @@ -13,7 +17,14 @@ except ImportError: class CfgCheetahGenerator(CfgGenerator): + """ The CfgCheetahGenerator allows you to use the `Cheetah + `_ templating system to generate + :ref:`server-plugins-generators-cfg` files. """ + + #: Handle .cheetah files __extensions__ = ['cheetah'] + + #: :class:`Cheetah.Template.Template` compiler settings settings = dict(useStackFrames=False) def __init__(self, fname, spec, encoding): @@ -22,6 +33,7 @@ class CfgCheetahGenerator(CfgGenerator): msg = "Cfg: Cheetah is not available: %s" % entry.get("name") logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): template = Template(self.data.decode(self.encoding), @@ -30,3 +42,4 @@ class CfgCheetahGenerator(CfgGenerator): template.path = entry.get('realname', entry.get('name')) template.source_path = self.name return template.respond() + get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py index 579fd4005..409d2cbf6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py @@ -1,3 +1,5 @@ +""" Handle .diff files, which apply diffs to plaintext files """ + import os import logging import tempfile @@ -8,7 +10,13 @@ from Bcfg2.Server.Plugins.Cfg import CfgFilter logger = logging.getLogger(__name__) class CfgDiffFilter(CfgFilter): + """ CfgDiffFilter applies diffs to plaintext + :ref:`server-plugins-generators-Cfg` files """ + + #: Handle .diff files __extensions__ = ['diff'] + + #: .diff files are deprecated deprecated = True def modify_data(self, entry, metadata, data): @@ -26,3 +34,4 @@ class CfgDiffFilter(CfgFilter): logger.error("Error applying diff %s: %s" % (delta.name, stderr)) raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta) return output + modify_data.__doc__ = CfgFilter.modify_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedCheetahGenerator.py index a75329d2a..3e714c01f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedCheetahGenerator.py @@ -1,14 +1,20 @@ -import logging +""" Handle encrypted Cheetah templates (.crypt.cheetah or +.cheetah.crypt files)""" + from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator import CfgEncryptedGenerator -logger = logging.getLogger(__name__) - class CfgEncryptedCheetahGenerator(CfgCheetahGenerator, CfgEncryptedGenerator): + """ CfgEncryptedCheetahGenerator lets you encrypt your Cheetah + :ref:`server-plugins-generators-cfg` files on the server """ + + #: handle .crypt.cheetah or .cheetah.crypt files __extensions__ = ['cheetah.crypt', 'crypt.cheetah'] def handle_event(self, event): CfgEncryptedGenerator.handle_event(self, event) + handle_event.__doc__ = CfgEncryptedGenerator.handle_event.__doc__ def get_data(self, entry, metadata): return CfgCheetahGenerator.get_data(self, entry, metadata) + get_data.__doc__ = CfgCheetahGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index 2c926fae7..71e407d17 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -1,35 +1,22 @@ +""" CfgEncryptedGenerator lets you encrypt your plaintext +:ref:`server-plugins-generators-cfg` files on the server. """ + import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP try: - from Bcfg2.Encryption import ssl_decrypt, EVPError + from Bcfg2.Encryption import bruteforce_decrypt, EVPError have_crypto = True except ImportError: have_crypto = False logger = logging.getLogger(__name__) -def passphrases(): - section = "encryption" - if SETUP.cfp.has_section(section): - return dict([(o, SETUP.cfp.get(section, o)) - for o in SETUP.cfp.options(section)]) - else: - return dict() - -def decrypt(crypted): - if not have_crypto: - msg = "Cfg: M2Crypto is not available: %s" % entry.get("name") - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - for passwd in passphrases().values(): - try: - return ssl_decrypt(crypted, passwd) - except EVPError: - pass - raise EVPError("Failed to decrypt") - class CfgEncryptedGenerator(CfgGenerator): + """ CfgEncryptedGenerator lets you encrypt your plaintext + :ref:`server-plugins-generators-cfg` files on the server. """ + + #: Handle .crypt files __extensions__ = ["crypt"] def __init__(self, fname, spec, encoding): @@ -38,6 +25,7 @@ class CfgEncryptedGenerator(CfgGenerator): msg = "Cfg: M2Crypto is not available: %s" % entry.get("name") logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + __init__.__doc__ = CfgGenerator.__init__.__doc__ def handle_event(self, event): if event.code2str() == 'deleted': @@ -51,13 +39,15 @@ class CfgEncryptedGenerator(CfgGenerator): return # todo: let the user specify a passphrase by name try: - self.data = decrypt(crypted) + self.data = bruteforce_decrypt(crypted, setup=SETUP) except EVPError: msg = "Failed to decrypt %s" % self.name logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + handle_event.__doc__ = CfgGenerator.handle_event.__doc__ def get_data(self, entry, metadata): if self.data is None: raise Bcfg2.Server.Plugin.PluginExecutionError("Failed to decrypt %s" % self.name) return CfgGenerator.get_data(self, entry, metadata) + get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 140d4a486..0d5d98ba6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -1,10 +1,15 @@ -import logging +""" Handle encrypted Genshi templates (.crypt.genshi or .genshi.crypt +files) """ + from Bcfg2.Compat import StringIO from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator -from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator import decrypt, \ - CfgEncryptedGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator import CfgEncryptedGenerator -logger = logging.getLogger(__name__) +try: + from Bcfg2.Encryption import bruteforce_decrypt +except ImportError: + # CfgGenshiGenerator will raise errors if crypto doesn't exist + pass try: from genshi.template import TemplateLoader @@ -14,13 +19,23 @@ except ImportError: class EncryptedTemplateLoader(TemplateLoader): + """ Subclass :class:`genshi.template.TemplateLoader` to decrypt + the data on the fly as it's read in using + :func:`Bcfg2.Encryption.bruteforce_decrypt` """ def _instantiate(self, cls, fileobj, filepath, filename, encoding=None): - plaintext = StringIO(decrypt(fileobj.read())) + plaintext = StringIO(bruteforce_decrypt(fileobj.read())) return TemplateLoader._instantiate(self, cls, plaintext, filepath, filename, encoding=encoding) class CfgEncryptedGenshiGenerator(CfgGenshiGenerator): + """ CfgEncryptedGenshiGenerator lets you encrypt your Genshi + :ref:`server-plugins-generators-cfg` files on the server """ + + #: handle .crypt.genshi or .genshi.crypt files __extensions__ = ['genshi.crypt', 'crypt.genshi'] + + #: Use a TemplateLoader class that decrypts the data on the fly + #: when it's read in __loader_cls__ = EncryptedTemplateLoader diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index f0c1109ec..87e11ab6d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -1,3 +1,5 @@ +""" Invoke an external command to verify file contents """ + import os import shlex import logging @@ -8,6 +10,10 @@ from Bcfg2.Server.Plugins.Cfg import CfgVerifier, CfgVerificationError logger = logging.getLogger(__name__) class CfgExternalCommandVerifier(CfgVerifier): + """ Invoke an external script to verify + :ref:`server-plugins-generators-cfg` file contents """ + + #: Handle :file:`:test` files __basenames__ = [':test'] def verify_entry(self, entry, metadata, data): @@ -16,6 +22,7 @@ class CfgExternalCommandVerifier(CfgVerifier): rv = proc.wait() if rv != 0: raise CfgVerificationError(err) + verify_entry.__doc__ = CfgVerifier.verify_entry.__doc__ def handle_event(self, event): if event.code2str() == 'deleted': @@ -30,4 +37,5 @@ class CfgExternalCommandVerifier(CfgVerifier): logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) self.cmd.append(self.name) + handle_event.__doc__ = CfgVerifier.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index 7e2f83962..dc128bbe9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -1,3 +1,7 @@ +""" The CfgGenshiGenerator allows you to use the `Genshi +`_ templating system to generate +:ref:`server-plugins-generators-cfg` files. """ + import re import sys import logging @@ -16,9 +20,15 @@ except ImportError: TemplateLoader = None have_genshi = False -# snipped from TGenshi def removecomment(stream): - """A genshi filter that removes comments from the stream.""" + """ A Genshi filter that removes comments from the stream. This + function is a generator. + + :param stream: The Genshi stream to remove comments from + :type stream: genshi.core.Stream + :returns: tuple of ``(kind, data, pos)``, as when iterating + through a Genshi stream + """ for kind, data, pos in stream: if kind is genshi.core.COMMENT: continue @@ -26,8 +36,27 @@ def removecomment(stream): class CfgGenshiGenerator(CfgGenerator): + """ The CfgGenshiGenerator allows you to use the `Genshi + `_ templating system to generate + :ref:`server-plugins-generators-cfg` files. """ + + #: Handle .genshi files __extensions__ = ['genshi'] + + #: ``__loader_cls__`` is the class that will be instantiated to + #: load the template files. It must implement one public function, + #: ``load()``, as :class:`genshi.template.TemplateLoader`. __loader_cls__ = TemplateLoader + + #: Ignore ``.genshi_include`` files so they can be used with the + #: Genshi ``{% include ... %}`` directive without raising warnings. + __ignore__ = ["genshi_include"] + + #: Error-handling in Genshi is pretty obtuse. This regex is used + #: to extract the first line of the code block that raised an + #: exception in a Genshi template so we can provide a decent error + #: message that actually tells the end user where an error + #: occurred. pyerror_re = re.compile('<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>') def __init__(self, fname, spec, encoding): @@ -38,11 +67,7 @@ class CfgGenshiGenerator(CfgGenerator): raise Bcfg2.Server.Plugin.PluginExecutionError(msg) self.loader = self.__loader_cls__() self.template = None - - @classmethod - def ignore(cls, event, basename=None): - return (event.filename.endswith(".genshi_include") or - CfgGenerator.ignore(event, basename=basename)) + __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): fname = entry.get('realname', entry.get('name')) @@ -109,6 +134,7 @@ class CfgGenshiGenerator(CfgGenerator): (fname, src[real_lineno], err.__class__.__name__, err)) raise + get_data.__doc__ = CfgGenerator.get_data.__doc__ def handle_event(self, event): if event.code2str() == 'deleted': @@ -122,4 +148,4 @@ class CfgGenshiGenerator(CfgGenerator): sys.exc_info()[1]) logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - + handle_event.__doc__ = CfgGenerator.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py index 956ebfe17..472a7dba3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py @@ -1,3 +1,5 @@ +""" Handle info.xml files """ + import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgInfo @@ -5,11 +7,16 @@ from Bcfg2.Server.Plugins.Cfg import CfgInfo logger = logging.getLogger(__name__) class CfgInfoXML(CfgInfo): + """ CfgInfoXML handles :file:`info.xml` files for + :ref:`server-plugins-generators-cfg` """ + + #: Handle :file:`info.xml` files __basenames__ = ['info.xml'] def __init__(self, path): CfgInfo.__init__(self, path) self.infoxml = Bcfg2.Server.Plugin.InfoXML(path) + __init__.__doc__ = CfgInfo.__init__.__doc__ def bind_info_to_entry(self, entry, metadata): mdata = dict() @@ -17,14 +24,17 @@ class CfgInfoXML(CfgInfo): if 'Info' not in mdata: logger.error("Failed to set metadata for file %s" % entry.get('name')) - raise PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError self._set_info(entry, mdata['Info'][None]) + bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ def handle_event(self, event): self.infoxml.HandleEvent() + handle_event.__doc__ = CfgInfo.handle_event.__doc__ def _set_info(self, entry, info): CfgInfo._set_info(self, entry, info) if '__children__' in info: for child in info['__children__']: entry.append(child) + _set_info.__doc__ = CfgInfo._set_info.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py index 3673cfcb2..a47663904 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py @@ -1,3 +1,5 @@ +""" Handle info and :info files """ + import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgInfo @@ -5,15 +7,24 @@ from Bcfg2.Server.Plugins.Cfg import CfgInfo logger = logging.getLogger(__name__) class CfgLegacyInfo(CfgInfo): + """ CfgLegacyInfo handles :file:`info` and :file:`:info` files for + :ref:`server-plugins-generators-cfg` """ + + #: Handle :file:`info` and :file:`:info` __basenames__ = ['info', ':info'] + + #: CfgLegacyInfo is deprecated. Use + #: :class:`Bcfg2.Server.Plugins.Cfg.CfgInfoXML.CfgInfoXML` instead. deprecated = True def __init__(self, path): CfgInfo.__init__(self, path) self.path = path + __init__.__doc__ = CfgInfo.__init__.__doc__ def bind_info_to_entry(self, entry, metadata): self._set_info(entry, self.metadata) + bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ def handle_event(self, event): if event.code2str() == 'deleted': @@ -31,3 +42,4 @@ class CfgLegacyInfo(CfgInfo): if ('perms' in self.metadata and len(self.metadata['perms']) == 3): self.metadata['perms'] = "0%s" % self.metadata['perms'] + handle_event.__doc__ = CfgInfo.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py index 8e9aab465..333e2f670 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py @@ -1,8 +1,14 @@ -import logging -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugins.Cfg import CfgGenerator +""" CfgPlaintextGenerator is a +:class:`Bcfg2.Server.Plugins.Cfg.CfgGenerator` that handles plain text +(i.e., non-templated) :ref:`server-plugins-generators-cfg` files.""" -logger = logging.getLogger(__name__) +from Bcfg2.Server.Plugins.Cfg import CfgGenerator class CfgPlaintextGenerator(CfgGenerator): + """ CfgPlaintextGenerator is a + :class:`Bcfg2.Server.Plugins.Cfg.CfgGenerator` that handles plain + text (i.e., non-templated) :ref:`server-plugins-generators-cfg` + files. The base Generator class already implements this + functionality, so CfgPlaintextGenerator doesn't need to do + anything itself.""" pass diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index d9a8b90db..e2832cd26 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -13,29 +13,85 @@ import Bcfg2.Server.Lint logger = logging.getLogger(__name__) -PROCESSORS = None +#: SETUP contains a reference to the +#: :class:`Bcfg2.Options.OptionParser` created by the Bcfg2 core for +#: parsing command-line and config file options. +#: :class:`Bcfg2.Server.Plugins.Cfg.Cfg` stores it in a module global +#: so that the handler objects can access it, because there is no other +#: facility for passing a setup object from a +#: :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` to its +#: :class:`Bcfg2.Server.Plugin.helpers.EntrySet` objects and thence to +#: the EntrySet children. SETUP = None class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): + """ CfgBaseFileMatcher is the parent class for all Cfg handler + objects. """ + + #: The set of filenames handled by this handler. If + #: ``__basenames__`` is the empty list, then the basename of each + #: :class:`Bcfg2.Server.Plugins.Cfg.CfgEntrySet` is used -- i.e., + #: the name of the directory that contains the file is used for + #: the basename. __basenames__ = [] + + #: This handler only handles files with the listed extensions + #: (which come *after* + #: :attr:`Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__specific__` + #: indicators). __extensions__ = [] + + #: This handler ignores files with the listed extensions. A file + #: that is ignored by a handler will not be handled by any other + #: handlers; that is, a file is ignored if any handler ignores it. + #: Ignoring a file is not simply a means to defer handling of that + #: file to another handler. __ignore__ = [] + + #: Whether or not the files handled by this handler are permitted + #: to have specificity indicators in their filenames -- e.g., + #: ``.H_client.example.com`` or ``.G10_foogroup``. __specific__ = True + + #: Flag to indicate a deprecated handler. deprecated = False - def __init__(self, fname, spec, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, fname, spec, encoding) + def __init__(self, name, specific, encoding): + Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, + encoding) self.encoding = encoding - self.regex = self.__class__.get_regex(fname) + self.regex = self.__class__.get_regex(name) + __init__.__doc__ = Bcfg2.Server.Plugin.SpecificData.__init__.__doc__ + \ +""" +.. ----- +.. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__basenames__ +.. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__extensions__ +.. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__ignore__ +.. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__specific__""" @classmethod - def get_regex(cls, fname=None, extensions=None): + def get_regex(cls, basename=None, extensions=None): + """ Get a compiled regular expression to match filenames (not + full paths) that this handler handles. + + :param basename: The base filename to use if + :attr:`Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__basenames__` + is not defined (i.e., the name of the + directory that contains the files the regex + will be applied to) + :type basename: string + :param extensions: Override the default list of + :attr:`Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__extensions__` + to include in the regex. + :type extensions: list of strings + :returns: compiled regex + """ if extensions is None: extensions = cls.__extensions__ if cls.__basenames__: - fname = '|'.join(cls.__basenames__) + basename = '|'.join(cls.__basenames__) - components = ['^(?P%s)' % fname] + components = ['^(?P%s)' % basename] if cls.__specific__: components.append('(|\\.H_(?P\S+?)|.G(?P\d+)_(?P\S+?))') if extensions: @@ -45,6 +101,22 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): @classmethod def handles(cls, event, basename=None): + """ Return True if this handler handles the file described by + ``event``. This is faster than just applying + :func:`Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.get_regex` + because it tries to do non-regex matching first. + + :param event: The FAM event to check + :type event: Bcfg2.Server.FileMonitor.Event + :param basename: The base filename to use if + :attr:`Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__basenames__` + is not defined (i.e., the name of the + directory that contains the files the regex + will be applied to) + :type basename: string + :returns: bool - True if this handler handles the file listed + in the event, False otherwise. + """ if cls.__basenames__: basenames = cls.__basenames__ else: @@ -57,10 +129,26 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): match = True break return (match and - cls.get_regex(fname=os.path.basename(basename)).match(event.filename)) + cls.get_regex(basename=os.path.basename(basename)).match(event.filename)) @classmethod def ignore(cls, event, basename=None): + """ Return True if this handler ignores the file described by + ``event``. See + :attr:`Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__ignore__` + for more information on how ignoring files works. + + :param event: The FAM event to check + :type event: Bcfg2.Server.FileMonitor.Event + :param basename: The base filename to use if + :attr:`Bcfg2.Server.Plugins.Cfg.CfgBaseFileMatcher.__basenames__` + is not defined (i.e., the name of the + directory that contains the files the regex + will be applied to) + :type basename: string + :returns: bool - True if this handler handles the file listed + in the event, False otherwise. + """ if not cls.__ignore__: return False @@ -76,10 +164,9 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): match = True break return (match and - cls.get_regex(fname=os.path.basename(basename), + cls.get_regex(basename=os.path.basename(basename), extensions=cls.__ignore__).match(event.filename)) - def __str__(self): return "%s(%s)" % (self.__class__.__name__, self.name) @@ -88,53 +175,170 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): class CfgGenerator(CfgBaseFileMatcher): - """ CfgGenerators generate the initial content of a file """ + """ CfgGenerators generate the initial content of a file. Every + valid :class:`Bcfg2.Server.Plugins.Cfg.CfgEntrySet` must have at + least one file handled by a CfgGenerator. Moreover, each + CfgEntrySet must have one unambiguously best handler for each + client. See :class:`Bcfg2.Server.Plugin.helpers.EntrySet` for more + details on how the best handler is chosen.""" + + def __init__(self, name, specific, encoding): + # we define an __init__ that just calls the parent __init__, + # so that we can set the docstring on __init__ to something + # different from the parent __init__ -- namely, the parent + # __init__ docstring, minus everything after ``.. -----``, + # which we use to delineate the actual docs from the + # .. autoattribute hacks we have to do to get private + # attributes included in sphinx 1.0 """ + CfgBaseFileMatcher.__init__(self, name, specific, encoding) + __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] + def get_data(self, entry, metadata): + """ get_data() returns the initial data of a file. + + :param entry: The entry to generate data for. ``entry`` should + not be modified in-place. + :type entry: lxml.etree._Element + :param metadata: The client metadata to generate data for. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: string - the contents of the entry + :raises: any + """ return self.data class CfgFilter(CfgBaseFileMatcher): - """ CfgFilters modify the initial content of a file after it's - been generated """ + """ CfgFilters modify the initial content of a file after it has + been generated by a :class:`Bcfg2.Server.Plugins.Cfg.CfgGenerator`. """ + + def __init__(self, name, specific, encoding): + # see comment on CfgGenerator.__init__ above + CfgBaseFileMatcher.__init__(self, name, specific, encoding) + __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] + def modify_data(self, entry, metadata, data): + """ Return new data for the entry, based on the initial data + produced by the :class:`Bcfg2.Server.Plugins.Cfg.CfgGenerator`. + + :param entry: The entry to filter data for. ``entry`` should + not be modified in-place. + :type entry: lxml.etree._Element + :param metadata: The client metadata to filter data for. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param data: The initial contents of the entry produced by the + CfgGenerator + :type data: string + :returns: string - the new contents of the entry + """ raise NotImplementedError class CfgInfo(CfgBaseFileMatcher): - """ CfgInfos provide metadata (owner, group, paranoid, etc.) for a - file entry """ + """ CfgInfo handlers provide metadata (owner, group, paranoid, + etc.) for a file entry. + + .. private-include: _set_info + """ + + #: Whether or not the files handled by this handler are permitted + #: to have specificity indicators in their filenames -- e.g., + #: ``.H_client.example.com`` or ``.G10_foogroup``. By default + #: CfgInfo handlers do not allow specificities __specific__ = False def __init__(self, fname): + """ + :param name: The full path to the file + :type name: string + + .. ----- + .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgInfo.__specific__ + """ CfgBaseFileMatcher.__init__(self, fname, None, None) def bind_info_to_entry(self, entry, metadata): + """ Assign the appropriate attributes to the entry, modifying + it in place. + + :param entry: The abstract entry to bind the info to + :type entry: lxml.etree._Element + :param metadata: The client metadata to get info for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: None + """ raise NotImplementedError def _set_info(self, entry, info): + """ Helper function to assign a dict of info attributes to an + entry object. ``entry`` is modified in-place. + + :param entry: The abstract entry to bind the info to + :type entry: lxml.etree._Element + :param info: A dict of attribute: value pairs + :type info: dict + :returns: None + """ for key, value in list(info.items()): if not key.startswith("__"): entry.attrib.__setitem__(key, value) class CfgVerifier(CfgBaseFileMatcher): - """ Verifiers validate entries """ + """ CfgVerifier handlers validate entry data once it has been + generated, filtered, and info applied. Validation can be enabled + or disabled in the configuration. Validation can apply to the + contents of an entry, the attributes on it (e.g., owner, group, + etc.), or both. + """ + + def __init__(self, name, specific, encoding): + # see comment on CfgGenerator.__init__ above + CfgBaseFileMatcher.__init__(self, name, specific, encoding) + __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] + def verify_entry(self, entry, metadata, data): + """ Perform entry contents. validation. + + :param entry: The entry to validate data for. ``entry`` should + not be modified in-place. Info attributes have + been bound to the entry, but the text data has + not been set. + :type entry: lxml.etree._Element + :param metadata: The client metadata to validate data for. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param data: The contents of the entry + :type data: string + :returns: None + :raises: Bcfg2.Server.Plugins.Cfg.CfgVerificationError + """ raise NotImplementedError class CfgVerificationError(Exception): + """ Raised by + :func:`Bcfg2.Server.Plugins.Cfg.CfgVerifier.verify_entry` when an + entry fails verification """ pass class CfgDefaultInfo(CfgInfo): + """ :class:`Bcfg2.Server.Plugins.Cfg.Cfg` handler that supplies a + default set of file metadata """ + def __init__(self, defaults): CfgInfo.__init__(self, '') self.defaults = defaults + __init__.__doc__ = CfgInfo.__init__.__doc__.split(".. -----")[0] def bind_info_to_entry(self, entry, metadata): self._set_info(entry, self.defaults) + bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ +#: A :class:`CfgDefaultInfo` object instantiated with +#: :attr:`Bcfg2.Server.Plugin.helper.default_file_metadata` as its +#: default metadata. This is used to set a default file metadata set +#: on an entry before a "real" :class:`CfgInfo` handler applies its +#: metadata to the entry. DEFAULT_INFO = CfgDefaultInfo(Bcfg2.Server.Plugin.default_file_metadata) class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): @@ -142,28 +346,36 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) self.specific = None - self.load_processors() - - def load_processors(self): - """ load Cfg file processors. this must be done at run-time, - not at compile-time, or we get a circular import and things - don't work. but finding the right way to do this at runtime - was ... problematic. so here it is, writing to a global - variable. Sorry 'bout that. """ - global PROCESSORS - if PROCESSORS is None: - PROCESSORS = [] + self._handlers = None + __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ + + @property + def handlers(self): + """ A list of Cfg handler classes. Loading the handlers must + be done at run-time, not at compile-time, or it causes a + circular import and Bad Things Happen.""" + if self._handlers is None: + self._handlers = [] for submodule in walk_packages(path=__path__, prefix=__name__ + "."): mname = submodule[1].rsplit('.', 1)[-1] module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, mname) - proc = getattr(module, mname) - if set(proc.__mro__).intersection([CfgInfo, CfgFilter, + hdlr = getattr(module, mname) + if set(hdlr.__mro__).intersection([CfgInfo, CfgFilter, CfgGenerator, CfgVerifier]): - PROCESSORS.append(proc) + self._handlers.append(hdlr) + return self._handlers def handle_event(self, event): + """ Dispatch a FAM event to :func:`entry_init` or the + appropriate child handler object. + + :param event: An event that applies to a file handled by this + CfgEntrySet + :type event: Bcfg2.Server.FileMonitor.Event + :returns: None + """ action = event.code2str() if event.filename not in self.entries: @@ -171,18 +383,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): # process a bogus changed event like a created return - for proc in PROCESSORS: - if proc.handles(event, basename=self.path): + for hdlr in self.handlers: + if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but # handle it like a 'created' logger.warning("Got %s event for unknown file %s" % (action, event.filename)) self.debug_log("%s handling %s event on %s" % - (proc.__name__, action, event.filename)) - self.entry_init(event, proc) + (hdlr.__name__, action, event.filename)) + self.entry_init(event, hdlr) return - elif proc.ignore(event, basename=self.path): + elif hdlr.ignore(event, basename=self.path): return elif action == 'changed': self.entries[event.filename].handle_event(event) @@ -198,18 +410,32 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): return [item for item in list(self.entries.values()) if (isinstance(item, CfgGenerator) and item.specific.matches(metadata))] - - def entry_init(self, event, proc): - if proc.__specific__: + get_matching.__doc__ = Bcfg2.Server.Plugin.EntrySet.get_matching.__doc__ + + def entry_init(self, event, hdlr): + """ Handle the creation of a file on the filesystem and the + creation of a Cfg handler object in this CfgEntrySet to track + it. + + :param event: An event that applies to a file handled by this + CfgEntrySet + :type event: Bcfg2.Server.FileMonitor.Event + :param hdlr: The Cfg handler class to be used to create an + object for the file described by ``event`` + :type hdlr: class + :returns: None + :raises: :class:`Bcfg2.Server.Plugin.exceptions.SpecificityError` + """ + if hdlr.__specific__: Bcfg2.Server.Plugin.EntrySet.entry_init( - self, event, entry_type=proc, - specific=proc.get_regex(os.path.basename(self.path))) + self, event, entry_type=hdlr, + specific=hdlr.get_regex(os.path.basename(self.path))) else: if event.filename in self.entries: logger.warn("Got duplicate add for %s" % event.filename) else: fpath = os.path.join(self.path, event.filename) - self.entries[event.filename] = proc(fpath) + self.entries[event.filename] = hdlr(fpath) self.entries[event.filename].handle_event(event) def bind_entry(self, entry, metadata): @@ -222,11 +448,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): continue if isinstance(ent, CfgInfo): info_handlers.append(ent) - elif isinstance(ent, CfgGenerator): + if isinstance(ent, CfgGenerator): generators.append(ent) - elif isinstance(ent, CfgFilter): + if isinstance(ent, CfgFilter): filters.append(ent) - elif isinstance(ent, CfgVerifier): + if isinstance(ent, CfgVerifier): verifiers.append(ent) if ent.deprecated: if ent.__basenames__: @@ -311,6 +537,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): entry.text = data else: entry.set('empty', 'true') + return entry + bind_entry.__doc__ = Bcfg2.Server.Plugin.EntrySet.bind_entry.__doc__ def list_accept_choices(self, entry, metadata): '''return a list of candidate pull locations''' @@ -391,7 +619,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): class Cfg(Bcfg2.Server.Plugin.GroupSpool, Bcfg2.Server.Plugin.PullTarget): - """This generator in the configuration file repository for Bcfg2.""" + """ The Cfg plugin provides a repository to describe configuration + file contents for clients. In its simplest form, the Cfg repository is + just a directory tree modeled off of the directory tree on your client + machines. + """ __author__ = 'bcfg-dev@mcs.anl.gov' es_cls = CfgEntrySet es_child_cls = Bcfg2.Server.Plugin.SpecificData @@ -405,10 +637,21 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, if 'validate' not in SETUP: SETUP.add_option('validate', Bcfg2.Options.CFG_VALIDATION) SETUP.reparse() + __init__.__doc__ = Bcfg2.Server.Plugin.GroupSpool.__init__.__doc__ def has_generator(self, entry, metadata): - """ return True if the given entry can be generated for the - given metadata; False otherwise """ + """ Return True if the given entry can be generated for the + given metadata; False otherwise + + :param entry: Determine if a + :class:`Bcfg2.Server.Plugins.Cfg.CfgGenerator` + object exists that handles this (abstract) entry + :type entry: lxml.etree._Element + :param metadata: Determine if a CfgGenerator has data that + applies to this client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: bool + """ if entry.get('name') not in self.entries: return False @@ -422,11 +665,15 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, def AcceptChoices(self, entry, metadata): return self.entries[entry.get('name')].list_accept_choices(entry, metadata) + AcceptChoices.__doc__ = Bcfg2.Server.Plugin.PullTarget.AcceptChoices.__doc__ def AcceptPullData(self, specific, new_entry, log): return self.entries[new_entry.get('name')].write_update(specific, new_entry, log) + AcceptPullData.__doc__ = \ + Bcfg2.Server.Plugin.PullTarget.AcceptPullData.__doc__ + class CfgLint(Bcfg2.Server.Lint.ServerPlugin): """ warn about usage of .cat and .diff files """ @@ -444,8 +691,8 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): def check_entry(self, basename, entry): cfg = self.core.plugins['Cfg'] for basename, entry in list(cfg.entries.items()): - for fname, processor in entry.entries.items(): - if self.HandlesFile(fname) and isinstance(processor, CfgFilter): + for fname, handler in entry.entries.items(): + if self.HandlesFile(fname) and isinstance(handler, CfgFilter): extension = fname.split(".")[-1] self.LintError("%s-file-used" % extension, "%s file used on %s: %s" % -- cgit v1.2.3-1-g7c22