From ebe7542db7217c2fac3d7111e80f94caedfb69e2 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 16 Jan 2013 13:28:06 -0500 Subject: added module-level OptionParser to avoid passing it as an argument or global all over --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Bundler.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index b200346bc..6dc3c2b1d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -1,15 +1,15 @@ """This provides bundle clauses with translation functionality.""" -import copy -import logging -import lxml.etree import os -import os.path import re import sys +import copy +import logging +import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin import Bcfg2.Server.Lint +from Bcfg2.Options import get_option_parser try: import genshi.template.base @@ -19,9 +19,6 @@ except ImportError: HAS_GENSHI = False -SETUP = None - - class BundleFile(Bcfg2.Server.Plugin.StructFile): """ Representation of a bundle XML file """ def get_xml_value(self, metadata): @@ -52,8 +49,9 @@ if HAS_GENSHI: msg = "No parsed template information for %s" % self.name self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - stream = self.template.generate(metadata=metadata, - repo=SETUP['repo']).filter( + stream = self.template.generate( + metadata=metadata, + repo=get_option_parser()['repo']).filter( Bcfg2.Server.Plugins.TGenshi.removecomment) data = lxml.etree.XML(stream.render('xml', strip_whitespace=False), @@ -102,9 +100,6 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) - global SETUP - SETUP = core.setup - def template_dispatch(self, name, _): """ Add the correct child entry type to Bundler depending on whether the XML file in question is a plain XML file or a -- cgit v1.2.3-1-g7c22 From 0fc1f472a0fb18911bde1cb99f03142681804476 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 30 Oct 2012 10:22:02 -0400 Subject: removed deprecated plugins: TGenshi, TCheetah, Account, Hostbase, Snapshots, Statistics, Editor --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 66 ++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 18 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Bundler.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 6dc3c2b1d..ae94a9b31 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -12,13 +12,30 @@ import Bcfg2.Server.Lint from Bcfg2.Options import get_option_parser try: - import genshi.template.base - import Bcfg2.Server.Plugins.TGenshi + import genshi.core + import genshi.input + from genshi.template import TemplateLoader, \ + TextTemplate, MarkupTemplate, TemplateError HAS_GENSHI = True except ImportError: HAS_GENSHI = False +def removecomment(stream): + """ A Genshi filter that removes comments from the stream. This + function is a generator. + + :param stream: The Genshi stream to remove comments from + :type stream: genshi.core.Stream + :returns: tuple of ``(kind, data, pos)``, as when iterating + through a Genshi stream + """ + for kind, data, pos in stream: + if kind is genshi.core.COMMENT: + continue + yield kind, data, pos + + class BundleFile(Bcfg2.Server.Plugin.StructFile): """ Representation of a bundle XML file """ def get_xml_value(self, metadata): @@ -31,17 +48,37 @@ class BundleFile(Bcfg2.Server.Plugin.StructFile): if HAS_GENSHI: - class BundleTemplateFile(Bcfg2.Server.Plugins.TGenshi.TemplateFile, - Bcfg2.Server.Plugin.StructFile): + class BundleTemplateFile(Bcfg2.Server.Plugin.StructFile): """ Representation of a Genshi-templated bundle XML file """ - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugins.TGenshi.TemplateFile.__init__(self, name, - specific, - encoding) + def __init__(self, name, encoding): Bcfg2.Server.Plugin.StructFile.__init__(self, name) + self.encoding = encoding self.logger = logging.getLogger(name) + def HandleEvent(self, event=None): + """Handle all fs events for this template.""" + if event and event.code2str() == 'deleted': + return + try: + loader = TemplateLoader() + try: + self.template = loader.load(self.name, + cls=MarkupTemplate, + encoding=self.encoding) + except LookupError: + err = sys.exc_info()[1] + self.logger.error('Genshi lookup error in %s: %s' % + (self.name, err)) + except TemplateError: + err = sys.exc_info()[1] + self.logger.error('Genshi template error in %s: %s' % + (self.name, err)) + except genshi.input.ParseError: + err = sys.exc_info()[1] + self.logger.error('Genshi parse error in %s: %s' % + (self.name, err)) + def get_xml_value(self, metadata): """ get the rendered XML data that applies to the given client """ @@ -51,8 +88,7 @@ if HAS_GENSHI: raise Bcfg2.Server.Plugin.PluginExecutionError(msg) stream = self.template.generate( metadata=metadata, - repo=get_option_parser()['repo']).filter( - Bcfg2.Server.Plugins.TGenshi.removecomment) + repo=get_option_parser()['repo']).filter(removecomment) data = lxml.etree.XML(stream.render('xml', strip_whitespace=False), parser=Bcfg2.Server.XMLParser) @@ -71,11 +107,6 @@ if HAS_GENSHI: len(rv))) return rv - class SGenshiTemplateFile(BundleTemplateFile): - """ provided for backwards compat with the deprecated SGenshi - plugin """ - pass - class Bundler(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Structure, @@ -111,8 +142,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, ('py' in nsmap and nsmap['py'] == 'http://genshi.edgewall.org/')): if HAS_GENSHI: - spec = Bcfg2.Server.Plugin.Specificity() - return BundleTemplateFile(name, spec, self.encoding) + return BundleTemplateFile(name, self.encoding) else: raise Bcfg2.Server.Plugin.PluginExecutionError("Genshi not " "available: %s" @@ -139,7 +169,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, continue try: bundleset.append(entries[0].get_xml_value(metadata)) - except genshi.template.base.TemplateError: + except TemplateError: err = sys.exc_info()[1] self.logger.error("Bundler: Failed to render templated bundle " "%s: %s" % (bundlename, err)) -- cgit v1.2.3-1-g7c22 From 45c7bbf24ae3c6530f33ebb33c062818ad44816d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 30 Oct 2012 11:35:46 -0400 Subject: added a module-level FAM object to avoid passing it as an argument a billion times --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Bundler.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index ae94a9b31..9c6c7946d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -122,9 +122,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, self.encoding = core.setup['encoding'] self.__child__ = self.template_dispatch try: - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, - self.data, - self.core.fam) + Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data) except OSError: err = sys.exc_info()[1] msg = "Failed to load Bundle repository %s: %s" % (self.data, err) @@ -148,7 +146,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, "available: %s" % name) else: - return BundleFile(name, self.fam) + return BundleFile(name) def BuildStructures(self, metadata): """Build all structures for client (metadata).""" -- cgit v1.2.3-1-g7c22 From cebd0d7ad54995c37f68586a14540ad64d99d762 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 31 Oct 2012 11:46:51 -0400 Subject: fixed unit tests --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 34 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 18 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Bundler.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 9c6c7946d..fa993cd85 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -14,8 +14,7 @@ from Bcfg2.Options import get_option_parser try: import genshi.core import genshi.input - from genshi.template import TemplateLoader, \ - TextTemplate, MarkupTemplate, TemplateError + from genshi.template import TemplateLoader, MarkupTemplate, TemplateError HAS_GENSHI = True except ImportError: HAS_GENSHI = False @@ -55,6 +54,7 @@ if HAS_GENSHI: Bcfg2.Server.Plugin.StructFile.__init__(self, name) self.encoding = encoding self.logger = logging.getLogger(name) + self.template = None def HandleEvent(self, event=None): """Handle all fs events for this template.""" @@ -62,22 +62,20 @@ if HAS_GENSHI: return try: loader = TemplateLoader() - try: - self.template = loader.load(self.name, - cls=MarkupTemplate, - encoding=self.encoding) - except LookupError: - err = sys.exc_info()[1] - self.logger.error('Genshi lookup error in %s: %s' % - (self.name, err)) - except TemplateError: - err = sys.exc_info()[1] - self.logger.error('Genshi template error in %s: %s' % - (self.name, err)) - except genshi.input.ParseError: - err = sys.exc_info()[1] - self.logger.error('Genshi parse error in %s: %s' % - (self.name, err)) + self.template = loader.load(self.name, cls=MarkupTemplate, + encoding=self.encoding) + except LookupError: + err = sys.exc_info()[1] + self.logger.error('Genshi lookup error in %s: %s' % + (self.name, err)) + except TemplateError: + err = sys.exc_info()[1] + self.logger.error('Genshi template error in %s: %s' % + (self.name, err)) + except genshi.input.ParseError: + err = sys.exc_info()[1] + self.logger.error('Genshi parse error in %s: %s' % + (self.name, err)) def get_xml_value(self, metadata): """ get the rendered XML data that applies to the given -- cgit v1.2.3-1-g7c22 From 22029e107420ff21cf9f1811bf4bb6dc2aba1dde Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 22 Jan 2013 11:16:19 -0500 Subject: made genshi a requirement --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 149 +++++++++++++------------------- 1 file changed, 61 insertions(+), 88 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Bundler.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index fa993cd85..3907794ae 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -11,28 +11,8 @@ import Bcfg2.Server.Plugin import Bcfg2.Server.Lint from Bcfg2.Options import get_option_parser -try: - import genshi.core - import genshi.input - from genshi.template import TemplateLoader, MarkupTemplate, TemplateError - HAS_GENSHI = True -except ImportError: - HAS_GENSHI = False - - -def removecomment(stream): - """ A Genshi filter that removes comments from the stream. This - function is a generator. - - :param stream: The Genshi stream to remove comments from - :type stream: genshi.core.Stream - :returns: tuple of ``(kind, data, pos)``, as when iterating - through a Genshi stream - """ - for kind, data, pos in stream: - if kind is genshi.core.COMMENT: - continue - yield kind, data, pos +import genshi.input +from genshi.template import TemplateLoader, MarkupTemplate, TemplateError class BundleFile(Bcfg2.Server.Plugin.StructFile): @@ -46,64 +26,64 @@ class BundleFile(Bcfg2.Server.Plugin.StructFile): return bundle -if HAS_GENSHI: - class BundleTemplateFile(Bcfg2.Server.Plugin.StructFile): - """ Representation of a Genshi-templated bundle XML file """ +class BundleTemplateFile(Bcfg2.Server.Plugin.StructFile): + """ Representation of a Genshi-templated bundle XML file """ - def __init__(self, name, encoding): - Bcfg2.Server.Plugin.StructFile.__init__(self, name) - self.encoding = encoding - self.logger = logging.getLogger(name) - self.template = None + def __init__(self, name, encoding): + Bcfg2.Server.Plugin.StructFile.__init__(self, name) + self.encoding = encoding + self.logger = logging.getLogger(name) + self.template = None - def HandleEvent(self, event=None): - """Handle all fs events for this template.""" - if event and event.code2str() == 'deleted': - return - try: - loader = TemplateLoader() - self.template = loader.load(self.name, cls=MarkupTemplate, - encoding=self.encoding) - except LookupError: - err = sys.exc_info()[1] - self.logger.error('Genshi lookup error in %s: %s' % - (self.name, err)) - except TemplateError: - err = sys.exc_info()[1] - self.logger.error('Genshi template error in %s: %s' % - (self.name, err)) - except genshi.input.ParseError: - err = sys.exc_info()[1] - self.logger.error('Genshi parse error in %s: %s' % - (self.name, err)) - - def get_xml_value(self, metadata): - """ get the rendered XML data that applies to the given - client """ - if not hasattr(self, 'template'): - msg = "No parsed template information for %s" % self.name - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - stream = self.template.generate( - metadata=metadata, - repo=get_option_parser()['repo']).filter(removecomment) - data = lxml.etree.XML(stream.render('xml', - strip_whitespace=False), - parser=Bcfg2.Server.XMLParser) - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - for item in self.Match(metadata, data): - bundle.append(copy.deepcopy(item)) - return bundle - - def Match(self, metadata, xdata): # pylint: disable=W0221 - """Return matching fragments of parsed template.""" - rv = [] - for child in xdata.getchildren(): - rv.extend(self._match(child, metadata)) - self.logger.debug("File %s got %d match(es)" % (self.name, - len(rv))) - return rv + def HandleEvent(self, event=None): + """Handle all fs events for this template.""" + if event and event.code2str() == 'deleted': + return + try: + loader = TemplateLoader() + self.template = loader.load(self.name, cls=MarkupTemplate, + encoding=self.encoding) + except LookupError: + err = sys.exc_info()[1] + self.logger.error('Genshi lookup error in %s: %s' % + (self.name, err)) + except TemplateError: + err = sys.exc_info()[1] + self.logger.error('Genshi template error in %s: %s' % + (self.name, err)) + except genshi.input.ParseError: + err = sys.exc_info()[1] + self.logger.error('Genshi parse error in %s: %s' % + (self.name, err)) + + def get_xml_value(self, metadata): + """ get the rendered XML data that applies to the given + client """ + if not hasattr(self, 'template'): + msg = "No parsed template information for %s" % self.name + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + stream = self.template.generate( + metadata=metadata, + repo=get_option_parser()['repo'] + ).filter(Bcfg2.Server.Plugin.removecomment) + data = lxml.etree.XML(stream.render('xml', + strip_whitespace=False), + parser=Bcfg2.Server.XMLParser) + bundlename = os.path.splitext(os.path.basename(self.name))[0] + bundle = lxml.etree.Element('Bundle', name=bundlename) + for item in self.Match(metadata, data): + bundle.append(copy.deepcopy(item)) + return bundle + + def Match(self, metadata, xdata): # pylint: disable=W0221 + """Return matching fragments of parsed template.""" + rv = [] + for child in xdata.getchildren(): + rv.extend(self._match(child, metadata)) + self.logger.debug("File %s got %d match(es)" % (self.name, + len(rv))) + return rv class Bundler(Bcfg2.Server.Plugin.Plugin, @@ -135,14 +115,8 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, parser=Bcfg2.Server.XMLParser) nsmap = bundle.getroot().nsmap if (name.endswith('.genshi') or - ('py' in nsmap and - nsmap['py'] == 'http://genshi.edgewall.org/')): - if HAS_GENSHI: - return BundleTemplateFile(name, self.encoding) - else: - raise Bcfg2.Server.Plugin.PluginExecutionError("Genshi not " - "available: %s" - % name) + ('py' in nsmap and nsmap['py'] == 'http://genshi.edgewall.org/')): + return BundleTemplateFile(name, self.encoding) else: return BundleFile(name) @@ -183,8 +157,7 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): self.missing_bundles() for bundle in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(bundle.name) and - (not HAS_GENSHI or - not isinstance(bundle, BundleTemplateFile))): + not isinstance(bundle, BundleTemplateFile)): self.bundle_names(bundle) @classmethod -- cgit v1.2.3-1-g7c22 From cfa769abf8e464acee43e1cac359b0f7e5fc5e4b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 23 Jan 2013 14:13:51 -0500 Subject: added genshi support to StructFile --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 96 ++------------------------------- 1 file changed, 5 insertions(+), 91 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Bundler.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 3907794ae..051443e22 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -4,101 +4,35 @@ import os import re import sys import copy -import logging import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin import Bcfg2.Server.Lint -from Bcfg2.Options import get_option_parser - -import genshi.input -from genshi.template import TemplateLoader, MarkupTemplate, TemplateError +from genshi.template import TemplateError class BundleFile(Bcfg2.Server.Plugin.StructFile): """ Representation of a bundle XML file """ def get_xml_value(self, metadata): """ get the XML data that applies to the given client """ - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) + bundle = lxml.etree.Element('Bundle', name=self.xdata.get("name")) for item in self.Match(metadata): bundle.append(copy.copy(item)) return bundle -class BundleTemplateFile(Bcfg2.Server.Plugin.StructFile): - """ Representation of a Genshi-templated bundle XML file """ - - def __init__(self, name, encoding): - Bcfg2.Server.Plugin.StructFile.__init__(self, name) - self.encoding = encoding - self.logger = logging.getLogger(name) - self.template = None - - def HandleEvent(self, event=None): - """Handle all fs events for this template.""" - if event and event.code2str() == 'deleted': - return - try: - loader = TemplateLoader() - self.template = loader.load(self.name, cls=MarkupTemplate, - encoding=self.encoding) - except LookupError: - err = sys.exc_info()[1] - self.logger.error('Genshi lookup error in %s: %s' % - (self.name, err)) - except TemplateError: - err = sys.exc_info()[1] - self.logger.error('Genshi template error in %s: %s' % - (self.name, err)) - except genshi.input.ParseError: - err = sys.exc_info()[1] - self.logger.error('Genshi parse error in %s: %s' % - (self.name, err)) - - def get_xml_value(self, metadata): - """ get the rendered XML data that applies to the given - client """ - if not hasattr(self, 'template'): - msg = "No parsed template information for %s" % self.name - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - stream = self.template.generate( - metadata=metadata, - repo=get_option_parser()['repo'] - ).filter(Bcfg2.Server.Plugin.removecomment) - data = lxml.etree.XML(stream.render('xml', - strip_whitespace=False), - parser=Bcfg2.Server.XMLParser) - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - for item in self.Match(metadata, data): - bundle.append(copy.deepcopy(item)) - return bundle - - def Match(self, metadata, xdata): # pylint: disable=W0221 - """Return matching fragments of parsed template.""" - rv = [] - for child in xdata.getchildren(): - rv.extend(self._match(child, metadata)) - self.logger.debug("File %s got %d match(es)" % (self.name, - len(rv))) - return rv - - class Bundler(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Structure, Bcfg2.Server.Plugin.XMLDirectoryBacked): """ The bundler creates dependent clauses based on the bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' + __child__ = BundleFile patterns = re.compile('^(?P.*)\.(xml|genshi)$') def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Structure.__init__(self) - self.encoding = core.setup['encoding'] - self.__child__ = self.template_dispatch try: Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data) except OSError: @@ -107,19 +41,6 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) - def template_dispatch(self, name, _): - """ Add the correct child entry type to Bundler depending on - whether the XML file in question is a plain XML file or a - templated bundle """ - bundle = lxml.etree.parse(name, - parser=Bcfg2.Server.XMLParser) - nsmap = bundle.getroot().nsmap - if (name.endswith('.genshi') or - ('py' in nsmap and nsmap['py'] == 'http://genshi.edgewall.org/')): - return BundleTemplateFile(name, self.encoding) - else: - return BundleFile(name) - def BuildStructures(self, metadata): """Build all structures for client (metadata).""" bundleset = [] @@ -156,8 +77,7 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): """ run plugin """ self.missing_bundles() for bundle in self.core.plugins['Bundler'].entries.values(): - if (self.HandlesFile(bundle.name) and - not isinstance(bundle, BundleTemplateFile)): + if self.HandlesFile(bundle.name): self.bundle_names(bundle) @classmethod @@ -186,14 +106,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): def bundle_names(self, bundle): """ verify bundle name attribute matches filename """ - try: - xdata = lxml.etree.XML(bundle.data) - except AttributeError: - # genshi template - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - fname = os.path.splitext(os.path.basename(bundle.name))[0] - bname = xdata.get('name') + bname = bundle.xdata.get('name') if fname != bname: self.LintError("inconsistent-bundle-name", "Inconsistent bundle name: filename is %s, " -- cgit v1.2.3-1-g7c22 From 1f6cb52d0c43f842766f3ecd6c8286f0f4eed5c2 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 4 Feb 2013 16:20:46 -0500 Subject: Bundler: various changes * Deprecated use of an explicit name attribute * Deprecated .genshi bundles * Minor restructuring for better performance * bcfg2-lint updates --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 101 ++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 39 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Bundler.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 051443e22..5eeb542ee 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -3,8 +3,6 @@ import os import re import sys -import copy -import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin import Bcfg2.Server.Lint @@ -13,12 +11,29 @@ from genshi.template import TemplateError class BundleFile(Bcfg2.Server.Plugin.StructFile): """ Representation of a bundle XML file """ - def get_xml_value(self, metadata): - """ get the XML data that applies to the given client """ - bundle = lxml.etree.Element('Bundle', name=self.xdata.get("name")) - for item in self.Match(metadata): - bundle.append(copy.copy(item)) - return bundle + bundle_name_re = re.compile('^(?P.*)\.(xml|genshi)$') + + def __init__(self, filename, should_monitor=False): + Bcfg2.Server.Plugin.StructFile.__init__(self, filename, + should_monitor=should_monitor) + if self.name.endswith(".genshi"): + self.logger.warning("Bundler: Bundle filenames ending with " + ".genshi are deprecated; add the Genshi XML " + "namespace to a .xml bundle instead") + __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ + + def Index(self): + Bcfg2.Server.Plugin.StructFile.Index(self) + if self.xdata.get("name"): + self.logger.warning("Bundler: Explicitly specifying bundle names " + "is deprecated") + Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + + @property + def bundle_name(self): + """ The name of the bundle, as determined from the filename """ + return self.bundle_name_re.match( + os.path.basename(self.name)).group("name") class Bundler(Bcfg2.Server.Plugin.Plugin, @@ -28,38 +43,35 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = BundleFile - patterns = re.compile('^(?P.*)\.(xml|genshi)$') def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Structure.__init__(self) - try: - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data) - except OSError: - err = sys.exc_info()[1] - msg = "Failed to load Bundle repository %s: %s" % (self.data, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) + Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data) + #: Bundles by bundle name, rather than filename + self.bundles = dict() + __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ - def BuildStructures(self, metadata): - """Build all structures for client (metadata).""" - bundleset = [] + def HandleEvent(self, event): + Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - bundle_entries = {} - for key, item in self.entries.items(): - bundle_entries.setdefault( - self.patterns.match(os.path.basename(key)).group('name'), - []).append(item) + self.bundles = dict() + for bundle in self.entries.values(): + self.bundles[bundle.bundle_name] = bundle + HandleEvent.__doc__ = \ + Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent.__doc__ + def BuildStructures(self, metadata): + bundleset = [] for bundlename in metadata.bundles: try: - entries = bundle_entries[bundlename] + bundle = self.bundles[bundlename] except KeyError: self.logger.error("Bundler: Bundle %s does not exist" % bundlename) continue try: - bundleset.append(entries[0].get_xml_value(metadata)) + bundleset.append(bundle.XMLMatch(metadata)) except TemplateError: err = sys.exc_info()[1] self.logger.error("Bundler: Failed to render templated bundle " @@ -68,6 +80,8 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, self.logger.error("Bundler: Unexpected bundler error for %s" % bundlename, exc_info=1) return bundleset + BuildStructures.__doc__ = \ + Bcfg2.Server.Plugin.Structure.BuildStructures.__doc__ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): @@ -83,7 +97,9 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): return {"bundle-not-found": "error", - "inconsistent-bundle-name": "warning"} + "unused-bundle": "warning", + "explicit-bundle-name": "error", + "genshi-extension-bundle": "error"} def missing_bundles(self): """ find bundles listed in Metadata but not implemented in Bundler """ @@ -94,21 +110,28 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): ref_bundles = set([b.get("name") for b in groupdata.findall("//Bundle")]) - allbundles = self.core.plugins['Bundler'].entries.keys() + allbundles = self.core.plugins['Bundler'].bundles.keys() for bundle in ref_bundles: - xmlbundle = "%s.xml" % bundle - genshibundle = "%s.genshi" % bundle - if (xmlbundle not in allbundles and - genshibundle not in allbundles): + if bundle not in allbundles: self.LintError("bundle-not-found", "Bundle %s referenced, but does not exist" % bundle) + for bundle in allbundles: + if bundle not in ref_bundles: + self.LintError("unused-bundle", + "Bundle %s defined, but is not referenced " + "in Metadata" % bundle) + def bundle_names(self, bundle): - """ verify bundle name attribute matches filename """ - fname = os.path.splitext(os.path.basename(bundle.name))[0] - bname = bundle.xdata.get('name') - if fname != bname: - self.LintError("inconsistent-bundle-name", - "Inconsistent bundle name: filename is %s, " - "bundle name is %s" % (fname, bname)) + """ Verify that deprecated bundle .genshi bundles and explicit + bundle names aren't used """ + if bundle.xdata.get('name'): + self.LintError("explicit-bundle-name", + "Deprecated explicit bundle name in %s" % + bundle.name) + + if bundle.name.endswith(".genshi"): + self.LintError("genshi-extension-bundle", + "Bundle %s uses deprecated .genshi extension" % + bundle.name) -- cgit v1.2.3-1-g7c22 From 0dab7e284017e4559019ac1e7b861ab7ccdadf5c Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 8 Feb 2013 10:29:03 -0500 Subject: Bundler: improved XInclude support, added inter-bundle dependencies --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 55 +++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 9 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Bundler.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 5eeb542ee..5a6767755 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -3,6 +3,7 @@ import os import re import sys +import copy import Bcfg2.Server import Bcfg2.Server.Plugin import Bcfg2.Server.Lint @@ -17,16 +18,17 @@ class BundleFile(Bcfg2.Server.Plugin.StructFile): Bcfg2.Server.Plugin.StructFile.__init__(self, filename, should_monitor=should_monitor) if self.name.endswith(".genshi"): - self.logger.warning("Bundler: Bundle filenames ending with " + self.logger.warning("Bundler: %s: Bundle filenames ending with " ".genshi are deprecated; add the Genshi XML " - "namespace to a .xml bundle instead") + "namespace to a .xml bundle instead" % + self.name) __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) if self.xdata.get("name"): - self.logger.warning("Bundler: Explicitly specifying bundle names " - "is deprecated") + self.logger.warning("Bundler: %s: Explicitly specifying bundle " + "names is deprecated" % self.name) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ @property @@ -55,30 +57,65 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, def HandleEvent(self, event): Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - self.bundles = dict() - for bundle in self.entries.values(): - self.bundles[bundle.bundle_name] = bundle + self.bundles = dict([(b.bundle_name, b) + for b in self.entries.values()]) HandleEvent.__doc__ = \ Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent.__doc__ def BuildStructures(self, metadata): bundleset = [] - for bundlename in metadata.bundles: + bundles = copy.copy(metadata.bundles) + bundles_added = set(bundles) + while bundles: + bundlename = bundles.pop() try: bundle = self.bundles[bundlename] except KeyError: self.logger.error("Bundler: Bundle %s does not exist" % bundlename) continue + try: - bundleset.append(bundle.XMLMatch(metadata)) + data = bundle.XMLMatch(metadata) except TemplateError: err = sys.exc_info()[1] self.logger.error("Bundler: Failed to render templated bundle " "%s: %s" % (bundlename, err)) + continue except: self.logger.error("Bundler: Unexpected bundler error for %s" % bundlename, exc_info=1) + continue + + data.set("name", bundlename) + + for child in data.findall("Bundle"): + if child.getchildren(): + # XInclude'd bundle -- "flatten" it so there + # aren't extra Bundle tags, since other bits in + # Bcfg2 only handle the direct children of the + # top-level Bundle tag + if data.get("name"): + self.logger.warning("Bundler: In file XIncluded from " + "%s: Explicitly specifying " + "bundle names is deprecated" % + self.name) + for el in child.getchildren(): + data.append(el) + data.remove(child) + elif child.get("name"): + # dependent bundle -- add it to the list of + # bundles for this client + if child.get("name") not in bundles_added: + bundles.append(child.get("name")) + bundles_added.add(child.get("name")) + data.remove(child) + else: + # neither name or children -- wat + self.logger.warning("Bundler: Useless empty Bundle tag " + "in %s" % self.name) + data.remove(child) + bundleset.append(data) return bundleset BuildStructures.__doc__ = \ Bcfg2.Server.Plugin.Structure.BuildStructures.__doc__ -- cgit v1.2.3-1-g7c22 From 398be2c5cb613d9506e0c115510c1b55881ca64e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 8 Feb 2013 13:43:40 -0500 Subject: Bundler: added support for independent bundles --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins/Bundler.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 5a6767755..d8290d844 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -87,6 +87,10 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, bundlename, exc_info=1) continue + if data.get("independent", "false").lower() == "true": + data.tag = "Independent" + del data.attrib['independent'] + data.set("name", bundlename) for child in data.findall("Bundle"): -- cgit v1.2.3-1-g7c22