From 255046b3f3484219bbfa6646c88bbc4bbc1b5256 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 16 Nov 2012 14:40:42 -0500 Subject: Templating updates: * Added "repo" variable to all template formats * Made variables available in Genshi and Cheetah template more similar * Improved docs --- src/lib/Bcfg2/Server/Plugins/Bundler.py | 15 ++++++++++++--- src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py | 4 +++- src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 6 ++++-- 3 files changed, 19 insertions(+), 6 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 7933fe9be..b200346bc 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -19,6 +19,9 @@ except ImportError: HAS_GENSHI = False +SETUP = None + + class BundleFile(Bcfg2.Server.Plugin.StructFile): """ Representation of a bundle XML file """ def get_xml_value(self, metadata): @@ -49,7 +52,8 @@ if HAS_GENSHI: msg = "No parsed template information for %s" % self.name self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - stream = self.template.generate(metadata=metadata).filter( + stream = self.template.generate(metadata=metadata, + repo=SETUP['repo']).filter( Bcfg2.Server.Plugins.TGenshi.removecomment) data = lxml.etree.XML(stream.render('xml', strip_whitespace=False), @@ -93,8 +97,13 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, self.data, self.core.fam) except OSError: - self.logger.error("Failed to load Bundle repository") - raise Bcfg2.Server.Plugin.PluginInitError + err = sys.exc_info()[1] + msg = "Failed to load Bundle repository %s: %s" % (self.data, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) + + global SETUP + SETUP = core.setup def template_dispatch(self, name, _): """ Add the correct child entry type to Bundler depending on diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index 8ebd8d921..724164cf5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -3,7 +3,7 @@ :ref:`server-plugins-generators-cfg` files. """ from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator +from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP try: from Cheetah.Template import Template @@ -37,7 +37,9 @@ class CfgCheetahGenerator(CfgGenerator): template = Template(self.data.decode(self.encoding), compilerSettings=self.settings) template.metadata = metadata + template.name = entry.get('realname', entry.get('name')) template.path = entry.get('realname', entry.get('name')) template.source_path = self.name + template.repo = SETUP['repo'] return template.respond() get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index df0c30c09..48f64ac7f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -6,7 +6,7 @@ import re import sys import traceback from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator +from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP try: import genshi.core @@ -74,7 +74,9 @@ class CfgGenshiGenerator(CfgGenerator): stream = \ self.template.generate(name=fname, metadata=metadata, - path=self.name).filter(removecomment) + path=self.name, + source_path=self.name, + repo=SETUP['repo']).filter(removecomment) try: try: return stream.render('text', encoding=self.encoding, -- cgit v1.2.3-1-g7c22 From 8399ec79ec9c286f7371e203a3cafe52cf574ace Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 19 Nov 2012 13:01:59 -0500 Subject: fixed vcs_root/vcs_path for Version plugins, esp. Svn --- src/lib/Bcfg2/Server/Plugins/Svn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index fda6b57b5..57689cf33 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -17,10 +17,9 @@ except ImportError: class Svn(Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" __author__ = 'bcfg-dev@mcs.anl.gov' + __vcs_metadata_path__ = ".svn" if HAS_SVN: __rmi__ = Bcfg2.Server.Plugin.Version.__rmi__ + ['Update', 'Commit'] - else: - __vcs_metadata_path__ = ".svn" def __init__(self, core, datastore): Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) -- cgit v1.2.3-1-g7c22 From 1701d9ec63fc1197a49384cc3341525bdff8d840 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 20 Nov 2012 12:41:02 -0500 Subject: SSLCA: read SSLCA config without instantiating a new ConfigParser --- src/lib/Bcfg2/Server/Plugins/SSLCA.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index 62396f860..b3a49c047 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -73,9 +73,8 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): cert_spec.get('append_chain', 'false').lower() == 'true', } - cfp = ConfigParser.ConfigParser() - cfp.read(self.core.cfile) - self.CAs[ca] = dict(cfp.items('sslca_' + ca)) + self.CAs[ca] = dict(self.core.setup.cfp.items('sslca_%s' % + ca)) self.Entries['Path'][ident] = self.get_cert elif event.filename.endswith("info.xml"): self.infoxml[ident] = Bcfg2.Server.Plugin.InfoXML(epath) -- cgit v1.2.3-1-g7c22 From d32fd476787eddc8921eeadbfba5fa7e601ca231 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 26 Nov 2012 10:59:49 -0500 Subject: avoid odd edge case in genshi error handling --- src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index 48f64ac7f..3a78b4847 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -137,9 +137,13 @@ class CfgGenshiGenerator(CfgGenerator): # single line break) real_lineno = lineno - contents.code.co_firstlineno src = re.sub(r'\n\n+', '\n', contents.source).splitlines() - raise PluginExecutionError("%s: %s at '%s'" % - (err.__class__.__name__, err, - src[real_lineno])) + try: + raise PluginExecutionError("%s: %s at '%s'" % + (err.__class__.__name__, err, + src[real_lineno])) + except IndexError: + raise PluginExecutionError("%s: %s" % + (err.__class__.__name__, err)) raise def handle_event(self, event): -- cgit v1.2.3-1-g7c22 From fbecb8553136649eaf563d4f7ec21553500e5f16 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 27 Nov 2012 11:41:26 -0500 Subject: gave ProbeSet a pretty stringification for debug output --- src/lib/Bcfg2/Server/Plugins/Probes.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 90dff4a66..f106b75a4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -162,6 +162,9 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ret.append(probe) return ret + def __str__(self): + return "ProbeSet for %s" % self.plugin_name + class Probes(Bcfg2.Server.Plugin.Probing, Bcfg2.Server.Plugin.Connector, -- cgit v1.2.3-1-g7c22 From c19e7da20ca6b67956338f9808a80673e06b1e94 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 27 Nov 2012 11:49:47 -0500 Subject: Threaded plugin fixes: * Added "Threaded" plugin interface for any plugin that uses threads * Start plugin threads after daemonization * Update existing plugins that use threads (Reporting, Snapshots, ThreadedStatistics interface) * Update unit tests --- src/lib/Bcfg2/Server/Plugins/Reporting.py | 12 ++++++++---- src/lib/Bcfg2/Server/Plugins/Snapshots.py | 2 ++ 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 1a8c3d941..b9d397590 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -7,8 +7,8 @@ import lxml.etree from Bcfg2.Reporting.Transport import load_transport_from_config, \ TransportError from Bcfg2.Options import REPORTING_COMMON_OPTIONS -from Bcfg2.Server.Plugin import Statistics, PullSource, PluginInitError, \ - PluginExecutionError +from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \ + PluginInitError, PluginExecutionError # required for reporting try: @@ -31,7 +31,7 @@ def _rpc_call(method): return _real_rpc_call -class Reporting(Statistics, PullSource): # pylint: disable=W0223 +class Reporting(Statistics, Threaded, PullSource): # pylint: disable=W0223 """ Unified statistics and reporting plugin """ __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] @@ -41,6 +41,7 @@ class Reporting(Statistics, PullSource): # pylint: disable=W0223 def __init__(self, core, datastore): Statistics.__init__(self, core, datastore) PullSource.__init__(self) + Threaded.__init__(self) self.core = core self.whoami = platform.node() @@ -54,8 +55,11 @@ class Reporting(Statistics, PullSource): # pylint: disable=W0223 self.logger.error(msg) raise PluginInitError(msg) + self.transport = None + + def start_threads(self): try: - self.transport = load_transport_from_config(core.setup) + self.transport = load_transport_from_config(self.core.setup) except TransportError: msg = "%s: Failed to load transport: %s" % \ (self.name, traceback.format_exc().splitlines()[-1]) diff --git a/src/lib/Bcfg2/Server/Plugins/Snapshots.py b/src/lib/Bcfg2/Server/Plugins/Snapshots.py index 1956af4ad..cc5946bb2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Snapshots.py +++ b/src/lib/Bcfg2/Server/Plugins/Snapshots.py @@ -65,6 +65,8 @@ class Snapshots(Bcfg2.Server.Plugin.Statistics): self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile) self.work_queue = Queue() self.loader = threading.Thread(target=self.load_snapshot) + + def start_threads(self): self.loader.start() def load_snapshot(self): -- cgit v1.2.3-1-g7c22 From 92122665e534978a1bdb499e6b8cf48e54b041d3 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 27 Nov 2012 12:09:18 -0500 Subject: made Reporting plugin and transports debuggable, respond to set_debug/toggle_debug RMI --- src/lib/Bcfg2/Server/Plugins/Reporting.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index b9d397590..60f5b1e09 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -8,7 +8,7 @@ from Bcfg2.Reporting.Transport import load_transport_from_config, \ TransportError from Bcfg2.Options import REPORTING_COMMON_OPTIONS from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \ - PluginInitError, PluginExecutionError + Debuggable, PluginInitError, PluginExecutionError # required for reporting try: @@ -31,9 +31,10 @@ def _rpc_call(method): return _real_rpc_call -class Reporting(Statistics, Threaded, PullSource): # pylint: disable=W0223 +# pylint: disable=W0223 +class Reporting(Statistics, Threaded, PullSource, Debuggable): """ Unified statistics and reporting plugin """ - __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry'] + __rmi__ = Debuggable.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry'] CLIENT_METADATA_FIELDS = ('profile', 'bundles', 'aliases', 'addresses', 'groups', 'categories', 'uuid', 'version') @@ -42,7 +43,7 @@ class Reporting(Statistics, Threaded, PullSource): # pylint: disable=W0223 Statistics.__init__(self, core, datastore) PullSource.__init__(self) Threaded.__init__(self) - self.core = core + Debuggable.__init__(self) self.whoami = platform.node() self.transport = None @@ -55,8 +56,6 @@ class Reporting(Statistics, Threaded, PullSource): # pylint: disable=W0223 self.logger.error(msg) raise PluginInitError(msg) - self.transport = None - def start_threads(self): try: self.transport = load_transport_from_config(self.core.setup) @@ -66,6 +65,11 @@ class Reporting(Statistics, Threaded, PullSource): # pylint: disable=W0223 self.logger.error(msg) raise PluginInitError(msg) + def set_debug(self, debug): + rv = Debuggable.set_debug(self, debug) + self.transport.set_debug(debug) + return rv + def process_statistics(self, client, xdata): stats = xdata.find("Statistics") stats.set('time', time.asctime(time.localtime())) @@ -88,8 +92,8 @@ class Reporting(Statistics, Threaded, PullSource): # pylint: disable=W0223 lxml.etree.tostring( stats, xml_declaration=False).decode('UTF-8')) - self.logger.debug("%s: Queued statistics data for %s" % - (self.__class__.__name__, client.hostname)) + self.debug_log("%s: Queued statistics data for %s" % + (self.__class__.__name__, client.hostname)) return except TransportError: continue @@ -98,7 +102,7 @@ class Reporting(Statistics, Threaded, PullSource): # pylint: disable=W0223 % (self.__class__.__name__, i, traceback.format_exc().splitlines()[-1])) self.logger.error("%s: Retry limit reached for %s" % - (self.__class__.__name__, client.hostname)) + (self.__class__.__name__, client.hostname)) def shutdown(self): super(Reporting, self).shutdown() -- cgit v1.2.3-1-g7c22 From 4c70626094248495bf2c11c09bf2f2f60917187d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 27 Nov 2012 13:06:45 -0500 Subject: catch pulp errors when creating/binding consumers --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 35 ++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 10 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 220146100..37171e1b1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -536,22 +536,37 @@ class YumCollection(Collection): consumerapi = ConsumerAPI() consumer = self._get_pulp_consumer(consumerapi=consumerapi) if consumer is None: - consumer = consumerapi.create(self.metadata.hostname, - self.metadata.hostname, - capabilities=dict(bind=False)) - lxml.etree.SubElement(independent, "BoundAction", - name="pulp-update", timing="pre", - when="always", status="check", - command="pulp-consumer consumer update") - self.pulp_cert_set.write_data(consumer['certificate'], - self.metadata) + try: + consumer = \ + consumerapi.create(self.metadata.hostname, + self.metadata.hostname, + capabilities=dict(bind=False)) + lxml.etree.SubElement( + independent, "BoundAction", name="pulp-update", + timing="pre", when="always", status="check", + command="pulp-consumer consumer update") + self.pulp_cert_set.write_data(consumer['certificate'], + self.metadata) + except server.ServerRequestError: + err = sys.exc_info()[1] + self.logger.error("Packages: Could not create Pulp " + "consumer %s: %s" % + (self.metadata.hostname, err)) for source in self: # each pulp source can only have one arch, so we don't # have to check the arch in url_map if (source.pulp_id and source.pulp_id not in consumer['repoids']): - consumerapi.bind(self.metadata.hostname, source.pulp_id) + try: + consumerapi.bind(self.metadata.hostname, + source.pulp_id) + except server.ServerRequestError: + err = sys.exc_info()[1] + self.logger.error("Packages: Could not bind %s to " + "Pulp repo %s: %s" % + (self.metadata.hostname, + source.pulp_id, err)) crt = lxml.etree.SubElement(independent, "BoundPath", name=self.pulp_cert_set.certpath) -- cgit v1.2.3-1-g7c22