From c35347887bb3d452d6104b13308d853b3da44b68 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 8 May 2012 15:09:54 -0400 Subject: modularized Cfg --- src/lib/Bcfg2/Server/Core.py | 4 +- src/lib/Bcfg2/Server/Plugin.py | 42 ++- src/lib/Bcfg2/Server/Plugins/Cfg.py | 293 ------------------ src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py | 20 ++ .../Server/Plugins/Cfg/CfgCheetahGenerator.py | 33 ++ src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py | 27 ++ .../Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 63 ++++ src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py | 24 ++ src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py | 28 ++ .../Server/Plugins/Cfg/CfgPlaintextGenerator.py | 8 + src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 331 +++++++++++++++++++++ 11 files changed, 566 insertions(+), 307 deletions(-) delete mode 100644 src/lib/Bcfg2/Server/Plugins/Cfg.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 0be28ee46..a253fd367 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -246,8 +246,8 @@ class Core(Component): exc = sys.exc_info()[1] if 'failure' not in entry.attrib: entry.set('failure', 'bind error: %s' % format_exc()) - logger.error("Failed to bind entry: %s %s" % \ - (entry.tag, entry.get('name'))) + logger.error("Failed to bind entry %s:%s: %s" % + (entry.tag, entry.get('name'), exc)) except Exception: exc = sys.exc_info()[1] if 'failure' not in entry.attrib: diff --git a/src/lib/Bcfg2/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin.py index dc70a630f..d7b4baf45 100644 --- a/src/lib/Bcfg2/Server/Plugin.py +++ b/src/lib/Bcfg2/Server/Plugin.py @@ -122,6 +122,9 @@ class Plugin(Debuggable): def shutdown(self): self.running = False + def __str__(self): + return "%s Plugin" % self.__class__.__name__ + class Generator(object): """Generator plugins contribute to literal client configurations.""" @@ -804,7 +807,7 @@ class XMLSrc(XMLFileBacked): return str(self.items) -class InfoXML (XMLSrc): +class InfoXML(XMLSrc): __node__ = InfoNode @@ -951,11 +954,12 @@ class SpecificData(object): logger.error("Failed to read file %s" % self.name) -class EntrySet(object): +class EntrySet(Debuggable): """Entry sets deal with the host- and group-specific entries.""" ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\\.genshi_include)$") def __init__(self, basename, path, entry_type, encoding): + Debuggable.__init__(self, name=basename) self.path = path self.entry_type = entry_type self.entries = {} @@ -966,14 +970,22 @@ class EntrySet(object): pattern += '(G(?P\d+)_(?P\S+))))?$' self.specific = re.compile(pattern) + def debug_log(self, message, flag=None): + if (flag is None and self.debug_flag) or flag: + logger.error(message) + + def sort_by_specific(self, one, other): + return cmp(one.specific, other.specific) + def get_matching(self, metadata): return [item for item in list(self.entries.values()) if item.specific.matches(metadata)] - def best_matching(self, metadata): + def best_matching(self, metadata, matching=None): """ Return the appropriate interpreted template from the set of available templates. """ - matching = self.get_matching(metadata) + if matching is None: + matching = self.get_matching(metadata) hspec = [ent for ent in matching if ent.specific.hostname] if hspec: @@ -1017,26 +1029,32 @@ class EntrySet(object): elif action == 'deleted': del self.entries[event.filename] - def entry_init(self, event): + def entry_init(self, event, entry_type=None, specific=None): """Handle template and info file creation.""" + if entry_type is None: + entry_type = self.entry_type + if event.filename in self.entries: logger.warn("Got duplicate add for %s" % event.filename) else: - fpath = "%s/%s" % (self.path, event.filename) + fpath = os.path.join(self.path, event.filename) try: - spec = self.specificity_from_filename(event.filename) + spec = self.specificity_from_filename(event.filename, + specific=specific) except SpecificityError: if not self.ignore.match(event.filename): logger.error("Could not process filename %s; ignoring" % fpath) return - self.entries[event.filename] = self.entry_type(fpath, - spec, self.encoding) + self.entries[event.filename] = entry_type(fpath, spec, + self.encoding) self.entries[event.filename].handle_event(event) - def specificity_from_filename(self, fname): + def specificity_from_filename(self, fname, specific=None): """Construct a specificity instance from a filename and regex.""" - data = self.specific.match(fname) + if specific is None: + specific = self.specific + data = specific.match(fname) if not data: raise SpecificityError(fname) kwargs = {} @@ -1053,7 +1071,7 @@ class EntrySet(object): def update_metadata(self, event): """Process info and info.xml files for the templates.""" - fpath = "%s/%s" % (self.path, event.filename) + fpath = os.path.join(self.path, event.filename) if event.filename == 'info.xml': if not self.infoxml: self.infoxml = InfoXML(fpath, True) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg.py b/src/lib/Bcfg2/Server/Plugins/Cfg.py deleted file mode 100644 index 81904d082..000000000 --- a/src/lib/Bcfg2/Server/Plugins/Cfg.py +++ /dev/null @@ -1,293 +0,0 @@ -"""This module implements a config file repository.""" - -import binascii -import logging -import lxml -import operator -import os -import os.path -import re -import stat -import sys -import tempfile -from subprocess import Popen, PIPE -from Bcfg2.Bcfg2Py3k import u_str - -import Bcfg2.Server.Plugin - -try: - import genshi.core - import genshi.input - from genshi.template import TemplateLoader, NewTextTemplate - have_genshi = True -except: - have_genshi = False - -try: - import Cheetah.Template - import Cheetah.Parser - have_cheetah = True -except: - have_cheetah = False - -# setup logging -logger = logging.getLogger('Bcfg2.Plugins.Cfg') - - -# snipped from TGenshi -def removecomment(stream): - """A genshi filter that removes comments from the stream.""" - for kind, data, pos in stream: - if kind is genshi.core.COMMENT: - continue - yield kind, data, pos - - -def process_delta(data, delta): - if not delta.specific.delta: - return data - if delta.specific.delta == 'cat': - datalines = data.strip().split('\n') - for line in delta.data.split('\n'): - if not line: - continue - if line[0] == '+': - datalines.append(line[1:]) - elif line[0] == '-': - if line[1:] in datalines: - datalines.remove(line[1:]) - return "\n".join(datalines) + "\n" - elif delta.specific.delta == 'diff': - basehandle, basename = tempfile.mkstemp() - basefile = open(basename, 'w') - basefile.write(data) - basefile.close() - os.close(basehandle) - - cmd = ["patch", "-u", "-f", basefile.name] - patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - stderr = patch.communicate(input=delta.data)[1] - ret = patch.wait() - output = open(basefile.name, 'r').read() - os.unlink(basefile.name) - if ret >> 8 != 0: - logger.error("Error applying diff %s: %s" % (delta.name, stderr)) - raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta) - return output - - -class CfgMatcher: - - def __init__(self, fname): - name = re.escape(fname) - self.basefile_reg = re.compile('^(?P%s)(|\\.H_(?P\S+?)|.G(?P\d+)_(?P\S+?))((?P\\.genshi)|(?P\\.cheetah))?$' % name) - self.delta_reg = re.compile('^(?P%s)(|\\.H_(?P\S+)|\\.G(?P\d+)_(?P\S+))\\.(?P(cat|diff))$' % name) - self.cat_count = fname.count(".cat") - self.diff_count = fname.count(".diff") - - def match(self, fname): - if fname.count(".cat") > self.cat_count \ - or fname.count('.diff') > self.diff_count: - return self.delta_reg.match(fname) - return self.basefile_reg.match(fname) - - -class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): - - def __init__(self, basename, path, entry_type, encoding): - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, - entry_type, encoding) - self.specific = CfgMatcher(path.split('/')[-1]) - path = path - - def debug_log(self, message, flag=None): - if (flag is None and self.debug_flag) or flag: - logger.error(message) - - def sort_by_specific(self, one, other): - return cmp(one.specific, other.specific) - - def get_pertinent_entries(self, entry, metadata): - """return a list of all entries pertinent - to a client => [base, delta1, delta2] - """ - matching = [ent for ent in list(self.entries.values()) if \ - ent.specific.matches(metadata)] - matching.sort(key=operator.attrgetter('specific')) - # base entries which apply to a client - # (e.g. foo, foo.G##_groupname, foo.H_hostname) - base_files = [matching.index(m) for m in matching - if not m.specific.delta] - if not base_files: - msg = "No base file found for %s" % entry.get('name') - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - base = min(base_files) - used = matching[:base + 1] - used.reverse() - return used - - def bind_entry(self, entry, metadata): - self.bind_info_to_entry(entry, metadata) - used = self.get_pertinent_entries(entry, metadata) - basefile = used.pop(0) - if entry.get('perms').lower() == 'inherit': - # use on-disk permissions - fname = os.path.join(self.path, entry.get('name')) - entry.set('perms', - str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) - if entry.tag == 'Path': - entry.set('type', 'file') - if basefile.name.endswith(".genshi"): - if not have_genshi: - msg = "Cfg: Genshi is not available: %s" % entry.get("name") - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - try: - template_cls = NewTextTemplate - loader = TemplateLoader() - template = loader.load(basefile.name, cls=template_cls, - encoding=self.encoding) - fname = entry.get('realname', entry.get('name')) - stream = template.generate(name=fname, - metadata=metadata, - path=basefile.name).filter(removecomment) - try: - data = stream.render('text', encoding=self.encoding, - strip_whitespace=False) - except TypeError: - data = stream.render('text', encoding=self.encoding) - if data == '': - entry.set('empty', 'true') - except Exception: - msg = "Cfg: genshi exception (%s): %s" % (entry.get("name"), - sys.exc_info()[1]) - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - elif basefile.name.endswith(".cheetah"): - if not have_cheetah: - msg = "Cfg: Cheetah is not available: %s" % entry.get("name") - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - try: - fname = entry.get('realname', entry.get('name')) - s = {'useStackFrames': False} - template = Cheetah.Template.Template(open(basefile.name).read(), - compilerSettings=s) - template.metadata = metadata - template.path = fname - template.source_path = basefile.name - data = template.respond() - if data == '': - entry.set('empty', 'true') - except Exception: - msg = "Cfg: cheetah exception (%s): %s" % (entry.get("name"), - sys.exc_info()[1]) - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - else: - data = basefile.data - for delta in used: - data = process_delta(data, delta) - if entry.get('encoding') == 'base64': - entry.text = binascii.b2a_base64(data) - else: - try: - entry.text = u_str(data, self.encoding) - except UnicodeDecodeError: - msg = "Failed to decode %s: %s" % (entry.get('name'), - sys.exc_info()[1]) - logger.error(msg) - logger.error("Please verify you are using the proper encoding.") - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - except ValueError: - msg = "Error in specification for %s: %s" % (entry.get('name'), - sys.exc_info()[1]) - logger.error(msg) - logger.error("You need to specify base64 encoding for %s." % - entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - if entry.text in ['', None]: - entry.set('empty', 'true') - - def list_accept_choices(self, entry, metadata): - '''return a list of candidate pull locations''' - used = self.get_pertinent_entries(entry, metadata) - ret = [] - if used: - ret.append(used[0].specific) - if not ret[0].hostname: - ret.append(Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)) - return ret - - def build_filename(self, specific): - bfname = self.path + '/' + self.path.split('/')[-1] - if specific.all: - return bfname - elif specific.group: - return "%s.G%02d_%s" % (bfname, specific.prio, specific.group) - elif specific.hostname: - return "%s.H_%s" % (bfname, specific.hostname) - - def write_update(self, specific, new_entry, log): - if 'text' in new_entry: - name = self.build_filename(specific) - if os.path.exists("%s.genshi" % name): - msg = "Cfg: Unable to pull data for genshi types" - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - elif os.path.exists("%s.cheetah" % name): - msg = "Cfg: Unable to pull data for cheetah types" - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - try: - etext = new_entry['text'].encode(self.encoding) - except: - msg = "Cfg: Cannot encode content of %s as %s" % (name, - self.encoding) - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - open(name, 'w').write(etext) - self.debug_log("Wrote file %s" % name, flag=log) - badattr = [attr for attr in ['owner', 'group', 'perms'] - if attr in new_entry] - if badattr: - # check for info files and inform user of their removal - if os.path.exists(self.path + "/:info"): - logger.info("Removing :info file and replacing with " - "info.xml") - os.remove(self.path + "/:info") - if os.path.exists(self.path + "/info"): - logger.info("Removing info file and replacing with " - "info.xml") - os.remove(self.path + "/info") - metadata_updates = {} - metadata_updates.update(self.metadata) - for attr in badattr: - metadata_updates[attr] = new_entry.get(attr) - infoxml = lxml.etree.Element('FileInfo') - infotag = lxml.etree.SubElement(infoxml, 'Info') - [infotag.attrib.__setitem__(attr, metadata_updates[attr]) \ - for attr in metadata_updates] - ofile = open(self.path + "/info.xml", "w") - ofile.write(lxml.etree.tostring(infoxml, pretty_print=True)) - ofile.close() - self.debug_log("Wrote file %s" % (self.path + "/info.xml"), - flag=log) - - -class Cfg(Bcfg2.Server.Plugin.GroupSpool, - Bcfg2.Server.Plugin.PullTarget): - """This generator in the configuration file repository for Bcfg2.""" - name = 'Cfg' - __author__ = 'bcfg-dev@mcs.anl.gov' - es_cls = CfgEntrySet - es_child_cls = Bcfg2.Server.Plugin.SpecificData - - def AcceptChoices(self, entry, metadata): - return self.entries[entry.get('name')].list_accept_choices(entry, metadata) - - def AcceptPullData(self, specific, new_entry, log): - return self.entries[new_entry.get('name')].write_update(specific, - new_entry, - log) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py new file mode 100644 index 000000000..f6b175832 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py @@ -0,0 +1,20 @@ +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgFilter + +logger = logging.getLogger(__name__) + +class CfgCatFilter(CfgFilter): + __extensions__ = ['cat'] + + def modify_data(self, entry, metadata, data): + datalines = data.strip().split('\n') + for line in self.data.split('\n'): + if not line: + continue + if line.startswith('+'): + datalines.append(line[1:]) + elif line.startswith('-'): + if line[1:] in datalines: + datalines.remove(line[1:]) + return "\n".join(datalines) + "\n" diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py new file mode 100644 index 000000000..08f01e005 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -0,0 +1,33 @@ +import copy +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + +logger = logging.getLogger(__name__) + +try: + import Cheetah.Template + import Cheetah.Parser + have_cheetah = True +except: + have_cheetah = False + + +class CfgCheetahGenerator(CfgGenerator): + __extensions__ = ['cheetah'] + settings = dict(useStackFrames=False) + + def __init__(self, fname, spec, encoding): + CfgGenerator.__init__(self, fname, spec, encoding) + if not have_cheetah: + msg = "Cfg: Cheetah is not available: %s" % entry.get("name") + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + def get_data(self, entry, metadata): + template = Cheetah.Template.Template(self.data, + compilerSettings=self.settings) + template.metadata = metadata + template.path = entry.get('realname', entry.get('name')) + template.source_path = self.path + return template.respond() diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py new file mode 100644 index 000000000..b408e1b55 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py @@ -0,0 +1,27 @@ +import os +import logging +import tempfile +import Bcfg2.Server.Plugin +from subprocess import Popen, PIPE +from Bcfg2.Server.Plugins.Cfg import CfgFilter + +logger = logging.getLogger(__name__) + +class CfgDiffFilter(CfgFilter): + __extensions__ = ['diff'] + + def modify_data(self, entry, metadata, data): + basehandle, basename = tempfile.mkstemp() + open(basename, 'w').write(data) + os.close(basehandle) + + cmd = ["patch", "-u", "-f", basefile.name] + patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + stderr = patch.communicate(input=self.data)[1] + ret = patch.wait() + output = open(basefile.name, 'r').read() + os.unlink(basefile.name) + if ret >> 8 != 0: + logger.error("Error applying diff %s: %s" % (delta.name, stderr)) + raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta) + return output diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py new file mode 100644 index 000000000..5e3b37127 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -0,0 +1,63 @@ +import sys +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + +logger = logging.getLogger(__name__) + +try: + import genshi.core + from genshi.template import TemplateLoader, NewTextTemplate + have_genshi = True +except: + have_genshi = False + +# snipped from TGenshi +def removecomment(stream): + """A genshi filter that removes comments from the stream.""" + for kind, data, pos in stream: + if kind is genshi.core.COMMENT: + continue + yield kind, data, pos + + +class CfgGenshiGenerator(CfgGenerator): + __extensions__ = ['genshi'] + + def __init__(self, fname, spec, encoding): + CfgGenerator.__init__(self, fname, spec, encoding) + self.loader = TemplateLoader() + if not have_genshi: + msg = "Cfg: Genshi is not available: %s" % entry.get("name") + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + @classmethod + def ignore(cls, basename, event): + return (event.filename.endswith(".genshi_include") or + CfgGenerator.ignore(basename, event)) + + def get_data(self, entry, metadata): + fname = entry.get('realname', entry.get('name')) + stream = \ + self.template.generate(name=fname, + metadata=metadata, + path=self.name).filter(removecomment) + try: + return stream.render('text', encoding=self.encoding, + strip_whitespace=False) + except TypeError: + return stream.render('text', encoding=self.encoding) + + def handle_event(self, event): + if event.code2str() == 'deleted': + return + try: + self.template = self.loader.load(self.name, cls=NewTextTemplate, + encoding=self.encoding) + except Exception: + msg = "Cfg: Could not load template %s: %s" % (self.name, + sys.exc_info()[1]) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py new file mode 100644 index 000000000..35aaa0442 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py @@ -0,0 +1,24 @@ +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgInfo + +logger = logging.getLogger('Bcfg2.Plugins.Cfg') + +class CfgInfoXML(CfgInfo): + names = ['info.xml'] + + def __init__(self, path): + CfgInfo.__init__(self, path) + self.infoxml = Bcfg2.Server.Plugin.InfoXML(path, noprio=True) + + def bind_info_to_entry(self, entry, metadata): + mdata = dict() + self.infoxml.pnode.Match(metadata, mdata, entry=entry) + if 'Info' not in mdata: + logger.error("Failed to set metadata for file %s" % + entry.get('name')) + raise PluginExecutionError + self._set_info(entry, mdata['Info'][None]) + + def handle_event(self, event): + self.infoxml.HandleEvent() diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py new file mode 100644 index 000000000..9616f8bba --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py @@ -0,0 +1,28 @@ +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgInfo + +logger = logging.getLogger('Bcfg2.Plugins.Cfg') + +class CfgLegacyInfo(CfgInfo): + names = ['info', ':info'] + + def bind_info_to_entry(self, entry, metadata): + self._set_info(entry, self.metadata) + + def handle_event(self, event): + if event.code2str() == 'deleted': + return + for line in open(self.path).readlines(): + match = Bcfg2.Server.Plugin.info_regex.match(line) + if not match: + logger.warning("Failed to parse line in %s: %s" % (fpath, line)) + continue + else: + self.metadata = \ + dict([(key, value) + for key, value in list(match.groupdict().items()) + if value]) + if ('perms' in self.metadata and + len(self.metadata['perms']) == 3): + self.metadata['perms'] = "0%s" % self.metadata['perms'] diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py new file mode 100644 index 000000000..3351209f3 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py @@ -0,0 +1,8 @@ +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + +logger = logging.getLogger('Bcfg2.Plugins.Cfg') + +class CfgPlaintextGenerator(CfgGenerator): + pass diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py new file mode 100644 index 000000000..9bac50e44 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -0,0 +1,331 @@ +"""This module implements a config file repository.""" + +import re +import os +import sys +import stat +import pkgutil +import logging +import binascii +import lxml.etree +import Bcfg2.Server.Plugin +from Bcfg2.Bcfg2Py3k import u_str + +logger = logging.getLogger('Bcfg2.Plugins.Cfg') + +PROCESSORS = None + +class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): + __extensions__ = [] + __ignore__ = [] + + def __init__(self, fname, spec, encoding): + Bcfg2.Server.Plugin.SpecificData.__init__(self, fname, spec, encoding) + self.encoding = encoding + self.regex = self.__class__.get_regex(fname) + + @classmethod + def get_regex(cls, fname, extensions=None): + if extensions is None: + extensions = cls.__extensions__ + + base_re = '^(?P%s)(|\\.H_(?P\S+?)|.G(?P\d+)_(?P\S+?))' % re.escape(fname) + if extensions: + base_re += '\\.(?P%s)' % '|'.join(extensions) + base_re += '$' + return re.compile(base_re) + + @classmethod + def handles(cls, basename, event): + return (event.filename.startswith(os.path.basename(basename)) and + cls.get_regex(os.path.basename(basename)).match(event.filename)) + + @classmethod + def ignore(cls, basename, event): + return (cls.__ignore__ and + event.filename.startswith(os.path.basename(basename)) and + cls.get_regex(os.path.basename(basename), + extensions=cls.__ignore__).match(event.filename)) + + + def __str__(self): + return "%s(%s)" % (self.__class__.__name__, self.name) + + def match(self, fname): + return self.regex.match(fname) + + +class CfgGenerator(CfgBaseFileMatcher): + def get_data(self, entry, metadata): + return self.data + + +class CfgFilter(CfgBaseFileMatcher): + def modify_data(self, entry, metadata, data): + raise NotImplementedError + + +class CfgInfo(Bcfg2.Server.Plugin.SpecificData): + names = [] + regex = re.compile('^$') + + def __init__(self, path): + self.path = path + self.name = os.path.basename(path) + + @classmethod + def handles(cls, basename, event): + return event.filename in cls.names or cls.regex.match(event.filename) + + @classmethod + def ignore(cls, basename, event): + return False + + def bind_info_to_entry(self, entry, metadata): + raise NotImplementedError + + def _set_info(self, entry, info): + for key, value in list(info.items()): + entry.attrib.__setitem__(key, value) + + def __str__(self): + return "%s(%s)" % (self.__class__.__name__, self.name) + + +class CfgDefaultInfo(CfgInfo): + def __init__(self, defaults): + self.name = '' + self.defaults = defaults + + def handles(self, event): + return False + + def bind_info_to_entry(self, entry, metadata): + self._set_info(entry, self.defaults) + + +class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): + def __init__(self, basename, path, entry_type, encoding): + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, + entry_type, encoding) + self.specific = None + self.default_info = CfgDefaultInfo(self.metadata) + self.load_processors() + + def load_processors(self): + """ load Cfg file processors. this must be done at run-time, + not at compile-time, or we get a circular import and things + don't work. but finding the right way to do this at runtime + was ... problematic. so here it is, writing to a global + variable. Sorry 'bout that. """ + global PROCESSORS + if PROCESSORS is None: + PROCESSORS = [] + for submodule in pkgutil.walk_packages(path=__path__): + module = getattr(__import__("%s.%s" % + (__name__, + submodule[1])).Server.Plugins.Cfg, + submodule[1]) + proc = getattr(module, submodule[1]) + if set(proc.__mro__).intersection([CfgInfo, CfgFilter, + CfgGenerator]): + PROCESSORS.append(proc) + + def handle_event(self, event): + action = event.code2str() + + for proc in PROCESSORS: + if proc.handles(self.path, event): + self.debug_log("%s handling %s event on %s" % + (proc.__name__, action, event.filename)) + if action in ['exists', 'created']: + self.entry_init(event, proc) + elif event.filename not in self.entries: + logger.warning("Got %s event for unknown file %s" % + (action, event.filename)) + if action == 'changed': + # received a bogus changed event; warn, but + # treat it like a created event + self.entry_init(event, proc) + elif action == 'changed': + self.entries[event.filename].handle_event(event) + elif action == 'deleted': + del self.entries[event.filename] + return + elif proc.ignore(self.path, event): + return + + logger.error("Could not process filename %s; ignoring" % + event.filename) + + def entry_init(self, event, proc): + if CfgBaseFileMatcher in proc.__mro__: + Bcfg2.Server.Plugin.EntrySet.entry_init( + self, event, entry_type=proc, + specific=proc.get_regex(os.path.basename(self.path))) + else: + if event.filename in self.entries: + logger.warn("Got duplicate add for %s" % event.filename) + else: + fpath = os.path.join(self.path, event.filename) + self.entries[event.filename] = proc(fpath) + self.entries[event.filename].handle_event(event) + + def bind_entry(self, entry, metadata): + info_handlers = [] + generators = [] + filters = [] + for ent in self.entries.values(): + if (hasattr(ent, 'specific') and + not ent.specific.matches(metadata)): + continue + if isinstance(ent, CfgInfo): + info_handlers.append(ent) + elif isinstance(ent, CfgGenerator): + generators.append(ent) + elif isinstance(ent, CfgFilter): + filters.append(ent) + + self.default_info.bind_info_to_entry(entry, metadata) + if len(info_handlers) > 1: + logger.error("More than one info supplier found for %s: %s" % + (self.name, info_handlers)) + if len(info_handlers): + info_handlers[0].bind_info_to_entry(entry, metadata) + if entry.tag == 'Path': + entry.set('type', 'file') + + generator = self.best_matching(metadata, generators) + if entry.get('perms').lower() == 'inherit': + # use on-disk permissions + fname = os.path.join(self.path, generator.name) + entry.set('perms', + str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) + try: + data = generator.get_data(entry, metadata) + except: + msg = "Cfg: exception rendering %s with %s: %s" % \ + (entry.get("name"), generator, sys.exc_info()[1]) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + for fltr in filters: + data = fltr.modify_data(entry, metadata, data) + + if entry.get('encoding') == 'base64': + data = binascii.b2a_base64(data) + else: + try: + data = u_str(data, self.encoding) + except UnicodeDecodeError: + msg = "Failed to decode %s: %s" % (entry.get('name'), + sys.exc_info()[1]) + logger.error(msg) + logger.error("Please verify you are using the proper encoding.") + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + except ValueError: + msg = "Error in specification for %s: %s" % (entry.get('name'), + sys.exc_info()[1]) + logger.error(msg) + logger.error("You need to specify base64 encoding for %s." % + entry.get('name')) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + if data: + entry.text = data + else: + entry.set('empty', 'true') + + def list_accept_choices(self, entry, metadata): + '''return a list of candidate pull locations''' + generators = [ent for ent in list(self.entries.values()) + if (isinstance(ent, CfgGenerator) and + ent.specific.matches(metadata))] + if not matching: + msg = "No base file found for %s" % entry.get('name') + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + rv = [] + try: + best = self.best_matching(metadata, generators) + rv.append(best.specific) + except: + pass + + if not rv or not rv[0].hostname: + rv.append(Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)) + return rv + + def build_filename(self, specific): + bfname = self.path + '/' + self.path.split('/')[-1] + if specific.all: + return bfname + elif specific.group: + return "%s.G%02d_%s" % (bfname, specific.prio, specific.group) + elif specific.hostname: + return "%s.H_%s" % (bfname, specific.hostname) + + def write_update(self, specific, new_entry, log): + if 'text' in new_entry: + name = self.build_filename(specific) + if os.path.exists("%s.genshi" % name): + msg = "Cfg: Unable to pull data for genshi types" + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + elif os.path.exists("%s.cheetah" % name): + msg = "Cfg: Unable to pull data for cheetah types" + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + try: + etext = new_entry['text'].encode(self.encoding) + except: + msg = "Cfg: Cannot encode content of %s as %s" % (name, + self.encoding) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + open(name, 'w').write(etext) + self.debug_log("Wrote file %s" % name, flag=log) + badattr = [attr for attr in ['owner', 'group', 'perms'] + if attr in new_entry] + if badattr: + # check for info files and inform user of their removal + if os.path.exists(self.path + "/:info"): + logger.info("Removing :info file and replacing with " + "info.xml") + os.remove(self.path + "/:info") + if os.path.exists(self.path + "/info"): + logger.info("Removing info file and replacing with " + "info.xml") + os.remove(self.path + "/info") + metadata_updates = {} + metadata_updates.update(self.metadata) + for attr in badattr: + metadata_updates[attr] = new_entry.get(attr) + infoxml = lxml.etree.Element('FileInfo') + infotag = lxml.etree.SubElement(infoxml, 'Info') + [infotag.attrib.__setitem__(attr, metadata_updates[attr]) \ + for attr in metadata_updates] + ofile = open(self.path + "/info.xml", "w") + ofile.write(lxml.etree.tostring(infoxml, pretty_print=True)) + ofile.close() + self.debug_log("Wrote file %s" % (self.path + "/info.xml"), + flag=log) + + +class Cfg(Bcfg2.Server.Plugin.GroupSpool, + Bcfg2.Server.Plugin.PullTarget): + """This generator in the configuration file repository for Bcfg2.""" + name = 'Cfg' + __author__ = 'bcfg-dev@mcs.anl.gov' + es_cls = CfgEntrySet + es_child_cls = Bcfg2.Server.Plugin.SpecificData + + def AcceptChoices(self, entry, metadata): + return self.entries[entry.get('name')].list_accept_choices(entry, + metadata) + + def AcceptPullData(self, specific, new_entry, log): + return self.entries[new_entry.get('name')].write_update(specific, + new_entry, + log) -- cgit v1.2.3-1-g7c22