From 10326a34dd813b88c6c8816115e91977a93a1f10 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 20 Dec 2012 10:02:41 -0600 Subject: Cfg: added creator handler to perform one-time creation of static data --- .../Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 2 +- src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py | 7 +- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 209 +++++++++++++++------ .../TestServer/TestPlugins/TestCfg/Test_init.py | 100 +++++++++- 4 files changed, 258 insertions(+), 60 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index 73550cd9d..7cce2b6f8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -101,7 +101,7 @@ class CfgGenshiGenerator(CfgGenerator): __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): - fname = entry.get('realname', entry.get('name')) + fname = entry.get('name') stream = \ self.template.generate(name=fname, metadata=metadata, diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py index 7277d5d08..5122d9aa1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py @@ -1,11 +1,8 @@ """ Handle info and :info files """ -import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Cfg import CfgInfo -LOGGER = logging.getLogger(__name__) - class CfgLegacyInfo(CfgInfo): """ CfgLegacyInfo handles :file:`info` and :file:`:info` files for @@ -37,8 +34,8 @@ class CfgLegacyInfo(CfgInfo): for line in open(self.path).readlines(): match = Bcfg2.Server.Plugin.INFO_REGEX.match(line) if not match: - LOGGER.warning("Failed to parse line in %s: %s" % - (event.filename, line)) + self.logger.warning("Failed to parse line in %s: %s" % + (event.filename, line)) continue else: for key, value in list(match.groupdict().items()): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index c8859dad3..2466d68a2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -4,19 +4,18 @@ import re import os import sys import stat -import logging +import errno import operator import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.Lint +from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 -from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, any, \ - oct_mode +from Bcfg2.Compat import u_str, unicode, b64encode, b64decode, walk_packages, \ + any, oct_mode # pylint: enable=W0622 -LOGGER = logging.getLogger(__name__) - #: SETUP contains a reference to the #: :class:`Bcfg2.Options.OptionParser` created by the Bcfg2 core for #: parsing command-line and config file options. @@ -29,7 +28,8 @@ LOGGER = logging.getLogger(__name__) SETUP = None -class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): +class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, + Bcfg2.Server.Plugin.Debuggable): """ .. currentmodule:: Bcfg2.Server.Plugins.Cfg CfgBaseFileMatcher is the parent class for all Cfg handler @@ -70,9 +70,13 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): #: Flag to indicate a deprecated handler. deprecated = False + #: Flag to indicate an experimental handler. + experimental = False + def __init__(self, name, specific, encoding): Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, encoding) + Bcfg2.Server.Plugin.Debuggable.__init__(self) self.encoding = encoding __init__.__doc__ = Bcfg2.Server.Plugin.SpecificData.__init__.__doc__ + \ """ @@ -183,7 +187,6 @@ class CfgGenerator(CfgBaseFileMatcher): :param metadata: The client metadata to generate data for. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: string - the contents of the entry - :raises: any """ return self.data @@ -290,11 +293,74 @@ class CfgVerifier(CfgBaseFileMatcher): :param data: The contents of the entry :type data: string :returns: None - :raises: Bcfg2.Server.Plugins.Cfg.CfgVerificationError + :raises: :exc:`Bcfg2.Server.Plugins.Cfg.CfgVerificationError` """ raise NotImplementedError +class CfgCreator(CfgBaseFileMatcher): + """ CfgCreator handlers create static entry data if no generator + was found to generate any. A CfgCreator runs at most once per + client, writes its data to disk as a static file, and is not + called on subsequent runs by the same client. """ + + def create_data(self, entry, metadata): + """ Create new data for the given entry and write it to disk + using :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.write_data`. + + :param entry: The entry to create data for. + :type entry: lxml.etree._Element + :param metadata: The client metadata to create data for. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: string - the contents of the entry + """ + raise NotImplementedError + + def write_data(self, data, host=None, group=None, prio=0): + """ Write the new data to disk. If ``host`` is given, it is + written as a host-specific file, or as a group-specific file + if ``group`` and ``prio`` are given. If neither ``host`` nor + ``group`` is given, it will be written as a non-specific file. + + :param data: The data to write + :type data: string + :param host: The data applies to the given host + :type host: bool + :param group: The data applies to the given group + :type group: string + :param prio: The data has the given priority relative to other + objects that also apply to the same group. + ``group`` must also be specified. + :type prio: int + :returns: None + :raises: :exc:`Bcfg2.Server.Plugins.Cfg.CfgCreationError` + """ + basefilename = \ + os.path.join(os.path.dirname(self.name), + os.path.basename(os.path.dirname(self.name))) + if group: + fileloc = "%s.G%02d_%s" % (basefilename, prio, group) + elif host: + fileloc = "%s.H_%s" % (basefilename, host) + else: + fileloc = basefilename + + self.debug_log("%s: Writing new file %s" % (self.name, fileloc)) + try: + os.makedirs(os.path.dirname(fileloc)) + except OSError: + err = sys.exc_info()[1] + if err.errno != errno.EEXIST: + raise CfgCreationError("Could not create parent directories " + "for %s: %s" % (fileloc, err)) + + try: + open(fileloc, 'wb').write(data) + except IOError: + err = sys.exc_info()[1] + raise CfgCreationError("Could not write %s: %s" % (fileloc, err)) + + class CfgVerificationError(Exception): """ Raised by :func:`Bcfg2.Server.Plugins.Cfg.CfgVerifier.verify_entry` when an @@ -302,6 +368,12 @@ class CfgVerificationError(Exception): pass +class CfgCreationError(Exception): + """ Raised by :class:`Bcfg2.Server.Plugins.Cfg.CfgCreator` when + various stages of data creation fail """ + pass + + class CfgDefaultInfo(CfgInfo): """ :class:`Bcfg2.Server.Plugins.Cfg.Cfg` handler that supplies a default set of file metadata """ @@ -323,13 +395,15 @@ class CfgDefaultInfo(CfgInfo): DEFAULT_INFO = CfgDefaultInfo(Bcfg2.Server.Plugin.DEFAULT_FILE_METADATA) -class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): +class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, + Bcfg2.Server.Plugin.Debuggable): """ Handle a collection of host- and group-specific Cfg files with multiple different Cfg handlers in a single directory. """ def __init__(self, basename, path, entry_type, encoding): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) + Bcfg2.Server.Plugin.Debuggable.__init__(self) self.specific = None self._handlers = None __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ @@ -347,8 +421,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, mname) hdlr = getattr(module, mname) - if set(hdlr.__mro__).intersection([CfgInfo, CfgFilter, - CfgGenerator, CfgVerifier]): + if CfgBaseFileMatcher in hdlr.__mro__: self._handlers.append(hdlr) self._handlers.sort(key=operator.attrgetter("__priority__")) return self._handlers @@ -374,16 +447,16 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if action == 'changed': # warn about a bogus 'changed' event, but # handle it like a 'created' - LOGGER.warning("Got %s event for unknown file %s" % - (action, event.filename)) + self.logger.warning("Got %s event for unknown file %s" + % (action, event.filename)) self.debug_log("%s handling %s event on %s" % (hdlr.__name__, action, event.filename)) try: self.entry_init(event, hdlr) except: # pylint: disable=W0702 err = sys.exc_info()[1] - LOGGER.error("Cfg: Failed to parse %s: %s" % - (event.filename, err)) + self.logger.error("Cfg: Failed to parse %s: %s" % + (event.filename, err)) return elif hdlr.ignore(event, basename=self.path): return @@ -394,8 +467,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): del self.entries[event.filename] return - LOGGER.error("Could not process event %s for %s; ignoring" % - (action, event.filename)) + self.logger.error("Could not process event %s for %s; ignoring" % + (action, event.filename)) def get_matching(self, metadata): return self.get_handlers(metadata, CfgGenerator) @@ -421,7 +494,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): specific=hdlr.get_regex([os.path.basename(self.path)])) else: if event.filename in self.entries: - LOGGER.warn("Got duplicate add for %s" % event.filename) + self.logger.warn("Got duplicate add for %s" % event.filename) else: fpath = os.path.join(self.path, event.filename) self.entries[event.filename] = hdlr(fpath) @@ -441,8 +514,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): msg = "Data for %s for %s failed to verify: %s" % \ (entry.get('name'), metadata.hostname, sys.exc_info()[1]) - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.logger.error(msg) + raise PluginExecutionError(msg) if entry.get('encoding') == 'base64': data = b64encode(data) @@ -453,16 +526,17 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) - LOGGER.error(msg) - LOGGER.error("Please verify you are using the proper encoding") - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.logger.error(msg) + self.logger.error("Please verify you are using the proper " + "encoding") + raise PluginExecutionError(msg) except ValueError: msg = "Error in specification for %s: %s" % (entry.get('name'), sys.exc_info()[1]) - LOGGER.error(msg) - LOGGER.error("You need to specify base64 encoding for %s" % - entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.logger.error(msg) + self.logger.error("You need to specify base64 encoding for %s" + % entry.get('name')) + raise PluginExecutionError(msg) except TypeError: # data is already unicode; newer versions of Cheetah # seem to return unicode @@ -489,13 +563,17 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if (isinstance(ent, handler_type) and (not ent.__specific__ or ent.specific.matches(metadata))): rv.append(ent) + + if ent.__basenames__: + fdesc = "/".join(ent.__basenames__) + elif ent.__extensions__: + fdesc = "." + "/.".join(ent.__extensions__) if ent.deprecated: - if ent.__basenames__: - fdesc = "/".join(ent.__basenames__) - elif ent.__extensions__: - fdesc = "." + "/.".join(ent.__extensions__) - LOGGER.warning("Cfg: %s: Use of %s files is deprecated" % - (ent.name, fdesc)) + self.logger.warning("Cfg: %s: Use of %s files is " + "deprecated" % (ent.name, fdesc)) + elif ent.experimental: + self.logger.warning("Cfg: %s: Use of %s files is " + "experimental" % (ent.name, fdesc)) return rv def bind_info_to_entry(self, entry, metadata): @@ -512,30 +590,55 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): info_handlers = self.get_handlers(metadata, CfgInfo) DEFAULT_INFO.bind_info_to_entry(entry, metadata) if len(info_handlers) > 1: - LOGGER.error("More than one info supplier found for %s: %s" % - (entry.get("name"), info_handlers)) + self.logger.error("More than one info supplier found for %s: %s" % + (entry.get("name"), info_handlers)) if len(info_handlers): info_handlers[0].bind_info_to_entry(entry, metadata) if entry.tag == 'Path': entry.set('type', 'file') + def _create_data(self, entry, metadata): + """ Create data for the given entry on the given client + + :param entry: The abstract entry to create data for. This + will not be modified + :type entry: lxml.etree._Element + :param metadata: The client metadata to create data for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: string - the data for the entry + """ + creator = self.best_matching(metadata, self.get_handlers(metadata, + CfgCreator)) + + try: + return creator.create_data(entry, metadata) + except: + raise PluginExecutionError("Cfg: Error creating data for %s: %s" % + (entry.get("name"), sys.exc_info()[1])) + def _generate_data(self, entry, metadata): """ Generate data for the given entry on the given client :param entry: The abstract entry to generate data for. This will not be modified :type entry: lxml.etree._Element - :param metadata: The client metadata generate data for + :param metadata: The client metadata to generate data for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: string - the data for the entry """ - generator = self.best_matching(metadata, - self.get_handlers(metadata, - CfgGenerator)) + try: + generator = self.best_matching(metadata, + self.get_handlers(metadata, + CfgGenerator)) + except PluginExecutionError: + # if no creators or generators exist, _create_data() + # raises an appropriate exception + return self._create_data(entry, metadata) + if entry.get('mode').lower() == 'inherit': # use on-disk permissions - LOGGER.warning("Cfg: %s: Use of mode='inherit' is deprecated" % - entry.get("name")) + self.logger.warning("Cfg: %s: Use of mode='inherit' is deprecated" + % entry.get("name")) fname = os.path.join(self.path, generator.name) entry.set('mode', oct_mode(stat.S_IMODE(os.stat(fname).st_mode))) @@ -544,8 +647,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): except: msg = "Cfg: Error rendering %s: %s" % (entry.get("name"), sys.exc_info()[1]) - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.logger.error(msg) + raise PluginExecutionError(msg) def _validate_data(self, entry, metadata, data): """ Validate data for the given entry on the given client @@ -578,8 +681,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): ent.specific.matches(metadata))] if not generators: msg = "No base file found for %s" % entry.get('name') - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.logger.error(msg) + raise PluginExecutionError(msg) rv = [] try: @@ -609,19 +712,19 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): name = self.build_filename(specific) if os.path.exists("%s.genshi" % name): msg = "Cfg: Unable to pull data for genshi types" - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.logger.error(msg) + raise PluginExecutionError(msg) elif os.path.exists("%s.cheetah" % name): msg = "Cfg: Unable to pull data for cheetah types" - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.logger.error(msg) + raise PluginExecutionError(msg) try: etext = new_entry['text'].encode(self.encoding) except: msg = "Cfg: Cannot encode content of %s as %s" % \ (name, self.encoding) - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.logger.error(msg) + raise PluginExecutionError(msg) open(name, 'w').write(etext) self.debug_log("Wrote file %s" % name, flag=log) badattr = [attr for attr in ['owner', 'group', 'mode'] @@ -631,8 +734,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): for ifile in ['info', ':info']: info = os.path.join(self.path, ifile) if os.path.exists(info): - LOGGER.info("Removing %s and replacing with info.xml" % - info) + self.logger.info("Removing %s and replacing with info.xml" + % info) os.remove(info) metadata_updates = {} metadata_updates.update(self.metadata) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py index d6313b073..6412480f0 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py @@ -1,5 +1,6 @@ import os import sys +import errno import lxml.etree import Bcfg2.Options from Bcfg2.Compat import walk_packages @@ -181,6 +182,64 @@ class TestCfgVerifier(TestCfgBaseFileMatcher): cf.verify_entry, Mock(), Mock(), Mock()) +class TestCfgCreator(TestCfgBaseFileMatcher): + test_obj = CfgCreator + path = "/foo/bar/test.txt" + + def test_create_data(self): + cc = self.get_obj() + self.assertRaises(NotImplementedError, + cc.create_data, Mock(), Mock()) + + @patch("os.makedirs") + @patch("%s.open" % builtins) + def test_write_data(self, mock_open, mock_makedirs): + cc = self.get_obj() + data = "test\ntest" + + def reset(): + mock_open.reset_mock() + mock_makedirs.reset_mock() + + # test writing non-specific file + cc.write_data(data) + mock_makedirs.assert_called_with("/foo/bar") + mock_open.assert_called_with("/foo/bar/bar", "wb") + mock_open.return_value.write.assert_called_with(data) + + # test writing group-specific file + reset() + cc.write_data(data, group="foogroup", prio=9) + mock_makedirs.assert_called_with("/foo/bar") + mock_open.assert_called_with("/foo/bar/bar.G09_foogroup", "wb") + mock_open.return_value.write.assert_called_with(data) + + # test writing host-specific file + reset() + cc.write_data(data, host="foo.example.com") + mock_makedirs.assert_called_with("/foo/bar") + mock_open.assert_called_with("/foo/bar/bar.H_foo.example.com", "wb") + mock_open.return_value.write.assert_called_with(data) + + # test already-exists error from makedirs + reset() + mock_makedirs.side_effect = OSError(errno.EEXIST, self.path) + cc.write_data(data) + mock_makedirs.assert_called_with("/foo/bar") + mock_open.assert_called_with("/foo/bar/bar", "wb") + mock_open.return_value.write.assert_called_with(data) + + # test error from open + reset() + mock_open.side_effect = IOError + self.assertRaises(CfgCreationError, cc.write_data, data) + + # test real error from makedirs + reset() + mock_makedirs.side_effect = OSError + self.assertRaises(CfgCreationError, cc.write_data, data) + + class TestCfgDefaultInfo(TestCfgInfo): test_obj = CfgDefaultInfo @@ -570,9 +629,36 @@ class TestCfgEntrySet(TestEntrySet): Bcfg2.Server.Plugins.Cfg.DEFAULT_INFO = default_info + def test_create_data(self): + eset = self.get_obj() + eset.best_matching = Mock() + creator = Mock() + creator.create_data.return_value = "data" + eset.best_matching.return_value = creator + eset.get_handlers = Mock() + entry = lxml.etree.Element("Path", name="/test.txt", mode="0640") + metadata = Mock() + + def reset(): + eset.best_matching.reset_mock() + eset.get_handlers.reset_mock() + + # test success + self.assertEqual(eset._create_data(entry, metadata), "data") + eset.get_handlers.assert_called_with(metadata, CfgCreator) + eset.best_matching.assert_called_with(metadata, + eset.get_handlers.return_value) + + # test failure to create data + reset() + creator.create_data.side_effect = OSError + self.assertRaises(PluginExecutionError, + eset._create_data, entry, metadata) + def test_generate_data(self): eset = self.get_obj() eset.best_matching = Mock() + eset._create_data = Mock() generator = Mock() generator.get_data.return_value = "data" eset.best_matching.return_value = generator @@ -583,7 +669,7 @@ class TestCfgEntrySet(TestEntrySet): def reset(): eset.best_matching.reset_mock() eset.get_handlers.reset_mock() - + eset._create_data.reset_mock() # test success self.assertEqual(eset._generate_data(entry, metadata), @@ -591,6 +677,7 @@ class TestCfgEntrySet(TestEntrySet): eset.get_handlers.assert_called_with(metadata, CfgGenerator) eset.best_matching.assert_called_with(metadata, eset.get_handlers.return_value) + self.assertFalse(eset._create_data.called) # test failure to generate data reset() @@ -598,6 +685,17 @@ class TestCfgEntrySet(TestEntrySet): self.assertRaises(PluginExecutionError, eset._generate_data, entry, metadata) + # test no generator found + reset() + eset.best_matching.side_effect = PluginExecutionError + self.assertEqual(eset._generate_data(entry, metadata), + eset._create_data.return_value) + eset.get_handlers.assert_called_with(metadata, CfgGenerator) + eset.best_matching.assert_called_with(metadata, + eset.get_handlers.return_value) + eset._create_data.assert_called_with(entry, metadata) + + def test_validate_data(self): class MockChild1(Mock): pass -- cgit v1.2.3-1-g7c22