diff options
Diffstat (limited to 'src/lib/Bcfg2/Server')
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Minestruct.py | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Pull.py | 10 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/BuiltinCore.py | 14 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Core.py | 217 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Validate.py | 28 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin/helpers.py | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Git.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/GroupLogic.py | 47 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 8 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Probes.py | 28 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Reporting.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/SSHbase.py | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/SSLCA.py | 2 |
18 files changed, 244 insertions, 139 deletions
diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index 13c0563ec..93e42305c 100644 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -3,6 +3,7 @@ import getopt import lxml.etree import sys import Bcfg2.Server.Admin +from Bcfg2.Server.Plugin import PullSource class Minestruct(Bcfg2.Server.Admin.StructureMode): @@ -39,7 +40,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): try: extra = set() - for source in self.bcore.pull_sources: + for source in self.bcore.plugins_by_type(PullSource): for item in source.GetExtra(client): extra.add(item) except: diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index 9f1b3d138..8001425df 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -6,6 +6,7 @@ import sys import getopt import select import Bcfg2.Server.Admin +from Bcfg2.Server.Plugin import PullSource, Generator from Bcfg2.Compat import input # pylint: disable=W0622 @@ -62,13 +63,14 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): given client/entry from statistics. """ new_entry = {'type': etype, 'name': ename} - for plugin in self.bcore.pull_sources: + pull_sources = self.bcore.plugins_by_type(PullSource) + for plugin in pull_sources: try: (owner, group, mode, contents) = \ plugin.GetCurrentEntry(client, etype, ename) break except Bcfg2.Server.Plugin.PluginExecutionError: - if plugin == self.bcore.pull_sources[-1]: + if plugin == pull_sources[-1]: print("Pull Source failure; could not fetch current state") raise SystemExit(1) @@ -121,8 +123,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): meta = self.bcore.build_metadata(client) # Find appropriate plugin in bcore - glist = [gen for gen in self.bcore.generators if - ename in gen.Entries.get(etype, {})] + glist = [gen for gen in self.bcore.plugins_by_type(Generator) + if ename in gen.Entries.get(etype, {})] if len(glist) != 1: self.errExit("Got wrong numbers of matching generators for entry:" "%s" % ([g.name for g in glist])) diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 4d7453840..c3302f1d0 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -9,12 +9,12 @@ from Bcfg2.Server.Core import BaseCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse from Bcfg2.SSLServer import XMLRPCServer -from lockfile import LockFailed +from lockfile import LockFailed, LockTimeout # pylint: disable=E0611 try: - from daemon.pidfile import PIDLockFile + from daemon.pidfile import TimeoutPIDLockFile except ImportError: - from daemon.pidlockfile import PIDLockFile + from daemon.pidlockfile import TimeoutPIDLockFile # pylint: enable=E0611 @@ -33,7 +33,8 @@ class Core(BaseCore): gid=self.setup['daemon_gid'], umask=int(self.setup['umask'], 8)) if self.setup['daemon']: - daemon_args['pidfile'] = PIDLockFile(self.setup['daemon']) + daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'], + acquire_timeout=5) #: The :class:`daemon.DaemonContext` used to drop #: privileges, write the PID file (with :class:`PidFile`), #: and daemonize this core. @@ -89,6 +90,11 @@ class Core(BaseCore): err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False + except LockTimeout: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: Failed to acquire lock " + "on %s" % (self.name, self.setup['daemon'])) + return False def _run(self): """ Create :attr:`server` to start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index deb9065a5..ab8cda3da 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -19,8 +19,9 @@ from Bcfg2.Cache import Cache import Bcfg2.Statistics from itertools import chain from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 -from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError, \ - track_statistics +from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 +from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 +from Bcfg2.Server.Plugin import track_statistics try: import psyco @@ -96,6 +97,7 @@ class BaseCore(object): .. automethod:: _block .. ----- .. automethod:: _file_monitor_thread + .. automethod:: _perflog_thread """ #: The Bcfg2 repository directory self.datastore = setup['repo'] @@ -174,6 +176,9 @@ class BaseCore(object): #: the first one loaded wins. self.plugin_blacklist = {} + #: The Metadata plugin + self.metadata = None + #: Revision of the Bcfg2 specification. This will be sent to #: the client in the configuration, and can be set by a #: :class:`Bcfg2.Server.Plugin.interfaces.Version` plugin. @@ -235,71 +240,6 @@ class BaseCore(object): self.logger.error("Failed to set ownership of database " "at %s: %s" % (db_settings['NAME'], err)) - if '' in setup['plugins']: - setup['plugins'].remove('') - - for plugin in setup['plugins']: - if not plugin in self.plugins: - self.init_plugin(plugin) - # Remove blacklisted plugins - for plugin, blacklist in list(self.plugin_blacklist.items()): - if len(blacklist) > 0: - self.logger.error("The following plugins conflict with %s;" - "Unloading %s" % (plugin, blacklist)) - for plug in blacklist: - del self.plugins[plug] - - # Log experimental plugins - expl = [plug for plug in list(self.plugins.values()) - if plug.experimental] - if expl: - self.logger.info("Loading experimental plugin(s): %s" % - (" ".join([x.name for x in expl]))) - self.logger.info("NOTE: Interfaces subject to change") - - # Log deprecated plugins - depr = [plug for plug in list(self.plugins.values()) - if plug.deprecated] - if depr: - self.logger.info("Loading deprecated plugin(s): %s" % - (" ".join([x.name for x in depr]))) - - # Find the metadata plugin and set self.metadata - mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata) - if len(mlist) >= 1: - #: The Metadata plugin - self.metadata = mlist[0] - if len(mlist) > 1: - self.logger.error("Multiple Metadata plugins loaded; " - "using %s" % self.metadata) - else: - self.logger.error("No Metadata plugin loaded; " - "failed to instantiate Core") - raise CoreInitError("No Metadata Plugin") - - #: The list of plugins that handle - #: :class:`Bcfg2.Server.Plugin.interfaces.Statistics` - self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics) - - #: The list of plugins that implement the - #: :class:`Bcfg2.Server.Plugin.interfaces.PullSource` - #: interface - self.pull_sources = \ - self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) - - #: The list of - #: :class:`Bcfg2.Server.Plugin.interfaces.Generator` plugins - self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator) - - #: The list of plugins that handle - #: :class:`Bcfg2.Server.Plugin.interfaces.Structure` - #: generation - self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure) - - #: The list of plugins that implement the - #: :class:`Bcfg2.Server.Plugin.interfaces.Connector` interface - self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector) - #: The CA that signed the server cert self.ca = setup['ca'] @@ -317,6 +257,12 @@ class BaseCore(object): threading.Thread(name="%sFAMThread" % setup['filemonitor'], target=self._file_monitor_thread) + self.perflog_thread = None + if self.setup['perflog']: + self.perflog_thread = \ + threading.Thread(name="PerformanceLoggingThread", + target=self._perflog_thread) + #: A :func:`threading.Lock` for use by #: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set` self.lock = threading.Lock() @@ -325,10 +271,6 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() - if self.debug_flag: - # enable debugging on everything else. - self.plugins[plugin].set_debug(self.debug_flag) - def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -349,11 +291,23 @@ class BaseCore(object): if isinstance(plugin, base_cls)], key=lambda p: (p.sort_order, p.name)) + def _perflog_thread(self): + """ The thread that periodically logs performance statistics + to syslog. """ + self.logger.debug("Performance logging thread starting") + while not self.terminate.isSet(): + self.terminate.wait(self.setup['perflog_interval']) + for name, stats in self.get_statistics(None).items(): + self.logger.info("Performance statistics: " + "%s min=%.06f, max=%.06f, average=%.06f, " + "count=%d" % ((name, ) + stats)) + def _file_monitor_thread(self): """ The thread that runs the :class:`Bcfg2.Server.FileMonitor.FileMonitor`. This also queries :class:`Bcfg2.Server.Plugin.interfaces.Version` plugins for the current revision of the Bcfg2 repo. """ + self.logger.debug("File monitor thread starting") famfd = self.fam.fileno() terminate = self.terminate while not terminate.isSet(): @@ -372,7 +326,7 @@ class BaseCore(object): def _update_vcs_revision(self): """ Update the revision of the current configuration on-disk from the VCS plugin """ - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version): + for plugin in self.plugins_by_type(Version): try: newrev = plugin.get_revision() if newrev != self.revision: @@ -384,6 +338,58 @@ class BaseCore(object): (plugin.name, sys.exc_info()[1])) self.revision = '-1' + def load_plugins(self): + """ Load all plugins, setting + :attr:`Bcfg2.Server.Core.BaseCore.plugins` and + :attr:`Bcfg2.Server.Core.BaseCore.metadata` as side effects. + This does not start plugin threads; that is done later, in + :func:`Bcfg2.Server.Core.BaseCore.run` """ + while '' in self.setup['plugins']: + self.setup['plugins'].remove('') + + for plugin in self.setup['plugins']: + if not plugin in self.plugins: + self.init_plugin(plugin) + + # Remove blacklisted plugins + for plugin, blacklist in list(self.plugin_blacklist.items()): + if len(blacklist) > 0: + self.logger.error("The following plugins conflict with %s;" + "Unloading %s" % (plugin, blacklist)) + for plug in blacklist: + del self.plugins[plug] + + # Log experimental plugins + expl = [plug for plug in list(self.plugins.values()) + if plug.experimental] + if expl: + self.logger.info("Loading experimental plugin(s): %s" % + (" ".join([x.name for x in expl]))) + self.logger.info("NOTE: Interfaces subject to change") + + # Log deprecated plugins + depr = [plug for plug in list(self.plugins.values()) + if plug.deprecated] + if depr: + self.logger.info("Loading deprecated plugin(s): %s" % + (" ".join([x.name for x in depr]))) + + # Find the metadata plugin and set self.metadata + mlist = self.plugins_by_type(Metadata) + if len(mlist) >= 1: + self.metadata = mlist[0] + if len(mlist) > 1: + self.logger.error("Multiple Metadata plugins loaded; using %s" + % self.metadata) + else: + self.logger.error("No Metadata plugin loaded; " + "failed to instantiate Core") + raise CoreInitError("No Metadata Plugin") + + if self.debug_flag: + # enable debugging on plugins + self.plugins[plugin].set_debug(self.debug_flag) + def init_plugin(self, plugin): """ Import and instantiate a single plugin. The plugin is stored to :attr:`plugins`. @@ -468,8 +474,7 @@ class BaseCore(object): metadata.hostname)) start = time.time() try: - for plugin in \ - self.plugins_by_type(Bcfg2.Server.Plugin.ClientRunHooks): + for plugin in self.plugins_by_type(ClientRunHooks): try: getattr(plugin, hook)(metadata) except AttributeError: @@ -500,11 +505,10 @@ class BaseCore(object): :type data: list of lxml.etree._Element objects """ self.logger.debug("Validating structures for %s" % metadata.hostname) - for plugin in \ - self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): + for plugin in self.plugins_by_type(StructureValidator): try: plugin.validate_structures(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: + except ValidationError: err = sys.exc_info()[1] self.logger.error("Plugin %s structure validation failed: %s" % (plugin.name, err)) @@ -527,10 +531,10 @@ class BaseCore(object): :type data: list of lxml.etree._Element objects """ self.logger.debug("Validating goals for %s" % metadata.hostname) - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): + for plugin in self.plugins_by_type(GoalValidator): try: plugin.validate_goals(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: + except ValidationError: err = sys.exc_info()[1] self.logger.error("Plugin %s goal validation failed: %s" % (plugin.name, err.message)) @@ -548,8 +552,9 @@ class BaseCore(object): :returns: list of :class:`lxml.etree._Element` objects """ self.logger.debug("Getting structures for %s" % metadata.hostname) - structures = list(chain(*[struct.BuildStructures(metadata) - for struct in self.structures])) + structures = list( + chain(*[struct.BuildStructures(metadata) + for struct in self.plugins_by_type(Structure)])) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] missing = [b for b in metadata.bundles if b not in sbundles] if missing: @@ -634,8 +639,9 @@ class BaseCore(object): self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) - glist = [gen for gen in self.generators if - entry.get('name') in gen.Entries.get(entry.tag, {})] + generators = self.plugins_by_type(Generator) + glist = [gen for gen in generators + if entry.get('name') in gen.Entries.get(entry.tag, {})] if len(glist) == 1: return glist[0].Entries[entry.tag][entry.get('name')](entry, metadata) @@ -643,8 +649,8 @@ class BaseCore(object): generators = ", ".join([gen.name for gen in glist]) self.logger.error("%s %s served by multiple generators: %s" % (entry.tag, entry.get('name'), generators)) - g2list = [gen for gen in self.generators if - gen.HandlesEntry(entry, metadata)] + g2list = [gen for gen in generators + if gen.HandlesEntry(entry, metadata)] try: if len(g2list) == 1: return g2list[0].HandleEntry(entry, metadata) @@ -671,7 +677,7 @@ class BaseCore(object): revision=self.revision) try: meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: self.logger.error("Metadata consistency error for client %s" % client) return lxml.etree.Element("error", type='metadata error') @@ -718,7 +724,8 @@ class BaseCore(object): :type event: Bcfg2.Server.FileMonitor.Event """ if event.filename != self.cfile: - print("Got event for unknown file: %s" % event.filename) + self.logger.error("Got event for unknown file: %s" % + event.filename) return if event.code2str() == 'deleted': return @@ -755,16 +762,25 @@ class BaseCore(object): return False try: + self.load_plugins() + self.fam.start() self.fam_thread.start() self.fam.AddMonitor(self.cfile, self) + if self.perflog_thread is not None: + self.perflog_thread.start() - for plug in self.plugins_by_type(Bcfg2.Server.Plugin.Threaded): + for plug in self.plugins_by_type(Threaded): plug.start_threads() except: self.shutdown() raise + if self.setup['fam_blocking']: + time.sleep(1) + while self.fam.pending() != 0: + time.sleep(1) + self.set_debug(None, self.debug_flag) self._block() @@ -796,7 +812,7 @@ class BaseCore(object): """ self.logger.debug("Getting decision list for %s" % metadata.hostname) result = [] - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): + for plugin in self.plugins_by_type(Decision): try: result.extend(plugin.GetDecisions(metadata, mode)) except: @@ -815,7 +831,7 @@ class BaseCore(object): """ if not hasattr(self, 'metadata'): # some threads start before metadata is even loaded - raise Bcfg2.Server.Plugin.MetadataRuntimeError + raise MetadataRuntimeError("Metadata not loaded yet") if self.metadata_cache_mode == 'initial': # the Metadata plugin handles loading the cached data if # we're only caching the initial metadata object @@ -825,10 +841,11 @@ class BaseCore(object): if not imd: self.logger.debug("Building metadata for %s" % client_name) imd = self.metadata.get_initial_metadata(client_name) - for conn in self.connectors: + connectors = self.plugins_by_type(Connector) + for conn in connectors: grps = conn.get_additional_groups(imd) self.metadata.merge_additional_groups(imd, grps) - for conn in self.connectors: + for conn in connectors: data = conn.get_additional_data(imd) self.metadata.merge_additional_data(imd, conn.name, data) imd.query.by_name = self.build_metadata @@ -849,7 +866,7 @@ class BaseCore(object): meta = self.build_metadata(client_name) state = statistics.find(".//Statistics") if state.get('version') >= '2.0': - for plugin in self.statistics: + for plugin in self.plugins_by_type(Statistics): try: plugin.process_statistics(meta, statistics) except: @@ -891,11 +908,11 @@ class BaseCore(object): meta = self.build_metadata(client) else: meta = None - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: err = sys.exc_info()[1] self.critical_error("Client metadata resolution error for %s: %s" % (address[0], err)) - except Bcfg2.Server.Plugin.MetadataRuntimeError: + except MetadataRuntimeError: err = sys.exc_info()[1] self.critical_error('Metadata system runtime failure for %s: %s' % (address[0], err)) @@ -989,8 +1006,7 @@ class BaseCore(object): version)) try: self.metadata.set_version(client, version) - except (Bcfg2.Server.Plugin.MetadataConsistencyError, - Bcfg2.Server.Plugin.MetadataRuntimeError): + except (MetadataConsistencyError, MetadataRuntimeError): err = sys.exc_info()[1] self.critical_error("Unable to set version for %s: %s" % (client, err)) @@ -1010,7 +1026,7 @@ class BaseCore(object): client, metadata = self.resolve_client(address, cleanup_cache=True) self.logger.debug("Getting probes for %s" % client) try: - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): + for plugin in self.plugins_by_type(Probing): for probe in plugin.GetProbes(metadata): resp.append(probe) return lxml.etree.tostring(resp, @@ -1080,8 +1096,7 @@ class BaseCore(object): self.logger.debug("%s sets its profile to %s" % (client, profile)) try: self.metadata.set_profile(client, profile, address) - except (Bcfg2.Server.Plugin.MetadataConsistencyError, - Bcfg2.Server.Plugin.MetadataRuntimeError): + except (MetadataConsistencyError, MetadataRuntimeError): err = sys.exc_info()[1] self.critical_error("Unable to assert profile for %s: %s" % (client, err)) @@ -1103,7 +1118,7 @@ class BaseCore(object): config = self.BuildConfiguration(client) return lxml.etree.tostring(config, xml_declaration=False).decode('UTF-8') - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: self.critical_error("Metadata consistency failure for %s" % client) @exposed diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 37bc230d1..ae7c75804 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -40,7 +40,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", "SSLCA/**/cert.xml": "sslca-cert.xsd", - "SSLCA/**/key.xml": "sslca-key.xsd" + "SSLCA/**/key.xml": "sslca-key.xsd", + "GroupLogic/groups.xml": "grouplogic.xsd" } self.filelists = {} @@ -83,17 +84,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): else: self.LintError("properties-schema-not-found", "No schema found for %s" % filename) + # ensure that it at least parses + self.parse(filename) - def validate(self, filename, schemafile, schema=None): - """validate a file against the given lxml.etree.Schema. - return True on success, False on failure """ - if schema is None: - # if no schema object was provided, instantiate one - schema = self._load_schema(schemafile) - if not schema: - return False + def parse(self, filename): + """ Parse an XML file, raising the appropriate LintErrors if + it can't be parsed or read. Return the + lxml.etree._ElementTree parsed from the file. """ try: - datafile = lxml.etree.parse(filename) + return lxml.etree.parse(filename) except SyntaxError: lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) self.LintError("xml-failed-to-parse", @@ -106,6 +105,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Failed to open file %s" % filename) return False + def validate(self, filename, schemafile, schema=None): + """validate a file against the given lxml.etree.Schema. + return True on success, False on failure """ + if schema is None: + # if no schema object was provided, instantiate one + schema = self._load_schema(schemafile) + if not schema: + return False + datafile = self.parse(filename) if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 2b878d7e2..b14968d77 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -530,8 +530,8 @@ class XMLFileBacked(FileBacked): #: XInclude. self.extra_monitors = [] - if ((create or (self.create is not None and self.create)) - and not os.path.exists(self.name)): + if ((create is not None or self.create not in [None, False]) and + not os.path.exists(self.name)): toptag = create or self.create self.logger.warning("%s does not exist, creating" % self.name) if hasattr(toptag, "getroottree"): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index 581a997d8..c7b62f352 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -69,7 +69,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): the given client metadata, and may be obtained by doing ``self.XMLMatch(metadata)`` :type spec: lxml.etree._Element - :returns: None + :returns: string - The filename of the private key """ if spec is None: spec = self.XMLMatch(metadata) @@ -140,7 +140,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if spec is None: spec = self.XMLMatch(metadata) category = spec.get("category", self.category) - print("category=%s" % category) if category is None: per_host_default = "true" else: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 926172e57..ffe93c25b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -599,6 +599,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, else: try: if not isinstance(data, unicode): + if not isinstance(data, str): + data = data.decode('utf-8') data = u_str(data, self.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index c8362db41..44971aba7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -44,7 +44,7 @@ class Git(Version): else: cmd = ["git", "--git-dir", self.vcs_path, "--work-tree", self.vcs_root, "rev-parse", "HEAD"] - self.debug_log("Git: Running cmd") + self.debug_log("Git: Running %s" % cmd) proc = Popen(cmd, stdout=PIPE, stderr=PIPE) rv, err = proc.communicate() if proc.wait(): diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py new file mode 100644 index 000000000..810b273af --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -0,0 +1,47 @@ +""" GroupLogic is a connector plugin that lets you use an XML Genshi +template to dynamically set additional groups for clients. """ + +import os +import lxml.etree +import Bcfg2.Server.Plugin +try: + from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile +except ImportError: + # BundleTemplateFile missing means that genshi is missing. we + # import genshi to get the _real_ error + import genshi # pylint: disable=W0611 + + +class GroupLogicConfig(BundleTemplateFile): + """ Representation of the GroupLogic groups.xml file """ + create = lxml.etree.Element("GroupLogic", + nsmap=dict(py="http://genshi.edgewall.org/")) + + def __init__(self, name, fam): + BundleTemplateFile.__init__(self, name, + Bcfg2.Server.Plugin.Specificity(), None) + self.fam = fam + self.should_monitor = True + self.fam.AddMonitor(self.name, self) + + def _match(self, item, metadata): + if item.tag == 'Group' and not len(item.getchildren()): + return [item] + return BundleTemplateFile._match(self, item, metadata) + + +class GroupLogic(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + """ GroupLogic is a connector plugin that lets you use an XML + Genshi template to dynamically set additional groups for + clients. """ + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), + core.fam) + + def get_additional_groups(self, metadata): + return [el.get("name") + for el in self.config.get_xml_value(metadata).findall("Group")] diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index bdf3b87fe..71e81c1fe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -945,7 +945,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Client %s set as nonexistent group %s" % (client, group)) - def set_profile(self, client, profile, addresspair): + def set_profile(self, client, profile, # pylint: disable=W0221 + addresspair, require_public=True): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) @@ -957,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] - if not group.is_public: + if require_public and not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) @@ -1128,7 +1129,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = self.default if pgroup: - self.set_profile(client, pgroup, (None, None)) + self.set_profile(client, pgroup, (None, None), + require_public=False) profile = _add_group(pgroup) else: msg = "Cannot add new client %s; no default group set" % client diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 4eefd0722..bc2928fa6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -93,6 +93,8 @@ class AptSource(Source): self.logger.error("Packages: Failed to read file %s" % fname) raise for line in reader.readlines(): + if not isinstance(line, str): + line = line.decode('utf-8') words = str(line.strip()).split(':', 1) if words[0] == 'Package': pkgname = words[1].strip().rstrip() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index b4d481459..7ba374dd3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -315,7 +315,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :raises: OSError - If the saved data cannot be read :raises: cPickle.UnpicklingError - If the saved data is corrupt """ - data = open(self.cachefile) + data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, self.essentialpkgs) = cPickle.load(data) @@ -615,7 +615,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 self.logger.info("Packages: Updating %s" % url) fname = self.escape_url(url) try: - open(fname, 'w').write(fetch_url(url)) + open(fname, 'wb').write(fetch_url(url)) except ValueError: self.logger.error("Packages: Bad url string %s" % url) raise diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 4f163a1ab..efbca28cd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -18,7 +18,8 @@ from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" #: The default path for generated apt configs -APT_CONFIG_DEFAULT = "/etc/apt/sources.d/bcfg2" +APT_CONFIG_DEFAULT = \ + "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" class Packages(Bcfg2.Server.Plugin.Plugin, diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index a8001d9af..309b96475 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -63,7 +63,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data) + return str.__new__(cls, data.encode('utf-8')) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -153,7 +153,20 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): probe = lxml.etree.Element('probe') probe.set('name', os.path.basename(name)) probe.set('source', self.plugin_name) - probe.text = entry.data + if (metadata.version_info and + metadata.version_info > (1, 3, 1, '', 0)): + try: + probe.text = entry.data.decode('utf-8') + except AttributeError: + probe.text = entry.data + else: + try: + probe.text = entry.data + except: # pylint: disable=W0702 + self.logger.error("Client unable to handle unicode " + "probes. Skipping %s" % + probe.get('name')) + continue match = self.bangline.match(entry.data.split('\n')[0]) if match: probe.set('interpreter', match.group('interpreter')) @@ -209,8 +222,15 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - lxml.etree.SubElement(ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str( + self.probedata[client][probe]).decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 3bd6fd14f..3354763d4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -96,7 +96,7 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): client.hostname, cdata, lxml.etree.tostring( stats, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False)) self.debug_log("%s: Queued statistics data for %s" % (self.__class__.__name__, client.hostname)) return diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index fc07a90e9..5aa7c4d9e 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -172,7 +172,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for name in names[cmeta.hostname]: newnames.add(name.split('.')[0]) try: - newips.add(self.get_ipcache_entry(name)[0]) + newips.update(self.get_ipcache_entry(name)[0]) except: # pylint: disable=W0702 continue names[cmeta.hostname].update(newnames) @@ -288,7 +288,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: # need to add entry try: - ipaddr = socket.gethostbyname(client) + ipaddr = set([addr[0] for (_, _, _, _, addr) in socket.getaddrinfo(client, None)]) self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index 7d00201da..f111ffc60 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -68,7 +68,7 @@ class SSLCACertSpec(SSLCAXMLSpec): def get_spec(self, metadata): rv = SSLCAXMLSpec.get_spec(self, metadata) rv['subjectaltname'] = [e.text for e in self.Match(metadata) - if e.tag == "SubjectAltName"] + if e.tag == "subjectAltName"] return rv |