summaryrefslogtreecommitdiffstats
path: root/src/lib/Bcfg2/Server
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/Bcfg2/Server')
-rw-r--r--src/lib/Bcfg2/Server/Admin/Minestruct.py3
-rw-r--r--src/lib/Bcfg2/Server/Admin/Pull.py10
-rw-r--r--src/lib/Bcfg2/Server/BuiltinCore.py14
-rw-r--r--src/lib/Bcfg2/Server/Core.py217
-rw-r--r--src/lib/Bcfg2/Server/Lint/Validate.py28
-rw-r--r--src/lib/Bcfg2/Server/Plugin/helpers.py4
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py3
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Git.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/GroupLogic.py47
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Metadata.py8
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Apt.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Source.py4
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/__init__.py3
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Probes.py28
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Reporting.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/SSHbase.py4
-rw-r--r--src/lib/Bcfg2/Server/Plugins/SSLCA.py2
18 files changed, 244 insertions, 139 deletions
diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py
index 13c0563ec..93e42305c 100644
--- a/src/lib/Bcfg2/Server/Admin/Minestruct.py
+++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py
@@ -3,6 +3,7 @@ import getopt
import lxml.etree
import sys
import Bcfg2.Server.Admin
+from Bcfg2.Server.Plugin import PullSource
class Minestruct(Bcfg2.Server.Admin.StructureMode):
@@ -39,7 +40,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode):
try:
extra = set()
- for source in self.bcore.pull_sources:
+ for source in self.bcore.plugins_by_type(PullSource):
for item in source.GetExtra(client):
extra.add(item)
except:
diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py
index 9f1b3d138..8001425df 100644
--- a/src/lib/Bcfg2/Server/Admin/Pull.py
+++ b/src/lib/Bcfg2/Server/Admin/Pull.py
@@ -6,6 +6,7 @@ import sys
import getopt
import select
import Bcfg2.Server.Admin
+from Bcfg2.Server.Plugin import PullSource, Generator
from Bcfg2.Compat import input # pylint: disable=W0622
@@ -62,13 +63,14 @@ class Pull(Bcfg2.Server.Admin.MetadataCore):
given client/entry from statistics.
"""
new_entry = {'type': etype, 'name': ename}
- for plugin in self.bcore.pull_sources:
+ pull_sources = self.bcore.plugins_by_type(PullSource)
+ for plugin in pull_sources:
try:
(owner, group, mode, contents) = \
plugin.GetCurrentEntry(client, etype, ename)
break
except Bcfg2.Server.Plugin.PluginExecutionError:
- if plugin == self.bcore.pull_sources[-1]:
+ if plugin == pull_sources[-1]:
print("Pull Source failure; could not fetch current state")
raise SystemExit(1)
@@ -121,8 +123,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore):
meta = self.bcore.build_metadata(client)
# Find appropriate plugin in bcore
- glist = [gen for gen in self.bcore.generators if
- ename in gen.Entries.get(etype, {})]
+ glist = [gen for gen in self.bcore.plugins_by_type(Generator)
+ if ename in gen.Entries.get(etype, {})]
if len(glist) != 1:
self.errExit("Got wrong numbers of matching generators for entry:"
"%s" % ([g.name for g in glist]))
diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py
index 4d7453840..c3302f1d0 100644
--- a/src/lib/Bcfg2/Server/BuiltinCore.py
+++ b/src/lib/Bcfg2/Server/BuiltinCore.py
@@ -9,12 +9,12 @@ from Bcfg2.Server.Core import BaseCore, NoExposedMethod
from Bcfg2.Compat import xmlrpclib, urlparse
from Bcfg2.SSLServer import XMLRPCServer
-from lockfile import LockFailed
+from lockfile import LockFailed, LockTimeout
# pylint: disable=E0611
try:
- from daemon.pidfile import PIDLockFile
+ from daemon.pidfile import TimeoutPIDLockFile
except ImportError:
- from daemon.pidlockfile import PIDLockFile
+ from daemon.pidlockfile import TimeoutPIDLockFile
# pylint: enable=E0611
@@ -33,7 +33,8 @@ class Core(BaseCore):
gid=self.setup['daemon_gid'],
umask=int(self.setup['umask'], 8))
if self.setup['daemon']:
- daemon_args['pidfile'] = PIDLockFile(self.setup['daemon'])
+ daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'],
+ acquire_timeout=5)
#: The :class:`daemon.DaemonContext` used to drop
#: privileges, write the PID file (with :class:`PidFile`),
#: and daemonize this core.
@@ -89,6 +90,11 @@ class Core(BaseCore):
err = sys.exc_info()[1]
self.logger.error("Failed to daemonize %s: %s" % (self.name, err))
return False
+ except LockTimeout:
+ err = sys.exc_info()[1]
+ self.logger.error("Failed to daemonize %s: Failed to acquire lock "
+ "on %s" % (self.name, self.setup['daemon']))
+ return False
def _run(self):
""" Create :attr:`server` to start the server listening. """
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index deb9065a5..ab8cda3da 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -19,8 +19,9 @@ from Bcfg2.Cache import Cache
import Bcfg2.Statistics
from itertools import chain
from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622
-from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError, \
- track_statistics
+from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614
+from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614
+from Bcfg2.Server.Plugin import track_statistics
try:
import psyco
@@ -96,6 +97,7 @@ class BaseCore(object):
.. automethod:: _block
.. -----
.. automethod:: _file_monitor_thread
+ .. automethod:: _perflog_thread
"""
#: The Bcfg2 repository directory
self.datastore = setup['repo']
@@ -174,6 +176,9 @@ class BaseCore(object):
#: the first one loaded wins.
self.plugin_blacklist = {}
+ #: The Metadata plugin
+ self.metadata = None
+
#: Revision of the Bcfg2 specification. This will be sent to
#: the client in the configuration, and can be set by a
#: :class:`Bcfg2.Server.Plugin.interfaces.Version` plugin.
@@ -235,71 +240,6 @@ class BaseCore(object):
self.logger.error("Failed to set ownership of database "
"at %s: %s" % (db_settings['NAME'], err))
- if '' in setup['plugins']:
- setup['plugins'].remove('')
-
- for plugin in setup['plugins']:
- if not plugin in self.plugins:
- self.init_plugin(plugin)
- # Remove blacklisted plugins
- for plugin, blacklist in list(self.plugin_blacklist.items()):
- if len(blacklist) > 0:
- self.logger.error("The following plugins conflict with %s;"
- "Unloading %s" % (plugin, blacklist))
- for plug in blacklist:
- del self.plugins[plug]
-
- # Log experimental plugins
- expl = [plug for plug in list(self.plugins.values())
- if plug.experimental]
- if expl:
- self.logger.info("Loading experimental plugin(s): %s" %
- (" ".join([x.name for x in expl])))
- self.logger.info("NOTE: Interfaces subject to change")
-
- # Log deprecated plugins
- depr = [plug for plug in list(self.plugins.values())
- if plug.deprecated]
- if depr:
- self.logger.info("Loading deprecated plugin(s): %s" %
- (" ".join([x.name for x in depr])))
-
- # Find the metadata plugin and set self.metadata
- mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata)
- if len(mlist) >= 1:
- #: The Metadata plugin
- self.metadata = mlist[0]
- if len(mlist) > 1:
- self.logger.error("Multiple Metadata plugins loaded; "
- "using %s" % self.metadata)
- else:
- self.logger.error("No Metadata plugin loaded; "
- "failed to instantiate Core")
- raise CoreInitError("No Metadata Plugin")
-
- #: The list of plugins that handle
- #: :class:`Bcfg2.Server.Plugin.interfaces.Statistics`
- self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics)
-
- #: The list of plugins that implement the
- #: :class:`Bcfg2.Server.Plugin.interfaces.PullSource`
- #: interface
- self.pull_sources = \
- self.plugins_by_type(Bcfg2.Server.Plugin.PullSource)
-
- #: The list of
- #: :class:`Bcfg2.Server.Plugin.interfaces.Generator` plugins
- self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator)
-
- #: The list of plugins that handle
- #: :class:`Bcfg2.Server.Plugin.interfaces.Structure`
- #: generation
- self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure)
-
- #: The list of plugins that implement the
- #: :class:`Bcfg2.Server.Plugin.interfaces.Connector` interface
- self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector)
-
#: The CA that signed the server cert
self.ca = setup['ca']
@@ -317,6 +257,12 @@ class BaseCore(object):
threading.Thread(name="%sFAMThread" % setup['filemonitor'],
target=self._file_monitor_thread)
+ self.perflog_thread = None
+ if self.setup['perflog']:
+ self.perflog_thread = \
+ threading.Thread(name="PerformanceLoggingThread",
+ target=self._perflog_thread)
+
#: A :func:`threading.Lock` for use by
#: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set`
self.lock = threading.Lock()
@@ -325,10 +271,6 @@ class BaseCore(object):
#: metadata
self.metadata_cache = Cache()
- if self.debug_flag:
- # enable debugging on everything else.
- self.plugins[plugin].set_debug(self.debug_flag)
-
def plugins_by_type(self, base_cls):
""" Return a list of loaded plugins that match the passed type.
@@ -349,11 +291,23 @@ class BaseCore(object):
if isinstance(plugin, base_cls)],
key=lambda p: (p.sort_order, p.name))
+ def _perflog_thread(self):
+ """ The thread that periodically logs performance statistics
+ to syslog. """
+ self.logger.debug("Performance logging thread starting")
+ while not self.terminate.isSet():
+ self.terminate.wait(self.setup['perflog_interval'])
+ for name, stats in self.get_statistics(None).items():
+ self.logger.info("Performance statistics: "
+ "%s min=%.06f, max=%.06f, average=%.06f, "
+ "count=%d" % ((name, ) + stats))
+
def _file_monitor_thread(self):
""" The thread that runs the
:class:`Bcfg2.Server.FileMonitor.FileMonitor`. This also
queries :class:`Bcfg2.Server.Plugin.interfaces.Version`
plugins for the current revision of the Bcfg2 repo. """
+ self.logger.debug("File monitor thread starting")
famfd = self.fam.fileno()
terminate = self.terminate
while not terminate.isSet():
@@ -372,7 +326,7 @@ class BaseCore(object):
def _update_vcs_revision(self):
""" Update the revision of the current configuration on-disk
from the VCS plugin """
- for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version):
+ for plugin in self.plugins_by_type(Version):
try:
newrev = plugin.get_revision()
if newrev != self.revision:
@@ -384,6 +338,58 @@ class BaseCore(object):
(plugin.name, sys.exc_info()[1]))
self.revision = '-1'
+ def load_plugins(self):
+ """ Load all plugins, setting
+ :attr:`Bcfg2.Server.Core.BaseCore.plugins` and
+ :attr:`Bcfg2.Server.Core.BaseCore.metadata` as side effects.
+ This does not start plugin threads; that is done later, in
+ :func:`Bcfg2.Server.Core.BaseCore.run` """
+ while '' in self.setup['plugins']:
+ self.setup['plugins'].remove('')
+
+ for plugin in self.setup['plugins']:
+ if not plugin in self.plugins:
+ self.init_plugin(plugin)
+
+ # Remove blacklisted plugins
+ for plugin, blacklist in list(self.plugin_blacklist.items()):
+ if len(blacklist) > 0:
+ self.logger.error("The following plugins conflict with %s;"
+ "Unloading %s" % (plugin, blacklist))
+ for plug in blacklist:
+ del self.plugins[plug]
+
+ # Log experimental plugins
+ expl = [plug for plug in list(self.plugins.values())
+ if plug.experimental]
+ if expl:
+ self.logger.info("Loading experimental plugin(s): %s" %
+ (" ".join([x.name for x in expl])))
+ self.logger.info("NOTE: Interfaces subject to change")
+
+ # Log deprecated plugins
+ depr = [plug for plug in list(self.plugins.values())
+ if plug.deprecated]
+ if depr:
+ self.logger.info("Loading deprecated plugin(s): %s" %
+ (" ".join([x.name for x in depr])))
+
+ # Find the metadata plugin and set self.metadata
+ mlist = self.plugins_by_type(Metadata)
+ if len(mlist) >= 1:
+ self.metadata = mlist[0]
+ if len(mlist) > 1:
+ self.logger.error("Multiple Metadata plugins loaded; using %s"
+ % self.metadata)
+ else:
+ self.logger.error("No Metadata plugin loaded; "
+ "failed to instantiate Core")
+ raise CoreInitError("No Metadata Plugin")
+
+ if self.debug_flag:
+ # enable debugging on plugins
+ self.plugins[plugin].set_debug(self.debug_flag)
+
def init_plugin(self, plugin):
""" Import and instantiate a single plugin. The plugin is
stored to :attr:`plugins`.
@@ -468,8 +474,7 @@ class BaseCore(object):
metadata.hostname))
start = time.time()
try:
- for plugin in \
- self.plugins_by_type(Bcfg2.Server.Plugin.ClientRunHooks):
+ for plugin in self.plugins_by_type(ClientRunHooks):
try:
getattr(plugin, hook)(metadata)
except AttributeError:
@@ -500,11 +505,10 @@ class BaseCore(object):
:type data: list of lxml.etree._Element objects
"""
self.logger.debug("Validating structures for %s" % metadata.hostname)
- for plugin in \
- self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator):
+ for plugin in self.plugins_by_type(StructureValidator):
try:
plugin.validate_structures(metadata, data)
- except Bcfg2.Server.Plugin.ValidationError:
+ except ValidationError:
err = sys.exc_info()[1]
self.logger.error("Plugin %s structure validation failed: %s" %
(plugin.name, err))
@@ -527,10 +531,10 @@ class BaseCore(object):
:type data: list of lxml.etree._Element objects
"""
self.logger.debug("Validating goals for %s" % metadata.hostname)
- for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator):
+ for plugin in self.plugins_by_type(GoalValidator):
try:
plugin.validate_goals(metadata, data)
- except Bcfg2.Server.Plugin.ValidationError:
+ except ValidationError:
err = sys.exc_info()[1]
self.logger.error("Plugin %s goal validation failed: %s" %
(plugin.name, err.message))
@@ -548,8 +552,9 @@ class BaseCore(object):
:returns: list of :class:`lxml.etree._Element` objects
"""
self.logger.debug("Getting structures for %s" % metadata.hostname)
- structures = list(chain(*[struct.BuildStructures(metadata)
- for struct in self.structures]))
+ structures = list(
+ chain(*[struct.BuildStructures(metadata)
+ for struct in self.plugins_by_type(Structure)]))
sbundles = [b.get('name') for b in structures if b.tag == 'Bundle']
missing = [b for b in metadata.bundles if b not in sbundles]
if missing:
@@ -634,8 +639,9 @@ class BaseCore(object):
self.logger.error("Falling back to %s:%s" %
(entry.tag, entry.get('name')))
- glist = [gen for gen in self.generators if
- entry.get('name') in gen.Entries.get(entry.tag, {})]
+ generators = self.plugins_by_type(Generator)
+ glist = [gen for gen in generators
+ if entry.get('name') in gen.Entries.get(entry.tag, {})]
if len(glist) == 1:
return glist[0].Entries[entry.tag][entry.get('name')](entry,
metadata)
@@ -643,8 +649,8 @@ class BaseCore(object):
generators = ", ".join([gen.name for gen in glist])
self.logger.error("%s %s served by multiple generators: %s" %
(entry.tag, entry.get('name'), generators))
- g2list = [gen for gen in self.generators if
- gen.HandlesEntry(entry, metadata)]
+ g2list = [gen for gen in generators
+ if gen.HandlesEntry(entry, metadata)]
try:
if len(g2list) == 1:
return g2list[0].HandleEntry(entry, metadata)
@@ -671,7 +677,7 @@ class BaseCore(object):
revision=self.revision)
try:
meta = self.build_metadata(client)
- except Bcfg2.Server.Plugin.MetadataConsistencyError:
+ except MetadataConsistencyError:
self.logger.error("Metadata consistency error for client %s" %
client)
return lxml.etree.Element("error", type='metadata error')
@@ -718,7 +724,8 @@ class BaseCore(object):
:type event: Bcfg2.Server.FileMonitor.Event
"""
if event.filename != self.cfile:
- print("Got event for unknown file: %s" % event.filename)
+ self.logger.error("Got event for unknown file: %s" %
+ event.filename)
return
if event.code2str() == 'deleted':
return
@@ -755,16 +762,25 @@ class BaseCore(object):
return False
try:
+ self.load_plugins()
+
self.fam.start()
self.fam_thread.start()
self.fam.AddMonitor(self.cfile, self)
+ if self.perflog_thread is not None:
+ self.perflog_thread.start()
- for plug in self.plugins_by_type(Bcfg2.Server.Plugin.Threaded):
+ for plug in self.plugins_by_type(Threaded):
plug.start_threads()
except:
self.shutdown()
raise
+ if self.setup['fam_blocking']:
+ time.sleep(1)
+ while self.fam.pending() != 0:
+ time.sleep(1)
+
self.set_debug(None, self.debug_flag)
self._block()
@@ -796,7 +812,7 @@ class BaseCore(object):
"""
self.logger.debug("Getting decision list for %s" % metadata.hostname)
result = []
- for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision):
+ for plugin in self.plugins_by_type(Decision):
try:
result.extend(plugin.GetDecisions(metadata, mode))
except:
@@ -815,7 +831,7 @@ class BaseCore(object):
"""
if not hasattr(self, 'metadata'):
# some threads start before metadata is even loaded
- raise Bcfg2.Server.Plugin.MetadataRuntimeError
+ raise MetadataRuntimeError("Metadata not loaded yet")
if self.metadata_cache_mode == 'initial':
# the Metadata plugin handles loading the cached data if
# we're only caching the initial metadata object
@@ -825,10 +841,11 @@ class BaseCore(object):
if not imd:
self.logger.debug("Building metadata for %s" % client_name)
imd = self.metadata.get_initial_metadata(client_name)
- for conn in self.connectors:
+ connectors = self.plugins_by_type(Connector)
+ for conn in connectors:
grps = conn.get_additional_groups(imd)
self.metadata.merge_additional_groups(imd, grps)
- for conn in self.connectors:
+ for conn in connectors:
data = conn.get_additional_data(imd)
self.metadata.merge_additional_data(imd, conn.name, data)
imd.query.by_name = self.build_metadata
@@ -849,7 +866,7 @@ class BaseCore(object):
meta = self.build_metadata(client_name)
state = statistics.find(".//Statistics")
if state.get('version') >= '2.0':
- for plugin in self.statistics:
+ for plugin in self.plugins_by_type(Statistics):
try:
plugin.process_statistics(meta, statistics)
except:
@@ -891,11 +908,11 @@ class BaseCore(object):
meta = self.build_metadata(client)
else:
meta = None
- except Bcfg2.Server.Plugin.MetadataConsistencyError:
+ except MetadataConsistencyError:
err = sys.exc_info()[1]
self.critical_error("Client metadata resolution error for %s: %s" %
(address[0], err))
- except Bcfg2.Server.Plugin.MetadataRuntimeError:
+ except MetadataRuntimeError:
err = sys.exc_info()[1]
self.critical_error('Metadata system runtime failure for %s: %s' %
(address[0], err))
@@ -989,8 +1006,7 @@ class BaseCore(object):
version))
try:
self.metadata.set_version(client, version)
- except (Bcfg2.Server.Plugin.MetadataConsistencyError,
- Bcfg2.Server.Plugin.MetadataRuntimeError):
+ except (MetadataConsistencyError, MetadataRuntimeError):
err = sys.exc_info()[1]
self.critical_error("Unable to set version for %s: %s" %
(client, err))
@@ -1010,7 +1026,7 @@ class BaseCore(object):
client, metadata = self.resolve_client(address, cleanup_cache=True)
self.logger.debug("Getting probes for %s" % client)
try:
- for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing):
+ for plugin in self.plugins_by_type(Probing):
for probe in plugin.GetProbes(metadata):
resp.append(probe)
return lxml.etree.tostring(resp,
@@ -1080,8 +1096,7 @@ class BaseCore(object):
self.logger.debug("%s sets its profile to %s" % (client, profile))
try:
self.metadata.set_profile(client, profile, address)
- except (Bcfg2.Server.Plugin.MetadataConsistencyError,
- Bcfg2.Server.Plugin.MetadataRuntimeError):
+ except (MetadataConsistencyError, MetadataRuntimeError):
err = sys.exc_info()[1]
self.critical_error("Unable to assert profile for %s: %s" %
(client, err))
@@ -1103,7 +1118,7 @@ class BaseCore(object):
config = self.BuildConfiguration(client)
return lxml.etree.tostring(config,
xml_declaration=False).decode('UTF-8')
- except Bcfg2.Server.Plugin.MetadataConsistencyError:
+ except MetadataConsistencyError:
self.critical_error("Metadata consistency failure for %s" % client)
@exposed
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index 37bc230d1..ae7c75804 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -40,7 +40,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
"NagiosGen/config.xml": "nagiosgen.xsd",
"FileProbes/config.xml": "fileprobes.xsd",
"SSLCA/**/cert.xml": "sslca-cert.xsd",
- "SSLCA/**/key.xml": "sslca-key.xsd"
+ "SSLCA/**/key.xml": "sslca-key.xsd",
+ "GroupLogic/groups.xml": "grouplogic.xsd"
}
self.filelists = {}
@@ -83,17 +84,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
else:
self.LintError("properties-schema-not-found",
"No schema found for %s" % filename)
+ # ensure that it at least parses
+ self.parse(filename)
- def validate(self, filename, schemafile, schema=None):
- """validate a file against the given lxml.etree.Schema.
- return True on success, False on failure """
- if schema is None:
- # if no schema object was provided, instantiate one
- schema = self._load_schema(schemafile)
- if not schema:
- return False
+ def parse(self, filename):
+ """ Parse an XML file, raising the appropriate LintErrors if
+ it can't be parsed or read. Return the
+ lxml.etree._ElementTree parsed from the file. """
try:
- datafile = lxml.etree.parse(filename)
+ return lxml.etree.parse(filename)
except SyntaxError:
lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT)
self.LintError("xml-failed-to-parse",
@@ -106,6 +105,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
"Failed to open file %s" % filename)
return False
+ def validate(self, filename, schemafile, schema=None):
+ """validate a file against the given lxml.etree.Schema.
+ return True on success, False on failure """
+ if schema is None:
+ # if no schema object was provided, instantiate one
+ schema = self._load_schema(schemafile)
+ if not schema:
+ return False
+ datafile = self.parse(filename)
if not schema.validate(datafile):
cmd = ["xmllint"]
if self.files is None:
diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index 2b878d7e2..b14968d77 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -530,8 +530,8 @@ class XMLFileBacked(FileBacked):
#: XInclude.
self.extra_monitors = []
- if ((create or (self.create is not None and self.create))
- and not os.path.exists(self.name)):
+ if ((create is not None or self.create not in [None, False]) and
+ not os.path.exists(self.name)):
toptag = create or self.create
self.logger.warning("%s does not exist, creating" % self.name)
if hasattr(toptag, "getroottree"):
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py
index 581a997d8..c7b62f352 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py
@@ -69,7 +69,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile):
the given client metadata, and may be obtained by
doing ``self.XMLMatch(metadata)``
:type spec: lxml.etree._Element
- :returns: None
+ :returns: string - The filename of the private key
"""
if spec is None:
spec = self.XMLMatch(metadata)
@@ -140,7 +140,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile):
if spec is None:
spec = self.XMLMatch(metadata)
category = spec.get("category", self.category)
- print("category=%s" % category)
if category is None:
per_host_default = "true"
else:
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
index 926172e57..ffe93c25b 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
@@ -599,6 +599,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet,
else:
try:
if not isinstance(data, unicode):
+ if not isinstance(data, str):
+ data = data.decode('utf-8')
data = u_str(data, self.encoding)
except UnicodeDecodeError:
msg = "Failed to decode %s: %s" % (entry.get('name'),
diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py
index c8362db41..44971aba7 100644
--- a/src/lib/Bcfg2/Server/Plugins/Git.py
+++ b/src/lib/Bcfg2/Server/Plugins/Git.py
@@ -44,7 +44,7 @@ class Git(Version):
else:
cmd = ["git", "--git-dir", self.vcs_path,
"--work-tree", self.vcs_root, "rev-parse", "HEAD"]
- self.debug_log("Git: Running cmd")
+ self.debug_log("Git: Running %s" % cmd)
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
rv, err = proc.communicate()
if proc.wait():
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py
new file mode 100644
index 000000000..810b273af
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py
@@ -0,0 +1,47 @@
+""" GroupLogic is a connector plugin that lets you use an XML Genshi
+template to dynamically set additional groups for clients. """
+
+import os
+import lxml.etree
+import Bcfg2.Server.Plugin
+try:
+ from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile
+except ImportError:
+ # BundleTemplateFile missing means that genshi is missing. we
+ # import genshi to get the _real_ error
+ import genshi # pylint: disable=W0611
+
+
+class GroupLogicConfig(BundleTemplateFile):
+ """ Representation of the GroupLogic groups.xml file """
+ create = lxml.etree.Element("GroupLogic",
+ nsmap=dict(py="http://genshi.edgewall.org/"))
+
+ def __init__(self, name, fam):
+ BundleTemplateFile.__init__(self, name,
+ Bcfg2.Server.Plugin.Specificity(), None)
+ self.fam = fam
+ self.should_monitor = True
+ self.fam.AddMonitor(self.name, self)
+
+ def _match(self, item, metadata):
+ if item.tag == 'Group' and not len(item.getchildren()):
+ return [item]
+ return BundleTemplateFile._match(self, item, metadata)
+
+
+class GroupLogic(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Connector):
+ """ GroupLogic is a connector plugin that lets you use an XML
+ Genshi template to dynamically set additional groups for
+ clients. """
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"),
+ core.fam)
+
+ def get_additional_groups(self, metadata):
+ return [el.get("name")
+ for el in self.config.get_xml_value(metadata).findall("Group")]
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index bdf3b87fe..71e81c1fe 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -945,7 +945,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
self.debug_log("Client %s set as nonexistent group %s"
% (client, group))
- def set_profile(self, client, profile, addresspair):
+ def set_profile(self, client, profile, # pylint: disable=W0221
+ addresspair, require_public=True):
"""Set group parameter for provided client."""
self.logger.info("Asserting client %s profile to %s" % (client,
profile))
@@ -957,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
self.logger.error(msg)
raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
group = self.groups[profile]
- if not group.is_public:
+ if require_public and not group.is_public:
msg = "Cannot set client %s to private group %s" % (client,
profile)
self.logger.error(msg)
@@ -1128,7 +1129,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
pgroup = self.default
if pgroup:
- self.set_profile(client, pgroup, (None, None))
+ self.set_profile(client, pgroup, (None, None),
+ require_public=False)
profile = _add_group(pgroup)
else:
msg = "Cannot add new client %s; no default group set" % client
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index 4eefd0722..bc2928fa6 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -93,6 +93,8 @@ class AptSource(Source):
self.logger.error("Packages: Failed to read file %s" % fname)
raise
for line in reader.readlines():
+ if not isinstance(line, str):
+ line = line.decode('utf-8')
words = str(line.strip()).split(':', 1)
if words[0] == 'Package':
pkgname = words[1].strip().rstrip()
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index b4d481459..7ba374dd3 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -315,7 +315,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
:raises: OSError - If the saved data cannot be read
:raises: cPickle.UnpicklingError - If the saved data is corrupt """
- data = open(self.cachefile)
+ data = open(self.cachefile, 'rb')
(self.pkgnames, self.deps, self.provides,
self.essentialpkgs) = cPickle.load(data)
@@ -615,7 +615,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
self.logger.info("Packages: Updating %s" % url)
fname = self.escape_url(url)
try:
- open(fname, 'w').write(fetch_url(url))
+ open(fname, 'wb').write(fetch_url(url))
except ValueError:
self.logger.error("Packages: Bad url string %s" % url)
raise
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 4f163a1ab..efbca28cd 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -18,7 +18,8 @@ from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo"
#: The default path for generated apt configs
-APT_CONFIG_DEFAULT = "/etc/apt/sources.d/bcfg2"
+APT_CONFIG_DEFAULT = \
+ "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list"
class Packages(Bcfg2.Server.Plugin.Plugin,
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index a8001d9af..309b96475 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -63,7 +63,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924
.json, and .yaml properties to provide convenient ways to use
ProbeData objects as XML, JSON, or YAML data """
def __new__(cls, data):
- return str.__new__(cls, data)
+ return str.__new__(cls, data.encode('utf-8'))
def __init__(self, data): # pylint: disable=W0613
str.__init__(self)
@@ -153,7 +153,20 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
probe = lxml.etree.Element('probe')
probe.set('name', os.path.basename(name))
probe.set('source', self.plugin_name)
- probe.text = entry.data
+ if (metadata.version_info and
+ metadata.version_info > (1, 3, 1, '', 0)):
+ try:
+ probe.text = entry.data.decode('utf-8')
+ except AttributeError:
+ probe.text = entry.data
+ else:
+ try:
+ probe.text = entry.data
+ except: # pylint: disable=W0702
+ self.logger.error("Client unable to handle unicode "
+ "probes. Skipping %s" %
+ probe.get('name'))
+ continue
match = self.bangline.match(entry.data.split('\n')[0])
if match:
probe.set('interpreter', match.group('interpreter'))
@@ -209,8 +222,15 @@ class Probes(Bcfg2.Server.Plugin.Probing,
lxml.etree.SubElement(top, 'Client', name=client,
timestamp=str(int(probedata.timestamp)))
for probe in sorted(probedata):
- lxml.etree.SubElement(ctag, 'Probe', name=probe,
- value=str(self.probedata[client][probe]))
+ try:
+ lxml.etree.SubElement(
+ ctag, 'Probe', name=probe,
+ value=str(
+ self.probedata[client][probe]).decode('utf-8'))
+ except AttributeError:
+ lxml.etree.SubElement(
+ ctag, 'Probe', name=probe,
+ value=str(self.probedata[client][probe]))
for group in sorted(self.cgroups[client]):
lxml.etree.SubElement(ctag, "Group", name=group)
try:
diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py
index 3bd6fd14f..3354763d4 100644
--- a/src/lib/Bcfg2/Server/Plugins/Reporting.py
+++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py
@@ -96,7 +96,7 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable):
client.hostname, cdata,
lxml.etree.tostring(
stats,
- xml_declaration=False).decode('UTF-8'))
+ xml_declaration=False))
self.debug_log("%s: Queued statistics data for %s" %
(self.__class__.__name__, client.hostname))
return
diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
index fc07a90e9..5aa7c4d9e 100644
--- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py
+++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
@@ -172,7 +172,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
for name in names[cmeta.hostname]:
newnames.add(name.split('.')[0])
try:
- newips.add(self.get_ipcache_entry(name)[0])
+ newips.update(self.get_ipcache_entry(name)[0])
except: # pylint: disable=W0702
continue
names[cmeta.hostname].update(newnames)
@@ -288,7 +288,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
else:
# need to add entry
try:
- ipaddr = socket.gethostbyname(client)
+ ipaddr = set([addr[0] for (_, _, _, _, addr) in socket.getaddrinfo(client, None)])
self.ipcache[client] = (ipaddr, client)
return (ipaddr, client)
except socket.gaierror:
diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py
index 7d00201da..f111ffc60 100644
--- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py
+++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py
@@ -68,7 +68,7 @@ class SSLCACertSpec(SSLCAXMLSpec):
def get_spec(self, metadata):
rv = SSLCAXMLSpec.get_spec(self, metadata)
rv['subjectaltname'] = [e.text for e in self.Match(metadata)
- if e.tag == "SubjectAltName"]
+ if e.tag == "subjectAltName"]
return rv
generated by cgit v1.2.3-1-g7c22 (git 2.25.1) at 2024-10-24 14:11:41 +0000