summaryrefslogtreecommitdiffstats
path: root/src/lib/Bcfg2/Server
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/Bcfg2/Server')
-rw-r--r--src/lib/Bcfg2/Server/Admin/Init.py27
-rw-r--r--src/lib/Bcfg2/Server/Core.py133
-rw-r--r--src/lib/Bcfg2/Server/FileMonitor/Inotify.py23
-rw-r--r--src/lib/Bcfg2/Server/FileMonitor/__init__.py14
-rw-r--r--src/lib/Bcfg2/Server/Lint/RequiredAttrs.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugin/base.py5
-rw-r--r--src/lib/Bcfg2/Server/Plugin/helpers.py1
-rw-r--r--src/lib/Bcfg2/Server/Plugin/interfaces.py1
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py4
-rw-r--r--src/lib/Bcfg2/Server/Plugins/DBStats.py5
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Metadata.py119
-rw-r--r--src/lib/Bcfg2/Server/Plugins/POSIXCompat.py5
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Yum.py85
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/__init__.py23
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Reporting.py5
-rw-r--r--src/lib/Bcfg2/Server/SSLServer.py9
16 files changed, 387 insertions, 74 deletions
diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py
index 724da124b..884405786 100644
--- a/src/lib/Bcfg2/Server/Admin/Init.py
+++ b/src/lib/Bcfg2/Server/Admin/Init.py
@@ -12,6 +12,7 @@ from Bcfg2.Utils import Executor
import Bcfg2.Server.Admin
import Bcfg2.Server.Plugin
import Bcfg2.Options
+import Bcfg2.Server.Plugins.Metadata
from Bcfg2.Compat import input # pylint: disable=W0622
# default config file
@@ -171,8 +172,6 @@ class Init(Bcfg2.Server.Admin.Mode):
self.data['certpath'] = os.path.join(basepath, 'bcfg2.crt')
def __call__(self, args):
- Bcfg2.Server.Admin.Mode.__call__(self, args)
-
# Parse options
setup = Bcfg2.Options.get_option_parser()
setup.add_options(dict(configfile=Bcfg2.Options.CFILE,
@@ -218,7 +217,7 @@ class Init(Bcfg2.Server.Admin.Mode):
"""Ask for the repository path."""
while True:
newrepo = safe_input("Location of Bcfg2 repository [%s]: " %
- self.data['repopath'])
+ self.data['repopath'])
if newrepo != '':
self.data['repopath'] = os.path.abspath(newrepo)
if os.path.isdir(self.data['repopath']):
@@ -296,7 +295,7 @@ class Init(Bcfg2.Server.Admin.Mode):
"created [%s]: " % self.data['keypath'])
if keypath:
self.data['keypath'] = keypath
- certpath = safe_input("Path where Bcfg2 server cert will be created"
+ certpath = safe_input("Path where Bcfg2 server cert will be created "
"[%s]: " % self.data['certpath'])
if certpath:
self.data['certpath'] = certpath
@@ -324,6 +323,16 @@ class Init(Bcfg2.Server.Admin.Mode):
def init_repo(self):
"""Setup a new repo and create the content of the
configuration file."""
+ # Create the repository
+ path = os.path.join(self.data['repopath'], 'etc')
+ try:
+ os.makedirs(path)
+ self._init_plugins()
+ print("Repository created successfuly in %s" %
+ self.data['repopath'])
+ except OSError:
+ print("Failed to create %s." % path)
+
confdata = CONFIG % (self.data['repopath'],
','.join(self.plugins),
self.data['sendmail'],
@@ -339,13 +348,3 @@ class Init(Bcfg2.Server.Admin.Mode):
create_key(self.data['shostname'], self.data['keypath'],
self.data['certpath'], self.data['country'],
self.data['state'], self.data['location'])
-
- # Create the repository
- path = os.path.join(self.data['repopath'], 'etc')
- try:
- os.makedirs(path)
- self._init_plugins()
- print("Repository created successfuly in %s" %
- self.data['repopath'])
- except OSError:
- print("Failed to create %s." % path)
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index 07f9e0588..c69e8b055 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -4,8 +4,9 @@ implementations inherit from. """
import os
import sys
import time
-import select
import atexit
+import select
+import signal
import logging
import inspect
import threading
@@ -119,13 +120,36 @@ class BaseCore(object):
#: A :class:`logging.Logger` object for use by the core
self.logger = logging.getLogger('bcfg2-server')
+ #: Log levels for the various logging handlers with debug True
+ #: and False. Each loglevel dict is a dict of ``logger name
+ #: => log level``; the logger names are set in
+ #: :mod:`Bcfg2.Logger`. The logger name ``default`` is
+ #: special, and will be used for any log handlers whose name
+ #: does not appear elsewhere in the dict. At a minimum,
+ #: ``default`` must be provided.
+ self._loglevels = {True: dict(default=logging.DEBUG),
+ False: dict(console=logging.INFO,
+ default=level)}
+
+ #: Used to keep track of the current debug state of the core.
+ self.debug_flag = False
+
+ # enable debugging on the core now. debugging is enabled on
+ # everything else later
+ if self.setup['debug']:
+ self.set_core_debug(None, setup['debug'])
+
if 'ignore' not in self.setup:
self.setup.add_option('ignore', SERVER_FAM_IGNORE)
self.setup.reparse()
+
famargs = dict(filemonitor=self.setup['filemonitor'],
debug=self.setup['debug'],
ignore=self.setup['ignore'])
- if self.setup['filemonitor'] not in Bcfg2.Server.FileMonitor.available:
+ try:
+ filemonitor = \
+ Bcfg2.Server.FileMonitor.available[setup['filemonitor']]
+ except KeyError:
self.logger.error("File monitor driver %s not available; "
"forcing to default" % self.setup['filemonitor'])
famargs['filemonitor'] = 'default'
@@ -281,6 +305,14 @@ class BaseCore(object):
#: The CA that signed the server cert
self.ca = self.setup['ca']
+ def hdlr(sig, frame): # pylint: disable=W0613
+ """ Handle SIGINT/Ctrl-C by shutting down the core and exiting
+ properly. """
+ self.shutdown()
+ os._exit(1) # pylint: disable=W0212
+
+ signal.signal(signal.SIGINT, hdlr)
+
#: The FAM :class:`threading.Thread`,
#: :func:`_file_monitor_thread`
self.fam_thread = \
@@ -295,6 +327,10 @@ class BaseCore(object):
#: metadata
self.metadata_cache = Cache()
+ if self.debug_flag:
+ # enable debugging on everything else.
+ self.plugins[plugin].set_debug(self.debug_flag)
+
def plugins_by_type(self, base_cls):
""" Return a list of loaded plugins that match the passed type.
@@ -392,6 +428,7 @@ class BaseCore(object):
def shutdown(self):
""" Perform plugin and FAM shutdown tasks. """
+ self.logger.debug("Shutting down core...")
if not self.terminate.isSet():
self.terminate.set()
self.fam.shutdown()
@@ -427,6 +464,8 @@ class BaseCore(object):
hook.
:type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
"""
+ self.logger.debug("Running %s hooks for %s" % (hook,
+ metadata.hostname))
start = time.time()
try:
for plugin in \
@@ -460,6 +499,7 @@ class BaseCore(object):
client
:type data: list of lxml.etree._Element objects
"""
+ self.logger.debug("Validating structures for %s" % metadata.hostname)
for plugin in \
self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator):
try:
@@ -486,6 +526,7 @@ class BaseCore(object):
client
:type data: list of lxml.etree._Element objects
"""
+ self.logger.debug("Validating goals for %s" % metadata.hostname)
for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator):
try:
plugin.validate_goals(metadata, data)
@@ -506,6 +547,7 @@ class BaseCore(object):
:type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
:returns: list of :class:`lxml.etree._Element` objects
"""
+ self.logger.debug("Getting structures for %s" % metadata.hostname)
structures = list(chain(*[struct.BuildStructures(metadata)
for struct in self.structures]))
sbundles = [b.get('name') for b in structures if b.tag == 'Bundle']
@@ -528,6 +570,7 @@ class BaseCore(object):
structures to. Modified in-place.
:type config: lxml.etree._Element
"""
+ self.logger.debug("Binding structures for %s" % metadata.hostname)
for astruct in structures:
try:
self.BindStructure(astruct, metadata)
@@ -544,6 +587,9 @@ class BaseCore(object):
:param metadata: Client metadata to bind structure for
:type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
"""
+ self.logger.debug("Binding structure %s for %s" %
+ (structure.get("name", "unknown"),
+ metadata.hostname))
for entry in structure.getchildren():
if entry.tag.startswith("Bound"):
entry.tag = entry.tag[5:]
@@ -619,6 +665,7 @@ class BaseCore(object):
:type client: string
:returns: :class:`lxml.etree._Element` - A complete Bcfg2
configuration document """
+ self.logger.debug("Building configuration for %s" % client)
start = time.time()
config = lxml.etree.Element("Configuration", version='2.0',
revision=self.revision)
@@ -718,6 +765,12 @@ class BaseCore(object):
self.shutdown()
raise
+ if self.setup['fam_blocking']:
+ time.sleep(1)
+ while self.fam.pending() != 0:
+ time.sleep(1)
+
+ self.set_debug(None, self.debug_flag)
self._block()
def _daemonize(self):
@@ -746,6 +799,7 @@ class BaseCore(object):
:type mode: string
:returns: list of Decision tuples ``(<entry tag>, <entry name>)``
"""
+ self.logger.debug("Getting decision list for %s" % metadata.hostname)
result = []
for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision):
try:
@@ -816,6 +870,7 @@ class BaseCore(object):
else:
imd = self.metadata_cache.get(client_name, None)
if not imd:
+ self.logger.debug("Building metadata for %s" % client_name)
imd = self.metadata.get_initial_metadata(client_name)
for conn in self.connectors:
grps = conn.get_additional_groups(imd)
@@ -837,6 +892,7 @@ class BaseCore(object):
:param statistics: The statistics document to process
:type statistics: lxml.etree._Element
"""
+ self.logger.debug("Processing statistics for %s" % client_name)
meta = self.build_metadata(client_name)
state = statistics.find(".//Statistics")
if state.get('version') >= '2.0':
@@ -907,10 +963,12 @@ class BaseCore(object):
def _get_rmi(self):
""" Get a list of RMI calls exposed by plugins """
rmi = dict()
- if self.plugins:
- for pname, pinst in list(self.plugins.items()):
- for mname in pinst.__rmi__:
- rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname)
+ for pname, pinst in list(self.plugins.items()):
+ for mname in pinst.__rmi__:
+ rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname)
+ famname = self.fam.__class__.__name__
+ for mname in self.fam.__rmi__:
+ rmi["%s.%s" % (famname, mname)] = getattr(self.fam, mname)
return rmi
def _resolve_exposed_method(self, method_name):
@@ -964,6 +1022,7 @@ class BaseCore(object):
return func.__doc__
@exposed
+ @track_statistics()
def DeclareVersion(self, address, version):
""" Declare the client version.
@@ -974,7 +1033,9 @@ class BaseCore(object):
:returns: bool - True on success
:raises: :exc:`xmlrpclib.Fault`
"""
- client = self.resolve_client(address)[0]
+ client = self.resolve_client(address, metadata=False)[0]
+ self.logger.debug("%s is running Bcfg2 client version %s" % (client,
+ version))
try:
self.metadata.set_version(client, version)
except (Bcfg2.Server.Plugin.MetadataConsistencyError,
@@ -996,6 +1057,7 @@ class BaseCore(object):
"""
resp = lxml.etree.Element('probes')
client, metadata = self.resolve_client(address, cleanup_cache=True)
+ self.logger.debug("Getting probes for %s" % client)
try:
for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing):
for probe in plugin.GetProbes(metadata):
@@ -1017,6 +1079,7 @@ class BaseCore(object):
:raises: :exc:`xmlrpclib.Fault`
"""
client, metadata = self.resolve_client(address)
+ self.logger.debug("Receiving probe data from %s" % client)
if self.metadata_cache_mode == 'cautious':
# clear the metadata cache right after building the
# metadata object; that way the cache is cleared for any
@@ -1063,6 +1126,7 @@ class BaseCore(object):
:raises: :exc:`xmlrpclib.Fault`
"""
client = self.resolve_client(address, metadata=False)[0]
+ self.logger.debug("%s sets its profile to %s" % (client, profile))
try:
self.metadata.set_profile(client, profile, address)
except (Bcfg2.Server.Plugin.MetadataConsistencyError,
@@ -1171,22 +1235,35 @@ class BaseCore(object):
:type address: tuple
:returns: bool - The new debug state of the FAM
"""
- for plugin in self.plugins.values():
- plugin.toggle_debug()
- return self.toggle_fam_debug(address)
+ return self.set_debug(address, not self.debug_flag)
@exposed
- def toggle_fam_debug(self, _):
+ def toggle_core_debug(self, address):
+ """ Toggle debug status of the server core
+
+ :param address: Client (address, hostname) pair
+ :type address: tuple
+ :returns: bool - The new debug state of the FAM
+ """
+ return self.set_core_debug(address, not self.debug_flag)
+
+ @exposed
+ def toggle_fam_debug(self, address):
""" Toggle debug status of the FAM
:returns: bool - The new debug state of the FAM
"""
- return self.fam.toggle_debug()
+ self.logger.warning("Deprecated method set_fam_debug called by %s" %
+ address[0])
+ return "This method is deprecated and will be removed in a future " + \
+ "release\n%s" % self.fam.toggle_debug()
@exposed
def set_debug(self, address, debug):
""" Explicitly set debug status of the FAM and all plugins
+ :param address: Client (address, hostname) pair
+ :type address: tuple
:param debug: The new debug status. This can either be a
boolean, or a string describing the state (e.g.,
"true" or "false"; case-insensitive)
@@ -1197,10 +1274,33 @@ class BaseCore(object):
debug = debug.lower() == "true"
for plugin in self.plugins.values():
plugin.set_debug(debug)
- return self.set_fam_debug(address, debug)
+ rv = self.set_core_debug(address, debug)
+ return self.fam.set_debug(debug) and rv
+
+ @exposed
+ def set_core_debug(self, _, debug):
+ """ Explicity set debug status of the server core
+
+ :param debug: The new debug status. This can either be a
+ boolean, or a string describing the state (e.g.,
+ "true" or "false"; case-insensitive)
+ :type debug: bool or string
+ :returns: bool - The new debug state of the FAM
+ """
+ if debug not in [True, False]:
+ debug = debug.lower() == "true"
+ self.debug_flag = debug
+ self.logger.info("Core: debug = %s" % debug)
+ levels = self._loglevels[self.debug_flag]
+ for handler in logging.root.handlers:
+ level = levels.get(handler.name, levels['default'])
+ self.logger.debug("Setting %s log handler to %s" %
+ (handler.name, logging.getLevelName(level)))
+ handler.setLevel(level)
+ return self.debug_flag
@exposed
- def set_fam_debug(self, _, debug):
+ def set_fam_debug(self, address, debug):
""" Explicitly set debug status of the FAM
:param debug: The new debug status of the FAM. This can
@@ -1212,4 +1312,7 @@ class BaseCore(object):
"""
if debug not in [True, False]:
debug = debug.lower() == "true"
- return self.fam.set_debug(debug)
+ self.logger.warning("Deprecated method set_fam_debug called by %s" %
+ address[0])
+ return "This method is deprecated and will be removed in a future " + \
+ "release\n%s" % self.fam.set_debug(debug)
diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
index 178a47b1a..cdd52dbb9 100644
--- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
+++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
@@ -2,6 +2,7 @@
support. """
import os
+import errno
import logging
import pyinotify
from Bcfg2.Compat import reduce # pylint: disable=W0622
@@ -15,6 +16,8 @@ class Inotify(Pseudo, pyinotify.ProcessEvent):
""" File monitor backend with `inotify
<http://inotify.aiken.cz/>`_ support. """
+ __rmi__ = Pseudo.__rmi__ + ["list_watches", "list_paths"]
+
#: Inotify is the best FAM backend, so it gets a very high
#: priority
__priority__ = 99
@@ -182,6 +185,9 @@ class Inotify(Pseudo, pyinotify.ProcessEvent):
try:
watchdir = self.watches_by_path[watch_path]
except KeyError:
+ if not os.path.exists(watch_path):
+ raise OSError(errno.ENOENT,
+ "No such file or directory: '%s'" % path)
watchdir = self.watchmgr.add_watch(watch_path, self.mask,
quiet=False)[watch_path]
self.watches_by_path[watch_path] = watchdir
@@ -211,3 +217,20 @@ class Inotify(Pseudo, pyinotify.ProcessEvent):
if self.notifier:
self.notifier.stop()
shutdown.__doc__ = Pseudo.shutdown.__doc__
+
+ def list_watches(self):
+ """ XML-RPC that returns a list of current inotify watches for
+ debugging purposes. """
+ return list(self.watches_by_path.keys())
+
+ def list_paths(self):
+ """ XML-RPC that returns a list of paths that are handled for
+ debugging purposes. Because inotify doesn't like watching
+ files, but prefers to watch directories, this will be
+ different from
+ :func:`Bcfg2.Server.FileMonitor.Inotify.Inotify.ListWatches`. For
+ instance, if a plugin adds a monitor to
+ ``/var/lib/bcfg2/Plugin/foo.xml``, :func:`ListPaths` will
+ return ``/var/lib/bcfg2/Plugin/foo.xml``, while
+ :func:`ListWatches` will return ``/var/lib/bcfg2/Plugin``. """
+ return list(self.handles.keys())
diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py
index d77f21b93..522ddb705 100644
--- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py
+++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py
@@ -116,6 +116,9 @@ class FileMonitor(Debuggable):
#: should have higher priorities.
__priority__ = -1
+ #: List of names of methods to be exposed as XML-RPC functions
+ __rmi__ = Debuggable.__rmi__ + ["list_event_handlers"]
+
def __init__(self, ignore=None, debug=False):
"""
:param ignore: A list of filename globs describing events that
@@ -288,6 +291,8 @@ class FileMonitor(Debuggable):
def shutdown(self):
""" Handle any tasks required to shut down the monitor. """
+ self.debug_log("Shutting down %s file monitor" %
+ self.__class__.__name__)
self.started = False
def AddMonitor(self, path, obj, handleID=None):
@@ -310,6 +315,15 @@ class FileMonitor(Debuggable):
"""
raise NotImplementedError
+ def list_event_handlers(self):
+ """ XML-RPC that returns
+ :attr:`Bcfg2.Server.FileMonitor.FileMonitor.handles` for
+ debugging purposes. """
+ rv = dict()
+ for watch, handler in self.handles.items():
+ rv[watch] = getattr(handler, "name", handler.__class__.__name__)
+ return rv
+
#: A module-level FAM object that all plugins, etc., can use. This
#: should not be used directly, but retrieved via :func:`get_fam`.
diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
index 60525d5a1..497e8fac6 100644
--- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
+++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
@@ -37,7 +37,7 @@ def is_octal_mode(val):
def is_username(val):
""" Return True if val is a string giving either a positive
integer uid, or a valid Unix username """
- return re.match(r'^([a-z]\w{0,30}|\d+)$', val)
+ return re.match(r'^([A-z][-_A-z0-9]{0,30}|\d+)$', val)
def is_device_mode(val):
diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py
index 25a687874..f7bc08717 100644
--- a/src/lib/Bcfg2/Server/Plugin/base.py
+++ b/src/lib/Bcfg2/Server/Plugin/base.py
@@ -34,8 +34,8 @@ class Debuggable(object):
:returns: bool - The new value of the debug flag
"""
self.debug_flag = debug
- self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__,
- self.debug_flag),
+ self.debug_log("%s: debug = %s" % (self.__class__.__name__,
+ self.debug_flag),
flag=True)
return debug
@@ -122,6 +122,7 @@ class Plugin(Debuggable):
""" Perform shutdown tasks for the plugin
:returns: None """
+ self.debug_log("Shutting down %s plugin" % self.name)
self.running = False
def __str__(self):
diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index 187c594fd..ded7dd8dc 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -478,6 +478,7 @@ class XMLFileBacked(FileBacked):
def Index(self):
self.xdata = lxml.etree.XML(self.data, base_url=self.name,
parser=Bcfg2.Server.XMLParser)
+ self.extras = []
self._follow_xincludes()
if self.extras:
try:
diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py
index 3ef29775d..11a61ff9c 100644
--- a/src/lib/Bcfg2/Server/Plugin/interfaces.py
+++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py
@@ -313,6 +313,7 @@ class Threaded(object):
"""
raise NotImplementedError
+
class ThreadedStatistics(Statistics, Threaded, threading.Thread):
""" ThreadedStatistics plugins process client statistics in a
separate thread. """
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
index 5f10879be..b3781e299 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
@@ -77,6 +77,10 @@ class CfgGenshiGenerator(CfgGenerator):
__init__.__doc__ = CfgGenerator.__init__.__doc__
def get_data(self, entry, metadata):
+ if self.template is None:
+ raise PluginExecutionError("Failed to load template %s" %
+ self.name)
+
fname = entry.get('realname', entry.get('name'))
stream = self.template.generate(
name=fname,
diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py
index e0794f019..e6ef50fa1 100644
--- a/src/lib/Bcfg2/Server/Plugins/DBStats.py
+++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py
@@ -9,7 +9,6 @@ class DBStats(Bcfg2.Server.Plugin.Plugin):
def __init__(self, core, datastore):
Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
self.logger.error("DBStats has been replaced with Reporting")
- self.logger.error("DBStats: Be sure to migrate your data "\
- "before running the report collector")
+ self.logger.error("DBStats: Be sure to migrate your data "
+ "before running the report collector")
raise Bcfg2.Server.Plugin.PluginInitError
-
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index b053e65d3..7f8db7b6d 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -138,7 +138,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked):
self.logger.error('Failed to parse %s' % self.basefile)
return
self.extras = []
- self.basedata = copy.copy(xdata)
+ self.basedata = copy.deepcopy(xdata)
self._follow_xincludes(xdata=xdata)
if self.extras:
try:
@@ -263,31 +263,63 @@ class ClientMetadata(object):
# pylint: disable=R0913
def __init__(self, client, profile, groups, bundles, aliases, addresses,
categories, uuid, password, version, query):
+ #: The client hostname (as a string)
self.hostname = client
+
+ #: The client profile (as a string)
self.profile = profile
+
+ #: The set of all bundles this client gets
self.bundles = bundles
+
+ #: A list of all client aliases
self.aliases = aliases
+
+ #: A list of all addresses this client is known by
self.addresses = addresses
+
+ #: A list of groups this client is a member of
self.groups = groups
+
+ #: A dict of categories of this client's groups. Keys are
+ #: category names, values are corresponding group names.
self.categories = categories
+
+ #: The UUID identifier for this client
self.uuid = uuid
+
+ #: The Bcfg2 password for this client
self.password = password
+
+ #: Connector plugins known to this client
self.connectors = []
+
+ #: The version of the Bcfg2 client this client is running, as
+ #: a string
self.version = version
try:
+ #: The version of the Bcfg2 client this client is running,
+ #: as a :class:`Bcfg2.version.Bcfg2VersionInfo` object.
self.version_info = Bcfg2VersionInfo(version)
except (ValueError, AttributeError):
self.version_info = None
+
+ #: A :class:`Bcfg2.Server.Plugins.Metadata.MetadataQuery`
+ #: object for this client.
self.query = query
# pylint: enable=R0913
def inGroup(self, group):
- """Test to see if client is a member of group."""
+ """Test to see if client is a member of group.
+
+ :returns: bool """
return group in self.groups
def group_in_category(self, category):
- """ return the group in the given category that the client is
- a member of, or the empty string """
+ """ Return the group in the given category that the client is
+ a member of, or an empty string.
+
+ :returns: string """
for grp in self.query.all_groups_in_category(category):
if grp in self.groups:
return grp
@@ -295,17 +327,59 @@ class ClientMetadata(object):
class MetadataQuery(object):
- """ object supplied to client metadata to allow client metadata
- objects to query metadata without being able to modify it """
+ """ This class provides query methods for the metadata of all
+ clients known to the Bcfg2 server, without being able to modify
+ that data.
+
+ Note that ``*by_groups()`` and ``*by_profiles()`` behave
+ differently; for a client to be included in the return value of a
+ ``*by_groups()`` method, it must be a member of *all* groups
+ listed in the argument; for a client to be included in the return
+ value of a ``*by_profiles()`` method, it must have *any* group
+ listed as its profile group. """
def __init__(self, by_name, get_clients, by_groups, by_profiles,
all_groups, all_groups_in_category):
- # resolver is set later
+ #: Get :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata`
+ #: object for the given hostname.
+ #:
+ #: :returns: Bcfg2.Server.Plugins.Metadata.ClientMetadata
self.by_name = by_name
+
+ #: Get a list of hostnames of clients that are in all given
+ #: groups.
+ #:
+ #: :param groups: The groups to check clients for membership in
+ #: :type groups: list
+ #:
+ #: :returns: list of strings
self.names_by_groups = self._warn_string(by_groups)
+
+ #: Get a list of hostnames of clients whose profile matches
+ #: any given profile group.
+ #:
+ #: :param profiles: The profiles to check clients for
+ #: membership in.
+ #: :type profiles: list
+ #: :returns: list of strings
self.names_by_profiles = self._warn_string(by_profiles)
+
+ #: Get all known client hostnames.
+ #:
+ #: :returns: list of strings
self.all_clients = get_clients
+
+ #: Get all known group names.
+ #:
+ #: :returns: list of strings
self.all_groups = all_groups
+
+ #: Get the names of all groups in the given category.
+ #:
+ #: :param category: The category to query for groups that
+ #: belong to it.
+ #: :type category: string
+ #: :returns: list of strings
self.all_groups_in_category = all_groups_in_category
def _warn_string(self, func):
@@ -326,22 +400,41 @@ class MetadataQuery(object):
return inner
def by_groups(self, groups):
- """ get a list of ClientMetadata objects that are in all given
- groups """
+ """ Get a list of
+ :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects
+ that are in all given groups.
+
+ :param groups: The groups to check clients for membership in.
+ :type groups: list
+ :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ objects
+ """
# don't need to decorate this with _warn_string because
# names_by_groups is decorated
return [self.by_name(name) for name in self.names_by_groups(groups)]
def by_profiles(self, profiles):
- """ get a list of ClientMetadata objects that are in any of
- the given profiles """
+ """ Get a list of
+ :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects
+ that have any of the given groups as their profile.
+
+ :param profiles: The profiles to check clients for membership
+ in.
+ :type profiles: list
+ :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ objects
+ """
# don't need to decorate this with _warn_string because
# names_by_profiles is decorated
return [self.by_name(name)
for name in self.names_by_profiles(profiles)]
def all(self):
- """ get a list of all ClientMetadata objects """
+ """ Get a list of all
+ :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` objects.
+
+ :returns: list of Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ """
return [self.by_name(name) for name in self.all_clients()]
@@ -1255,7 +1348,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
def end_statistics(self, metadata):
""" Hook to toggle clients in bootstrap mode """
if self.auth.get(metadata.hostname,
- self.core.setup('authentication')) == 'bootstrap':
+ self.core.setup['authentication']) == 'bootstrap':
self.update_client(metadata.hostname, dict(auth='cert'))
def viz(self, hosts, bundles, key, only_client, colors):
diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py
index 0dd42c9cb..490ee6f20 100644
--- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py
+++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py
@@ -15,6 +15,11 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin,
def validate_goals(self, metadata, goals):
"""Verify that we are generating correct old POSIX entries."""
+ if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0):
+ # do not care about a client that is _any_ 1.3.0 release
+ # (including prereleases and RCs)
+ return
+
for goal in goals:
for entry in goal.getchildren():
if entry.tag == 'Path' and 'mode' in entry.keys():
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 3799b1723..4535fb76d 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -313,9 +313,7 @@ class YumCollection(Collection):
@property
def __package_groups__(self):
- """ YumCollections support package groups only if
- :attr:`use_yum` is True """
- return self.use_yum
+ return True
@property
def helper(self):
@@ -663,11 +661,6 @@ class YumCollection(Collection):
In this implementation the packages may be strings or tuples.
See :ref:`yum-pkg-objects` for more information. """
- if not self.use_yum:
- self.logger.warning("Packages: Package groups are not supported "
- "by Bcfg2's internal Yum dependency generator")
- return dict()
-
if not grouplist:
return dict()
@@ -679,7 +672,15 @@ class YumCollection(Collection):
ptype = "default"
gdicts.append(dict(group=group, type=ptype))
- return self.call_helper("get_groups", inputdata=gdicts)
+ if self.use_yum:
+ return self.call_helper("get_groups", inputdata=gdicts)
+ else:
+ pkgs = dict()
+ for gdict in gdicts:
+ pkgs[gdict['group']] = Collection.get_group(self,
+ gdict['group'],
+ gdict['type'])
+ return pkgs
def _element_to_pkg(self, el, name):
""" Convert a Package or Instance element to a package tuple """
@@ -975,6 +976,7 @@ class YumSource(Source):
for x in ['global'] + self.arches])
self.needed_paths = set()
self.file_to_arch = dict()
+ self.yumgroups = dict()
__init__.__doc__ = Source.__init__.__doc__
@property
@@ -992,7 +994,8 @@ class YumSource(Source):
if not self.use_yum:
cache = open(self.cachefile, 'wb')
cPickle.dump((self.packages, self.deps, self.provides,
- self.filemap, self.url_map), cache, 2)
+ self.filemap, self.url_map,
+ self.yumgroups), cache, 2)
cache.close()
def load_state(self):
@@ -1002,7 +1005,7 @@ class YumSource(Source):
if not self.use_yum:
data = open(self.cachefile)
(self.packages, self.deps, self.provides,
- self.filemap, self.url_map) = cPickle.load(data)
+ self.filemap, self.url_map, self.yumgroups) = cPickle.load(data)
@property
def urls(self):
@@ -1057,7 +1060,7 @@ class YumSource(Source):
urls = []
for elt in xdata.findall(RPO + 'data'):
- if elt.get('type') in ['filelists', 'primary']:
+ if elt.get('type') in ['filelists', 'primary', 'group']:
floc = elt.find(RPO + 'location')
fullurl = url + floc.get('href')
urls.append(fullurl)
@@ -1074,11 +1077,14 @@ class YumSource(Source):
# we have to read primary.xml first, and filelists.xml afterwards;
primaries = list()
filelists = list()
+ groups = list()
for fname in self.files:
if fname.endswith('primary.xml.gz'):
primaries.append(fname)
elif fname.endswith('filelists.xml.gz'):
filelists.append(fname)
+ elif fname.find('comps'):
+ groups.append(fname)
for fname in primaries:
farch = self.file_to_arch[fname]
@@ -1088,6 +1094,9 @@ class YumSource(Source):
farch = self.file_to_arch[fname]
fdata = lxml.etree.parse(fname).getroot()
self.parse_filelist(fdata, farch)
+ for fname in groups:
+ fdata = lxml.etree.parse(fname).getroot()
+ self.parse_group(fdata)
# merge data
sdata = list(self.packages.values())
@@ -1151,6 +1160,35 @@ class YumSource(Source):
self.provides[arch][prov] = list()
self.provides[arch][prov].append(pkgname)
+ @Bcfg2.Server.Plugin.track_statistics()
+ def parse_group(self, data):
+ """ parse comps.xml.gz data """
+ for group in data.getchildren():
+ if not group.tag.endswith('group'):
+ continue
+ try:
+ groupid = group.xpath('id')[0].text
+ self.yumgroups[groupid] = {'mandatory': list(),
+ 'default': list(),
+ 'optional': list(),
+ 'conditional': list()}
+ except IndexError:
+ continue
+ try:
+ packagelist = group.xpath('packagelist')[0]
+ except IndexError:
+ continue
+ for pkgreq in packagelist.getchildren():
+ pkgtype = pkgreq.get('type', None)
+ if pkgtype == 'mandatory':
+ self.yumgroups[groupid]['mandatory'].append(pkgreq.text)
+ elif pkgtype == 'default':
+ self.yumgroups[groupid]['default'].append(pkgreq.text)
+ elif pkgtype == 'optional':
+ self.yumgroups[groupid]['optional'].append(pkgreq.text)
+ elif pkgtype == 'conditional':
+ self.yumgroups[groupid]['conditional'].append(pkgreq.text)
+
def is_package(self, metadata, package):
arch = [a for a in self.arches if a in metadata.groups]
if not arch:
@@ -1230,3 +1268,26 @@ class YumSource(Source):
return self.pulp_id
else:
return Source.get_repo_name(self, url_map)
+
+ def get_group(self, metadata, group, ptype=None): # pylint: disable=W0613
+ """ Get the list of packages of the given type in a package
+ group.
+
+ :param group: The name of the group to query
+ :type group: string
+ :param ptype: The type of packages to get, for backends that
+ support multiple package types in package groups
+ (e.g., "recommended," "optional," etc.)
+ :type ptype: string
+ :returns: list of strings - package names
+ """
+ try:
+ yumgroup = self.yumgroups[group]
+ except KeyError:
+ return []
+ packages = yumgroup['conditional'] + yumgroup['mandatory']
+ if ptype in ['default', 'optional', 'all']:
+ packages += yumgroup['default']
+ if ptype in ['optional', 'all']:
+ packages += yumgroup['optional']
+ return packages
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 5d3fbae2e..2175cf0aa 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -5,6 +5,7 @@ determine the completeness of the client configuration. """
import os
import sys
import glob
+import copy
import shutil
import lxml.etree
import Bcfg2.Logger
@@ -296,20 +297,22 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
"""
if self.disableResolver:
# Config requests no resolver
+ for struct in structures:
+ for pkg in struct.xpath('//Package | //BoundPackage'):
+ if pkg.get("group"):
+ if pkg.get("type"):
+ pkg.set("choose", pkg.get("type"))
return
if collection is None:
collection = self.get_collection(metadata)
- # base is the set of initial packages -- explicitly
- # given in the specification, from expanded package groups,
- # and essential to the distribution
- base = set()
+ initial = set()
to_remove = []
groups = []
for struct in structures:
for pkg in struct.xpath('//Package | //BoundPackage'):
if pkg.get("name"):
- base.update(collection.packages_from_entry(pkg))
+ initial.update(collection.packages_from_entry(pkg))
elif pkg.get("group"):
groups.append((pkg.get("group"),
pkg.get("type")))
@@ -321,6 +324,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
pkg,
xml_declaration=False).decode('UTF-8'))
+ # base is the set of initial packages explicitly given in the
+ # specification, packages from expanded package groups, and
+ # packages essential to the distribution
+ base = set(initial)
+
# remove package groups
for el in to_remove:
el.getparent().remove(el)
@@ -336,7 +344,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
if unknown:
self.logger.info("Packages: Got %d unknown entries" % len(unknown))
self.logger.info("Packages: %s" % list(unknown))
- newpkgs = collection.get_new_packages(base, packages)
+ newpkgs = collection.get_new_packages(initial, packages)
self.debug_log("Packages: %d base, %d complete, %d new" %
(len(base), len(packages), len(newpkgs)))
newpkgs.sort()
@@ -514,7 +522,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
:return: dict of lists of ``url_map`` data
"""
collection = self.get_collection(metadata)
- return dict(sources=collection.get_additional_data())
+ return dict(sources=collection.get_additional_data(),
+ allsources=copy.deepcopy(self.sources))
def end_client_run(self, metadata):
""" Hook to clear the cache for this client in
diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py
index d072f1a33..a6dc2c1ef 100644
--- a/src/lib/Bcfg2/Server/Plugins/Reporting.py
+++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py
@@ -65,10 +65,13 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable):
(self.name, traceback.format_exc().splitlines()[-1])
self.logger.error(msg)
raise PluginInitError(msg)
+ if self.debug_flag:
+ self.transport.set_debug(self.debug_flag)
def set_debug(self, debug):
rv = Debuggable.set_debug(self, debug)
- self.transport.set_debug(debug)
+ if self.transport is not None:
+ self.transport.set_debug(debug)
return rv
def process_statistics(self, client, xdata):
diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py
index 0d1246d85..28450aa1a 100644
--- a/src/lib/Bcfg2/Server/SSLServer.py
+++ b/src/lib/Bcfg2/Server/SSLServer.py
@@ -412,12 +412,9 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer,
name = instance.name
except AttributeError:
name = "unknown"
- if hasattr(instance, 'plugins'):
- for pname, pinst in list(instance.plugins.items()):
- for mname in pinst.__rmi__:
- xmname = "%s.%s" % (pname, mname)
- fn = getattr(pinst, mname)
- self.register_function(fn, name=xmname)
+ if hasattr(instance, '_get_rmi'):
+ for fname, func in instance._get_rmi().items():
+ self.register_function(func, name=fname)
self.logger.info("serving %s at %s" % (name, self.url))
def serve_forever(self):
generated by cgit v1.2.3-1-g7c22 (git 2.25.1) at 2024-07-06 04:35:15 +0000