diff options
Diffstat (limited to 'src/lib')
36 files changed, 377 insertions, 222 deletions
diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index cf4e7c7ea..1003ab842 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -5,6 +5,7 @@ import warnings warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) import os +import sys import apt.cache import Bcfg2.Options import Bcfg2.Client.Tools @@ -12,7 +13,7 @@ import Bcfg2.Client.Tools class APT(Bcfg2.Client.Tools.Tool): """The Debian toolset implements package and service operations - and inherits the rest from Tools.Tool. """ + and inherits the rest from Toolset.Toolset.""" options = Bcfg2.Client.Tools.Tool.options + [ Bcfg2.Options.PathOption( @@ -79,10 +80,14 @@ class APT(Bcfg2.Client.Tools.Tool): try: self.pkg_cache = apt.cache.Cache() except SystemError: - e = sys.exc_info()[1] - self.logger.info("Failed to initialize APT cache: %s" % e) + err = sys.exc_info()[1] + self.logger.info("Failed to initialize APT cache: %s" % err) raise Bcfg2.Client.Tools.ToolInstantiationError - self.pkg_cache.update() + try: + self.pkg_cache.update() + except apt.cache.FetchFailedException: + err = sys.exc_info()[1] + self.logger.info("Failed to update APT cache: %s" % err) self.pkg_cache = apt.cache.Cache() if 'req_reinstall_pkgs' in dir(self.pkg_cache): self._newapi = True @@ -103,9 +108,10 @@ class APT(Bcfg2.Client.Tools.Tool): for (name, version) in extras] def VerifyDebsums(self, entry, modlist): + """Verify the package contents with debsum information.""" output = \ self.cmd.run("%s -as %s" % - (self.debsums, entry.get('name'))).stdout.splitlines() + (self.debsums, entry.get('name'))).stderr.splitlines() if len(output) == 1 and "no md5sums for" in output[0]: self.logger.info("Package %s has no md5sums. Cannot verify" % entry.get('name')) @@ -127,11 +133,11 @@ class APT(Bcfg2.Client.Tools.Tool): # these files should not exist continue elif "is not installed" in item or "missing file" in item: - self.logger.error("Package %s is not fully installed" % - entry.get('name')) + self.logger.error("Package %s is not fully installed" + % entry.get('name')) else: - self.logger.error("Got Unsupported pattern %s from debsums" % - item) + self.logger.error("Got Unsupported pattern %s from debsums" + % item) files.append(item) files = list(set(files) - set(self.ignores)) # We check if there is file in the checksum to do @@ -142,30 +148,31 @@ class APT(Bcfg2.Client.Tools.Tool): bad = [filename for filename in files if filename not in modlist] if bad: self.logger.debug("It is suggested that you either manage " - "these files, revert the changes, or ignore " - "false failures:") - self.logger.info("Package %s failed validation. Bad files " - "are:" % entry.get('name')) + "these files, revert the changes, or " + "ignore false failures:") + self.logger.info("Package %s failed validation. Bad files are:" + % entry.get('name')) self.logger.info(bad) - entry.set('qtext', - "Reinstall Package %s-%s to fix failing files? " - "(y/N) " % (entry.get('name'), entry.get('version'))) + entry.set( + 'qtext', + "Reinstall Package %s-%s to fix failing files? (y/N) " + % (entry.get('name'), entry.get('version'))) return False return True def VerifyPackage(self, entry, modlist, checksums=True): """Verify package for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (entry.attrib['name'])) return False pkgname = entry.get('name') - if self.pkg_cache.has_key(pkgname): # nopep8 + if self.pkg_cache.has_key(pkgname): # noqa if self._newapi: is_installed = self.pkg_cache[pkgname].is_installed else: is_installed = self.pkg_cache[pkgname].isInstalled - if not self.pkg_cache.has_key(pkgname) or not is_installed: # nopep8 + if not self.pkg_cache.has_key(pkgname) or not is_installed: # noqa self.logger.info("Package %s not installed" % (entry.get('name'))) entry.set('current_exists', 'false') return False @@ -178,31 +185,33 @@ class APT(Bcfg2.Client.Tools.Tool): installed_version = pkg.installedVersion candidate_version = pkg.candidateVersion if entry.get('version') == 'auto': + # pylint: disable=W0212 if self._newapi: - is_upgradable = \ - self.pkg_cache._depcache.is_upgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.is_upgradable( + pkg._pkg) else: - is_upgradable = \ - self.pkg_cache._depcache.IsUpgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.IsUpgradable( + pkg._pkg) + # pylint: enable=W0212 if is_upgradable: - desiredVersion = candidate_version + desired_version = candidate_version else: - desiredVersion = installed_version + desired_version = installed_version elif entry.get('version') == 'any': - desiredVersion = installed_version + desired_version = installed_version else: - desiredVersion = entry.get('version') - if desiredVersion != installed_version: + desired_version = entry.get('version') + if desired_version != installed_version: entry.set('current_version', installed_version) entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % (entry.get('name'), entry.get('current_version'), - desiredVersion)) + desired_version)) return False else: # version matches - if (not Bcfg2.Options.setup.quick and - entry.get('verify', 'true') == 'true' - and checksums): + if not Bcfg2.Options.setup.quick \ + and entry.get('verify', 'true') == 'true' \ + and checksums: pkgsums = self.VerifyDebsums(entry, modlist) return pkgsums return True @@ -220,7 +229,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.pkg_cache[pkg].mark_delete(purge=True) else: self.pkg_cache[pkg].markDelete(purge=True) - except: + except: # pylint: disable=W0702 if self._newapi: self.pkg_cache[pkg].mark_delete() else: @@ -240,24 +249,26 @@ class APT(Bcfg2.Client.Tools.Tool): ipkgs = [] bad_pkgs = [] for pkg in packages: - if not self.pkg_cache.has_key(pkg.get('name')): # nopep8 - self.logger.error("APT has no information about package %s" % - (pkg.get('name'))) + if not self.pkg_cache.has_key(pkg.get('name')): # noqa + self.logger.error("APT has no information about package %s" + % (pkg.get('name'))) continue if pkg.get('version') in ['auto', 'any']: if self._newapi: try: - cversion = \ - self.pkg_cache[pkg.get('name')].candidate.version - ipkgs.append("%s=%s" % (pkg.get('name'), cversion)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidate.version)) except AttributeError: self.logger.error("Failed to find %s in apt package " "cache" % pkg.get('name')) continue else: - cversion = self.pkg_cache[pkg.get('name')].candidateVersion - ipkgs.append("%s=%s" % (pkg.get('name'), cversion)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidateVersion)) continue + # pylint: disable=W0212 if self._newapi: avail_vers = [ x.ver_str for x in @@ -266,13 +277,14 @@ class APT(Bcfg2.Client.Tools.Tool): avail_vers = [ x.VerStr for x in self.pkg_cache[pkg.get('name')]._pkg.VersionList] + # pylint: enable=W0212 if pkg.get('version') in avail_vers: ipkgs.append("%s=%s" % (pkg.get('name'), pkg.get('version'))) continue else: - self.logger.error("Package %s: desired version %s not in %s" % - (pkg.get('name'), pkg.get('version'), - avail_vers)) + self.logger.error("Package %s: desired version %s not in %s" + % (pkg.get('name'), pkg.get('version'), + avail_vers)) bad_pkgs.append(pkg.get('name')) if bad_pkgs: self.logger.error("Cannot find correct versions of packages:") @@ -290,6 +302,6 @@ class APT(Bcfg2.Client.Tools.Tool): self.modified.append(package) return states - def VerifyPath(self, entry, _): + def VerifyPath(self, entry, _): # pylint: disable=W0613 """Do nothing here since we only verify Path type=ignore.""" return True diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index dedc50d89..ca0502b75 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -2,7 +2,6 @@ import Bcfg2.Client.Tools from Bcfg2.Utils import safe_input -from Bcfg2.Client import matches_white_list, passes_black_list class Action(Bcfg2.Client.Tools.Tool): @@ -11,23 +10,6 @@ class Action(Bcfg2.Client.Tools.Tool): __handles__ = [('Action', None)] __req__ = {'Action': ['name', 'timing', 'when', 'command', 'status']} - def _action_allowed(self, action): - """ Return true if the given action is allowed to be run by - the whitelist or blacklist """ - if (Bcfg2.Options.setup.decision == 'whitelist' and - not matches_white_list(action, - Bcfg2.Options.setup.decision_list)): - self.logger.info("In whitelist mode: suppressing Action: %s" % - action.get('name')) - return False - if (Bcfg2.Options.setup.decision == 'blacklist' and - not passes_black_list(action, - Bcfg2.Options.setup.decision_list)): - self.logger.info("In blacklist mode: suppressing Action: %s" % - action.get('name')) - return False - return True - def RunAction(self, entry): """This method handles command execution and status return.""" shell = False @@ -76,7 +58,7 @@ class Action(Bcfg2.Client.Tools.Tool): states = dict() for action in bundle.findall("Action"): if action.get('timing') in ['post', 'both']: - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) return states @@ -87,7 +69,7 @@ class Action(Bcfg2.Client.Tools.Tool): for action in bundle.findall("Action"): if (action.get('timing') in ['post', 'both'] and action.get('when') != 'modified'): - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) return states diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index a7fcb6709..7200b0fc2 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -160,7 +160,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group") + if entry.get("name") in g[3] and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 5698f237a..332638de4 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,6 +4,8 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML +from Bcfg2.Compat import urlretrieve + # pylint: disable=C0103 noask = ''' @@ -37,6 +39,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): # noaskfile needs to live beyond __init__ otherwise file is removed self.noaskfile = tempfile.NamedTemporaryFile() self.noaskname = self.noaskfile.name + # for any pkg files downloaded + self.tmpfiles = [] try: self.noaskfile.write(noask) # flush admin file contents to disk @@ -45,6 +49,41 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.pkgtool[1]) except: # pylint: disable=W0702 self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1]) + self.origpkgtool = self.pkgtool + + def pkgmogrify(self, packages): + """ Take a list of pkg objects, check for a 'simplefile' attribute. + If present, insert a _sysv_pkg_path attribute to the package and + download the datastream format SYSV package to a temporary file. + """ + for pkg in packages: + if pkg.get('simplefile'): + tmpfile = tempfile.NamedTemporaryFile() + self.tmpfiles.append(tmpfile) + self.logger.info("Downloading %s to %s" % (pkg.get('url'), + tmpfile.name)) + urlretrieve(pkg.get('url'), tmpfile.name) + pkg.set('_sysv_pkg_path', tmpfile.name) + + def _get_package_command(self, packages): + """Override the default _get_package_command, replacing the attribute + 'url' if '_sysv_pkg_path' if necessary in the returned command + string + """ + if hasattr(self, 'origpkgtool'): + if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): + self.pkgtool = (self.pkgtool[0], ('%s %s', + ['_sysv_pkg_path', 'name'])) + else: + self.pkgtool = self.origpkgtool + + pkgcmd = super(SYSV, self)._get_package_command(packages) + self.logger.debug("Calling install command: %s" % pkgcmd) + return pkgcmd + + def Install(self, packages): + self.pkgmogrify(packages) + super(SYSV, self).Install(packages) def RefreshPackages(self): """Refresh memory hashes of packages.""" @@ -80,8 +119,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Package %s not installed" % entry.get("name")) else: - if (Bcfg2.Options.setup.quick or - entry.attrib.get('verify', 'true') == 'false'): + if Bcfg2.Options.setup.quick or \ + entry.attrib.get('verify', 'true') == 'false': return True rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name')) if rv.success: diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 027d91c71..3b60c8285 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,15 +13,25 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} + def get_svc_name(self, service): + """Append .service to name if name doesn't specify a unit type.""" + svc = service.get('name') + if svc.endswith(('.service', '.socket', '.device', '.mount', + '.automount', '.swap', '.target', '.path', + '.timer', '.snapshot', '.slice', '.scope')): + return svc + else: + return '%s.service' % svc + def get_svc_command(self, service, action): - return "/bin/systemctl %s %s.service" % (action, service.get('name')) + return "/bin/systemctl %s %s" % (action, self.get_svc_name(service)) def VerifyService(self, entry, _): """Verify Service status for entry.""" if entry.get('status') == 'ignore': return True - cmd = "/bin/systemctl status %s.service " % (entry.get('name')) + cmd = "/bin/systemctl status %s" % (self.get_svc_name(entry)) rv = self.cmd.run(cmd) if 'Loaded: error' in rv.stdout: diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 85839cf46..a8a80974a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -11,6 +11,7 @@ import yum.callbacks import yum.Errors import yum.misc import rpmUtils.arch +import rpmUtils.miscutils import Bcfg2.Client.XML import Bcfg2.Client.Tools import Bcfg2.Options @@ -146,14 +147,14 @@ class YUM(Bcfg2.Client.Tools.PkgTool): Bcfg2.Options.Option( cf=('YUM', 'verify_flags'), default=[], dest="yum_verify_flags", type=Bcfg2.Options.Types.comma_list, - help="YUM verify flags")] + help="YUM verify flags"), Bcfg2.Options.Option( - cf=('YUM', 'disabled_plugins'), default=[], - type=Bcfg2.Options.Types.comma_list, dest="yum_disabled_plugins", - help="YUM disabled plugins")] + cf=('YUM', 'disabled_plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, dest="yum_disabled_plugins", + help="YUM disabled plugins"), Bcfg2.Options.Option( - cf=('YUM', 'enabled_plugins'), default=[], - type=Bcfg2.Options.Types.comma_list, dest="yum_enabled_plugins", + cf=('YUM', 'enabled_plugins'), default=[], + type=Bcfg2.Options.Types.comma_list, dest="yum_enabled_plugins", help="YUM enabled plugins")] pkgtype = 'yum' @@ -254,12 +255,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): debuglevel = 0 if len(Bcfg2.Options.setup.yum_disabled_plugins) > 0: - rv.preconf.disabled_plugins= - Bcfg2.Options.setup.yum_disabled_plugins + rv.preconf.disabled_plugins = \ + Bcfg2.Options.setup.yum_disabled_plugins if len(Bcfg2.Options.setup.yum_enabled_plugins) > 0: - rv.preconf.enabled_plugins= - Bcfg2.Options.setup.yum_enabled_plugins + rv.preconf.enabled_plugins = \ + Bcfg2.Options.setup.yum_enabled_plugins # pylint: disable=E1121,W0212 try: @@ -660,7 +661,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra.get('release', 'any')) entry.set('current_version', "%s:%s-%s" % current_evr) entry.set('version', "%s:%s-%s" % wanted_evr) - if yum.compareEVR(current_evr, wanted_evr) == 1: + if rpmUtils.miscutils.compareEVR(current_evr, wanted_evr) == 1: entry.set("package_fail_action", "downgrade") else: entry.set("package_fail_action", "update") @@ -976,8 +977,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): nevra2string(build_yname(pkg.get('name'), inst))) continue status = self.instance_status[inst] - if (not status.get('installed', False) and - Bcfg2.Options.setup.yum_install_missing): + if not status.get('installed', False) and \ + Bcfg2.Options.setup.yum_install_missing: queue_pkg(pkg, inst, install_pkgs) elif (status.get('version_fail', False) and Bcfg2.Options.setup.yum_fix_version): diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index cd294db98..ae7fa3aed 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -129,6 +129,23 @@ class Tool(object): raise ToolInstantiationError("%s: %s not executable" % (self.name, filename)) + def _install_allowed(self, entry): + """ Return true if the given entry is allowed to be installed by + the whitelist or blacklist """ + if (Bcfg2.Options.setup.decision == 'whitelist' and + not Bcfg2.Client.matches_white_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In whitelist mode: suppressing Action: %s" % + entry.get('name')) + return False + if (Bcfg2.Options.setup.decision == 'blacklist' and + not Bcfg2.Client.passes_black_list( + entry, Bcfg2.Options.setup.decision_list)): + self.logger.info("In blacklist mode: suppressing Action: %s" % + entry.get('name')) + return False + return True + def BundleUpdated(self, bundle): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. @@ -587,7 +604,8 @@ class SvcTool(Tool): return for entry in bundle: - if not self.handlesEntry(entry): + if (not self.handlesEntry(entry) + or not self._install_allowed(entry)): continue estatus = entry.get('status') diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 073aa7694..5f4f15dcc 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -873,15 +873,21 @@ class Client(object): def GenerateStats(self): """Generate XML summary of execution statistics.""" + states = {} + for (item, val) in list(self.states.items()): + if not Bcfg2.Options.setup.only_important or \ + item.get('important', 'false').lower() == 'true': + states[item] = val + feedback = XML.Element("upload-statistics") stats = XML.SubElement(feedback, - 'Statistics', total=str(len(self.states)), + 'Statistics', total=str(len(states)), version='2.0', revision=self.config.get('revision', '-1')) - good_entries = [key for key, val in list(self.states.items()) if val] + good_entries = [key for key, val in list(states.items()) if val] good = len(good_entries) stats.set('good', str(good)) - if any(not val for val in list(self.states.values())): + if any(not val for val in list(states.values())): stats.set('state', 'dirty') else: stats.set('state', 'clean') @@ -890,8 +896,8 @@ class Client(object): for (data, ename) in [(self.modified, 'Modified'), (self.extra, "Extra"), (good_entries, "Good"), - ([entry for entry in self.states - if not self.states[entry]], "Bad")]: + ([entry for entry in states + if not states[entry]], "Bad")]: container = XML.SubElement(stats, ename) for item in data: item.set('qtext', '') diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 049236e03..b8a75a0c5 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -20,6 +20,7 @@ except ImportError: # urllib imports try: from urllib import quote_plus + from urllib import urlretrieve from urlparse import urljoin, urlparse from urllib2 import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ @@ -27,7 +28,8 @@ try: except ImportError: from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, urlopen + HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ + urlopen, urlretrieve from urllib.error import HTTPError, URLError try: diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 982a299c0..f5b5d16aa 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -8,6 +8,7 @@ import Bcfg2.Options try: import django + import django.core.management import django.conf HAS_DJANGO = True except ImportError: diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 12c9cdaa8..90b9f0ec7 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -1,3 +1,4 @@ +import os import sys import atexit import daemon @@ -5,13 +6,12 @@ import logging import time import threading -# pylint: disable=E0611 from lockfile import LockFailed, LockTimeout +# pylint: disable=E0611 try: - from lockfile.pidlockfile import PIDLockFile - from lockfile import Error as PIDFileError + from daemon.pidfile import TimeoutPIDLockFile except ImportError: - from daemon.pidlockfile import PIDLockFile, PIDFileError + from daemon.pidlockfile import TimeoutPIDLockFile # pylint: enable=E0611 import Bcfg2.Logger @@ -30,7 +30,7 @@ class ReportingError(Exception): class ReportingStoreThread(threading.Thread): """Thread for calling the storage backend""" def __init__(self, interaction, storage, group=None, target=None, - name=None, args=(), kwargs=None): + name=None, semaphore=None, args=(), kwargs=None): """Initialize the thread with a reference to the interaction as well as the storage engine to use""" threading.Thread.__init__(self, group, target, name, args, @@ -38,26 +38,37 @@ class ReportingStoreThread(threading.Thread): self.interaction = interaction self.storage = storage self.logger = logging.getLogger('bcfg2-report-collector') + self.semaphore = semaphore def run(self): """Call the database storage procedure (aka import)""" try: - start = time.time() - self.storage.import_interaction(self.interaction) - self.logger.info("Imported interaction for %s in %ss" % - (self.interaction.get('hostname', '<unknown>'), - time.time() - start)) - except: - #TODO requeue? - self.logger.error("Unhandled exception in import thread %s" % - sys.exc_info()[1]) + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', + '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + sys.exc_info()[1]) + finally: + if self.semaphore: + self.semaphore.release() class ReportingCollector(object): """The collecting process for reports""" options = [Bcfg2.Options.Common.reporting_storage, Bcfg2.Options.Common.reporting_transport, - Bcfg2.Options.Common.daemon] + Bcfg2.Options.Common.daemon, + Bcfg2.Options.Option( + '--max-children', dest="children", + cf=('reporting', 'max_children'), type=int, + default=0, + help='Maximum number of children for the reporting collector')] def __init__(self): """Setup the collector. This may be called by the daemon or though @@ -67,6 +78,10 @@ class ReportingCollector(object): self.children = [] self.cleanup_threshold = 25 + if Bcfg2.Options.setup.children > 0: + self.semaphore = threading.Semaphore( + value=Bcfg2.Options.setup.children) + if Bcfg2.Options.setup.debug: level = logging.DEBUG elif Bcfg2.Options.setup.verbose: @@ -113,25 +128,31 @@ class ReportingCollector(object): if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") + self.context.pidfile = TimeoutPIDLockFile( + Bcfg2.Options.setup.daemon, acquire_timeout=5) + # Attempt to ensure lockfile is able to be created and not stale try: - self.context.pidfile = PIDLockFile(Bcfg2.Options.setup.daemon) - self.context.open() + self.context.pidfile.acquire() except LockFailed: self.logger.error("Failed to daemonize: %s" % sys.exc_info()[1]) self.shutdown() return except LockTimeout: - self.logger.error("Failed to daemonize: " - "Failed to acquire lock on %s" % - self.setup['daemon']) - self.shutdown() - return - except PIDFileError: - self.logger.error("Error writing pid file: %s" % - sys.exc_info()[1]) - self.shutdown() - return + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except (OSError, TypeError): # No process with locked PID + self.context.pidfile.break_lock() + else: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + Bcfg2.Options.setup.daemon) + self.shutdown() + return + else: + self.context.pidfile.release() + + self.context.open() self.logger.info("Starting daemon") self.transport.start_monitor(self) @@ -141,7 +162,10 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - store_thread = ReportingStoreThread(interaction, self.storage) + if Bcfg2.Options.setup.children > 0: + self.semaphore.acquire() + store_thread = ReportingStoreThread(interaction, self.storage, + semaphore=self.semaphore) store_thread.start() self.children.append(store_thread) diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 406216861..96226c424 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -168,7 +168,7 @@ class DjangoORM(StorageBase): # TODO - vcs output act_dict['detail_type'] = PathEntry.DETAIL_UNUSED if path_type == 'directory' and entry.get('prune', 'false') == 'true': - unpruned_elist = [e.get('path') for e in entry.findall('Prune')] + unpruned_elist = [e.get('name') for e in entry.findall('Prune')] if unpruned_elist: act_dict['detail_type'] = PathEntry.DETAIL_PRUNED act_dict['details'] = "\n".join(unpruned_elist) @@ -367,10 +367,11 @@ class DjangoORM(StorageBase): def import_interaction(self, interaction): """Import the data into the backend""" try: - self._import_interaction(interaction) - except: - self.logger.error("Failed to import interaction: %s" % - traceback.format_exc().splitlines()[-1]) + try: + self._import_interaction(interaction) + except: + self.logger.error("Failed to import interaction: %s" % + traceback.format_exc().splitlines()[-1]) finally: self.logger.debug("%s: Closing database connection" % self.__class__.__name__) diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 2d96990b1..ae6f6731b 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -717,9 +717,6 @@ class PathEntry(SuccessEntry): def has_detail(self): return self.detail_type != PathEntry.DETAIL_UNUSED - def is_sensitive(self): - return self.detail_type == PathEntry.DETAIL_SENSITIVE - def is_diff(self): return self.detail_type == PathEntry.DETAIL_DIFF diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index b03d48045..c6e6df020 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -107,7 +107,7 @@ div.entry_list h3 { {{ item.details|syntaxhilight }} </div> {% else %} - {{ item.details }} + {{ item.details|linebreaks }} {% endif %} </div> {% endif %} diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 4a93e77e0..09aebc7fd 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -111,47 +111,58 @@ def filter_navigator(context): try: path = context['request'].META['PATH_INFO'] view, args, kwargs = resolve(path) + except (Resolver404, KeyError): + return dict() - # Strip any page limits and numbers - if 'page_number' in kwargs: - del kwargs['page_number'] - if 'page_limit' in kwargs: - del kwargs['page_limit'] - - # get a query string - qs = context['request'].GET.urlencode() - if qs: - qs = '?' + qs - - filters = [] - for filter in filter_list: - if filter == 'group': - continue - if filter in kwargs: - myargs = kwargs.copy() - del myargs[filter] + # Strip any page limits and numbers + if 'page_number' in kwargs: + del kwargs['page_number'] + if 'page_limit' in kwargs: + del kwargs['page_limit'] + + # get a query string + qs = context['request'].GET.urlencode() + if qs: + qs = '?' + qs + + filters = [] + for filter in filter_list: + if filter == 'group': + continue + if filter in kwargs: + myargs = kwargs.copy() + del myargs[filter] + try: filters.append((filter, reverse(view, args=args, kwargs=myargs) + qs)) - filters.sort(key=lambda x: x[0]) - - myargs = kwargs.copy() - selected = True - if 'group' in myargs: - del myargs['group'] - selected = False - groups = [('---', - reverse(view, args=args, kwargs=myargs) + qs, - selected)] - for group in Group.objects.values('name'): + except NoReverseMatch: + pass + filters.sort(key=lambda x: x[0]) + + myargs = kwargs.copy() + selected = True + if 'group' in myargs: + del myargs['group'] + selected = False + + groups = [] + try: + groups.append(('---', + reverse(view, args=args, kwargs=myargs) + qs, + selected)) + except NoReverseMatch: + pass + + for group in Group.objects.values('name'): + try: myargs['group'] = group['name'] groups.append((group['name'], reverse(view, args=args, kwargs=myargs) + qs, group['name'] == kwargs.get('group', ''))) + except NoReverseMatch: + pass - return {'filters': filters, 'groups': groups} - except (Resolver404, NoReverseMatch, ValueError, KeyError): - pass - return dict() + return {'filters': filters, 'groups': groups} def _subtract_or_na(mdict, x, y): diff --git a/src/lib/Bcfg2/Reporting/utils.py b/src/lib/Bcfg2/Reporting/utils.py index 0d394fcd8..694f38824 100755 --- a/src/lib/Bcfg2/Reporting/utils.py +++ b/src/lib/Bcfg2/Reporting/utils.py @@ -96,12 +96,12 @@ def filteredUrls(pattern, view, kwargs=None, name=None): tail = mtail.group(1) pattern = pattern[:len(pattern) - len(tail)] for filter in ('/state/(?P<state>\w+)', - '/group/(?P<group>[\w\-\.]+)', - '/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)', - '/server/(?P<server>[\w\-\.]+)', - '/server/(?P<server>[\w\-\.]+)/(?P<state>[A-Za-z]+)', - '/server/(?P<server>[\w\-\.]+)/group/(?P<group>[\w\-\.]+)', - '/server/(?P<server>[\w\-\.]+)/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)'): + '/group/(?P<group>[^/]+)', + '/group/(?P<group>[^/]+)/(?P<state>[A-Za-z]+)', + '/server/(?P<server>[^/]+)', + '/server/(?P<server>[^/]+)/(?P<state>[A-Za-z]+)', + '/server/(?P<server>[^/]+)/group/(?P<group>[^/]+)', + '/server/(?P<server>[^/]+)/group/(?P<group>[^/]+)/(?P<state>[A-Za-z]+)'): results += [(pattern + filter + tail, view, kwargs)] return results diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 769addf55..e138c57e4 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -1,5 +1,6 @@ """ The core of the builtin Bcfg2 server. """ +import os import sys import time import socket @@ -85,20 +86,30 @@ class BuiltinCore(NetworkCore): def _daemonize(self): """ Open :attr:`context` to drop privileges, write the PID file, and daemonize the server core. """ + # Attempt to ensure lockfile is able to be created and not stale try: - self.context.open() - self.logger.info("%s daemonized" % self.name) - return True + self.context.pidfile.acquire() except LockFailed: err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False except LockTimeout: - err = sys.exc_info()[1] - self.logger.error("Failed to daemonize %s: Failed to acquire lock " - "on %s" % (self.name, - Bcfg2.Options.setup.daemon)) - return False + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except (OSError, TypeError): # No process with locked PID + self.context.pidfile.break_lock() + else: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: Failed to acquire" + "lock on %s" % (self.name, + Bcfg2.Options.setup.daemon)) + return False + else: + self.context.pidfile.release() + + self.context.open() + self.logger.info("%s daemonized" % self.name) + return True def _run(self): """ Create :attr:`server` to start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 892f2832a..bc305e47a 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -84,7 +84,7 @@ def close_db_connection(func): if self._database_available: # pylint: disable=W0212 from django import db self.logger.debug("%s: Closing database connection" % - threading.current_thread().name) + threading.current_thread().getName()) db.close_connection() return rv @@ -783,13 +783,13 @@ class Core(object): for plug in self.plugins_by_type(Threaded): plug.start_threads() + + self.block_for_fam_events() + self._block() except: self.shutdown() raise - self.block_for_fam_events() - self._block() - def _run(self): """ Start up the server; this method should return immediately. This must be overridden by a core diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index d0fd70c5c..8e0dd2efe 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -238,6 +238,8 @@ class FileMonitor(Debuggable): self.handles[event.requestID])) try: self.handles[event.requestID].HandleEvent(event) + except KeyboardInterrupt: + raise except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Error in handling of event %s for %s: %s" % diff --git a/src/lib/Bcfg2/Server/Lint/AWSTags.py b/src/lib/Bcfg2/Server/Lint/AWSTags.py index 25ad4ef61..c6d7a3a30 100644 --- a/src/lib/Bcfg2/Server/Lint/AWSTags.py +++ b/src/lib/Bcfg2/Server/Lint/AWSTags.py @@ -9,6 +9,7 @@ import Bcfg2.Server.Lint class AWSTags(Bcfg2.Server.Lint.ServerPlugin): """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags <server-plugins-connectors-awstags>` patterns for validity. """ + __serverplugin__ = 'AWSTags' def Run(self): cfg = self.core.plugins['AWSTags'].config diff --git a/src/lib/Bcfg2/Server/Lint/Bundler.py b/src/lib/Bcfg2/Server/Lint/Bundler.py index aee15cb5d..576e157ad 100644 --- a/src/lib/Bcfg2/Server/Lint/Bundler.py +++ b/src/lib/Bcfg2/Server/Lint/Bundler.py @@ -7,6 +7,7 @@ from Bcfg2.Server.Lint import ServerPlugin class Bundler(ServerPlugin): """ Perform various :ref:`Bundler <server-plugins-structures-bundler>` checks. """ + __serverplugin__ = 'Bundler' def Run(self): self.missing_bundles() diff --git a/src/lib/Bcfg2/Server/Lint/Cfg.py b/src/lib/Bcfg2/Server/Lint/Cfg.py index 7716cd5c7..13b04a6b8 100644 --- a/src/lib/Bcfg2/Server/Lint/Cfg.py +++ b/src/lib/Bcfg2/Server/Lint/Cfg.py @@ -10,6 +10,7 @@ from Bcfg2.Server.Plugins.Cfg import CfgGenerator class Cfg(ServerPlugin): """ warn about Cfg issues """ + __serverplugin__ = 'Cfg' def Run(self): for basename, entry in list(self.core.plugins['Cfg'].entries.items()): diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index fc4506c12..fbe84de87 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -93,13 +93,21 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for info.xml files"), Bcfg2.Options.Option( - cf=("Comments", "probe_keywords"), + cf=("Comments", "probes_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for probes"), Bcfg2.Options.Option( - cf=("Comments", "probe_comments"), + cf=("Comments", "probes_comments"), type=Bcfg2.Options.Types.comma_list, default=[], - help="Required comments for probes")] + help="Required comments for probes"), + Bcfg2.Options.Option( + cf=("Comments", "metadata_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for metadata files"), + Bcfg2.Options.Option( + cf=("Comments", "metadata_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for metadata files")] def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) @@ -248,7 +256,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): rtype = "jinja2" elif isinstance(entry, CfgInfoXML): self.check_xml(entry.infoxml.name, - entry.infoxml.pnode.data, + entry.infoxml.xdata, "infoxml") continue if rtype: diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index a2581e70b..a2581e70b 100755..100644 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py diff --git a/src/lib/Bcfg2/Server/Lint/GroupPatterns.py b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py index d8142cab9..8ddb9e796 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py @@ -13,6 +13,7 @@ class GroupPatterns(ServerPlugin): :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for each pattern, and catching exceptions and presenting them as ``bcfg2-lint`` errors.""" + __serverplugin__ = 'GroupPatterns' def Run(self): cfg = self.core.plugins['GroupPatterns'].config diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index 4b1513a11..950a86f01 100644 --- a/src/lib/Bcfg2/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -15,6 +15,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): * Paranoid mode disabled in an ``info.xml`` file; * Required attributes missing from ``info.xml`` """ + __serverplugin__ = 'Cfg' options = Bcfg2.Server.Lint.ServerPlugin.options + [ Bcfg2.Options.Common.default_paranoid, diff --git a/src/lib/Bcfg2/Server/Lint/Jinja2.py b/src/lib/Bcfg2/Server/Lint/Jinja2.py index 333249cc2..333249cc2 100755..100644 --- a/src/lib/Bcfg2/Server/Lint/Jinja2.py +++ b/src/lib/Bcfg2/Server/Lint/Jinja2.py diff --git a/src/lib/Bcfg2/Server/Lint/Metadata.py b/src/lib/Bcfg2/Server/Lint/Metadata.py index 248b1610c..e445892d1 100644 --- a/src/lib/Bcfg2/Server/Lint/Metadata.py +++ b/src/lib/Bcfg2/Server/Lint/Metadata.py @@ -15,6 +15,7 @@ class Metadata(ServerPlugin): * Multiple default groups or a default group that isn't a profile group. """ + __serverplugin__ = 'Metadata' def Run(self): self.nested_clients() diff --git a/src/lib/Bcfg2/Server/Lint/Pkgmgr.py b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py index 3f0b9477c..eed6d4c19 100644 --- a/src/lib/Bcfg2/Server/Lint/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py @@ -12,6 +12,7 @@ class Pkgmgr(ServerlessPlugin): """ Find duplicate :ref:`Pkgmgr <server-plugins-generators-pkgmgr>` entries with the same priority. """ + __serverplugin__ = 'Pkgmgr' def Run(self): pset = set() diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py index 5a80a5884..a437c1318 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py @@ -62,7 +62,7 @@ class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin): # finally, check for executable permissions in info.xml for entry in entryset.entries.values(): if isinstance(entry, CfgInfoXML): - for pinfo in entry.infoxml.pnode.data.xpath("//FileInfo"): + for pinfo in entry.infoxml.xdata.xpath("//FileInfo/Info"): try: mode = int( pinfo.get("mode", diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py index a952da724..9d05516f1 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -20,6 +20,7 @@ class TemplateHelper(ServerPlugin): * Bogus symbols listed in ``__export__``, including symbols that don't exist, that are reserved, or that start with underscores. """ + __serverplugin__ = 'TemplateHelper' def __init__(self, *args, **kwargs): ServerPlugin.__init__(self, *args, **kwargs) diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 0b3f1e24d..cab5d248d 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -18,7 +18,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): options = Bcfg2.Server.Lint.ServerlessPlugin.options + [ Bcfg2.Options.PathOption( "--schema", cf=("Validate", "schema"), - default="/usr/share/bcfg2/schema", + default="/usr/share/bcfg2/schemas", help="The full path to the XML schema files")] def __init__(self, *args, **kwargs): diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 9b3e6ece2..526bdf159 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -14,6 +14,7 @@ import Bcfg2.Options import Bcfg2.Server.Core import Bcfg2.Server.Plugins from Bcfg2.Compat import walk_packages +from Bcfg2.Options import _debug def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -48,6 +49,11 @@ def get_termsize(): class Plugin(object): """ Base class for all bcfg2-lint plugins """ + #: Name of the matching server plugin or None if there is no + #: matching one. If this is None the lint plugin will only loaded + #: by default if the matching server plugin is enabled, too. + __serverplugin__ = None + options = [Bcfg2.Options.Common.repository] def __init__(self, errorhandler=None, files=None): @@ -291,19 +297,41 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 class LintPluginAction(Bcfg2.Options.ComponentAction): - """ We want to load all lint plugins that pertain to server - plugins. In order to do this, we hijack the __call__() method of - this action and add all of the server plugins on the fly """ - + """ Option parser action to load lint plugins """ bases = ['Bcfg2.Server.Lint'] - def __call__(self, parser, namespace, values, option_string=None): - plugins = getattr(Bcfg2.Options.setup, "plugins", []) - for lint_plugin in walk_packages(path=__path__): - if lint_plugin[1] in plugins: - values.append(lint_plugin[1]) - Bcfg2.Options.ComponentAction.__call__(self, parser, namespace, values, - option_string) + +class LintPluginOption(Bcfg2.Options.Option): + """ Option class for the lint_plugins """ + + def early_parsing_hook(self, namespace): + """ + We want a usefull default for the enabled lint plugins. + Therfore we use all importable plugins, that either pertain + with enabled server plugins or that has no matching plugin. + """ + + plugins = [p.__name__ for p in namespace.plugins] + for loader, name, _is_pkg in walk_packages(path=__path__): + try: + module = loader.find_module(name).load_module(name) + plugin = getattr(module, name) + if plugin.__serverplugin__ is None or \ + plugin.__serverplugin__ in plugins: + _debug("Automatically adding lint plugin %s" % + plugin.__name__) + self.default.append(plugin.__name__) + except ImportError: + pass + + +class _EarlyOptions(object): + """ We need the server.plugins options in an early parsing hook + for determining the default value for the lint_plugins. So we + create a component that is parsed before the other options. """ + + parse_first = True + options = [Bcfg2.Options.Common.plugins] class CLI(object): @@ -313,7 +341,7 @@ class CLI(object): '--lint-config', default='/etc/bcfg2-lint.conf', action=Bcfg2.Options.ConfigFileAction, help='Specify bcfg2-lint configuration file'), - Bcfg2.Options.Option( + LintPluginOption( "--lint-plugins", cf=('lint', 'plugins'), default=[], type=Bcfg2.Options.Types.comma_list, action=LintPluginAction, help='bcfg2-lint plugin list'), @@ -328,28 +356,11 @@ class CLI(object): def __init__(self): parser = Bcfg2.Options.get_parser( description="Manage a running Bcfg2 server", - components=[self]) + components=[self, _EarlyOptions]) parser.parse() self.logger = logging.getLogger(parser.prog) - # automatically add Lint plugins for loaded server plugins - for plugin in Bcfg2.Options.setup.plugins: - try: - Bcfg2.Options.setup.lint_plugins.append( - getattr( - __import__("Bcfg2.Server.Lint.%s" % plugin.__name__, - fromlist=[plugin.__name__]), - plugin.__name__)) - self.logger.debug("Automatically adding lint plugin %s" % - plugin.__name__) - except ImportError: - # no lint plugin for this server plugin - self.logger.debug("No lint plugin for %s" % plugin.__name__) - except AttributeError: - self.logger.error("Failed to load plugin %s: %s" % - (plugin.__name__, sys.exc_info()[1])) - self.logger.debug("Running lint with plugins: %s" % [p.__name__ for p in Bcfg2.Options.setup.lint_plugins]) @@ -428,9 +439,9 @@ class CLI(object): def run_server_plugins(self): """ run plugins that require a running server to run """ core = Bcfg2.Server.Core.Core() - core.load_plugins() - core.block_for_fam_events(handle_events=True) try: + core.load_plugins() + core.block_for_fam_events(handle_events=True) self.logger.debug("Running server plugins: %s" % [p.__name__ for p in self.serverplugins]) for plugin in self.serverplugins: diff --git a/src/lib/Bcfg2/Server/Plugins/ACL.py b/src/lib/Bcfg2/Server/Plugins/ACL.py index 1c1e54312..37f51a2a1 100644 --- a/src/lib/Bcfg2/Server/Plugins/ACL.py +++ b/src/lib/Bcfg2/Server/Plugins/ACL.py @@ -62,6 +62,7 @@ def ip_matches(ip, entry): class IPACLFile(Bcfg2.Server.Plugin.XMLFileBacked): """ representation of ACL ip.xml, for IP-based ACLs """ + __identifier__ = None actions = dict(Allow=True, Deny=False, Defer=None) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 3d5c68e3f..cfabd8457 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -102,7 +102,8 @@ class AptSource(Source): bdeps[barch][pkgname] = [] brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: - self.essentialpkgs.add(pkgname) + if words[1].strip() == 'yes': + self.essentialpkgs.add(pkgname) elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 24db2963d..67ada2399 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -199,6 +199,9 @@ class Source(Debuggable): # pylint: disable=R0902 #: The "version" attribute from :attr:`xsource` self.version = xsource.get('version', '') + #: The "name" attribute from :attr:`xsource` + self.name = xsource.get('name', None) + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -274,11 +277,11 @@ class Source(Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch) + arch=arch, debsrc=self.debsrc) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch)] + arch=arch, debsrc=self.debsrc)] for setting in usettings: if not self.rawurl: @@ -286,6 +289,7 @@ class Source(Debuggable): # pylint: disable=R0902 else: setting['baseurl'] = self.rawurl setting['url'] = baseurl % setting + setting['name'] = self.get_repo_name(setting) self.url_map.extend(usettings) @property @@ -353,7 +357,7 @@ class Source(Debuggable): # pylint: disable=R0902 if os.path.exists(self.cachefile): try: self.load_state() - except: + except (OSError, cPickle.UnpicklingError): err = sys.exc_info()[1] self.logger.error("Packages: Cachefile %s load failed: %s" % (self.cachefile, err)) @@ -388,8 +392,10 @@ class Source(Debuggable): # pylint: disable=R0902 doing other operations that require repository names. This function tries several approaches: - #. First, if the map contains a ``component`` key, use that as - the name. + #. First, if the source element containts a ``name`` attribute, + use that as the name. + #. If the map contains a ``component`` key, use that as the + name. #. If not, then try to match the repository URL against :attr:`Bcfg2.Server.Plugins.Packages.Source.REPO_RE`. If that succeeds, use the first matched group; additionally, @@ -419,6 +425,9 @@ class Source(Debuggable): # pylint: disable=R0902 :type url_map: dict :returns: string - the name of the repository. """ + if self.name: + return self.name + if url_map['component']: rname = url_map['component'] else: |