From d398edf58679413af7ec85a4828368a2c6c10248 Mon Sep 17 00:00:00 2001 From: igorwidlinski Date: Thu, 4 Jul 2013 10:05:57 -0700 Subject: Stop adding chkconfig service to every run level Let chkconfig assign default runlevels. --- src/lib/Bcfg2/Client/Tools/Chkconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index edcc86b85..4833f3f68 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -89,7 +89,7 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): if bootstatus is not None: if bootstatus == 'on': # make sure service is enabled on boot - bootcmd = '/sbin/chkconfig %s %s --level 0123456' % \ + bootcmd = '/sbin/chkconfig %s %s' % \ (entry.get('name'), bootstatus) elif bootstatus == 'off': # make sure service is disabled on boot -- cgit v1.2.3-1-g7c22 From accbb1831ae88e57baabf01a783fe7a8ee0a0ccb Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 9 Jul 2013 08:49:54 -0400 Subject: Core: apply FAM blocking option to all commands that start a server --- src/lib/Bcfg2/Server/Core.py | 26 +++++++++++++++++++++----- src/sbin/bcfg2-info | 2 +- src/sbin/bcfg2-lint | 2 +- src/sbin/bcfg2-test | 2 +- tools/bcfg2-profile-templates.py | 2 +- tools/bcfg2_local.py | 2 +- 6 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index ecd68e1e4..a22531d2d 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -728,6 +728,26 @@ class BaseCore(object): self.setup.reparse() self.metadata_cache.expire() + def block_for_fam_events(self, handle_events=False): + """ Block until all fam events have been handleed, optionally + handling events as well. (Setting ``handle_events=True`` is + useful for local server cores that don't spawn an event + handling thread.)""" + slept = 0 + log_interval = 3 + if handle_events: + self.fam.handle_events_in_interval(1) + slept += 1 + if self.setup['fam_blocking']: + time.sleep(1) + slept += 1 + while self.fam.pending() != 0: + time.sleep(1) + slept += 1 + if slept % log_interval == 0: + self.logger.debug("Sleeping to handle FAM events...") + self.logger.debug("Slept %s seconds while handling FAM events" % slept) + def run(self): """ Run the server core. This calls :func:`_daemonize`, :func:`_run`, starts the :attr:`fam_thread`, and calls @@ -777,13 +797,9 @@ class BaseCore(object): self.shutdown() raise - if self.setup['fam_blocking']: - time.sleep(1) - while self.fam.pending() != 0: - time.sleep(1) - if self.debug_flag: self.set_debug(None, self.debug_flag) + self.block_for_fam_events() self._block() def _daemonize(self): diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 406e7e003..cc9b03b80 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -717,7 +717,7 @@ Bcfg2 client itself.""") def run(self, args): # pylint: disable=W0221 try: self.load_plugins() - self.fam.handle_events_in_interval(1) + self.block_for_fam_events(handle_events=True) if args: self.onecmd(" ".join(args)) else: diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index ab3b6450f..9ceb1dd04 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -73,7 +73,7 @@ def load_server(setup): """ load server """ core = Bcfg2.Server.Core.BaseCore(setup) core.load_plugins() - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) return core diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index a7a6781a9..7c38a65d8 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -157,7 +157,7 @@ def get_core(setup): """ Get a server core, with events handled """ core = Bcfg2.Server.Core.BaseCore(setup) core.load_plugins() - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) return core diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py index f4069e454..2b0ca6d63 100755 --- a/tools/bcfg2-profile-templates.py +++ b/tools/bcfg2-profile-templates.py @@ -67,7 +67,7 @@ def main(): logger.info("Bcfg2 server core loaded") core.load_plugins() logger.debug("Plugins loaded") - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) logger.debug("Repository events processed") if setup['args']: diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index 8c164e52e..3c90a3ea5 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -20,7 +20,7 @@ class LocalCore(BaseCore): Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) setup['syslog'], setup['logging'] = saved self.load_plugins() - self.fam.handle_events_in_interval(0.1) + self.block_for_fam_events(handle_events=True) def _daemonize(self): return True -- cgit v1.2.3-1-g7c22 From 217b8c75e3f5163debb56150c27440b5268e3542 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 9 Jul 2013 14:08:00 -0400 Subject: Core: Gracefully handle failure to add client without default group --- src/lib/Bcfg2/Server/Core.py | 7 ++++++- src/lib/Bcfg2/Server/Plugins/Metadata.py | 9 ++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index a22531d2d..1291584b0 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -858,7 +858,12 @@ class BaseCore(object): imd = self.metadata_cache.get(client_name, None) if not imd: self.logger.debug("Building metadata for %s" % client_name) - imd = self.metadata.get_initial_metadata(client_name) + try: + imd = self.metadata.get_initial_metadata(client_name) + except MetadataConsistencyError: + self.critical_error( + "Client metadata resolution error for %s: %s" % + (client_name, sys.exc_info()[1])) connectors = self.plugins_by_type(Connector) for conn in connectors: grps = conn.get_additional_groups(imd) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 213f29cd0..67fd4c93d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1142,9 +1142,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, require_public=False) profile = _add_group(pgroup) else: - msg = "Cannot add new client %s; no default group set" % client - self.logger.error(msg) - raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError( + "Cannot add new client %s; no default group set" % client) for cgroup in self.clientgroups.get(client, []): if cgroup in groups: @@ -1167,8 +1166,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, try: bundles.update(self.groups[group].bundles) except KeyError: - self.logger.warning("%s: %s is a member of undefined group %s" - % (self.name, client, group)) + self.logger.warning("%s: %s is a member of undefined group %s" % + (self.name, client, group)) aliases = self.raliases.get(client, set()) addresses = self.raddresses.get(client, set()) -- cgit v1.2.3-1-g7c22 From 149ac6a932813e4f5b9d2e58a84570861a33711c Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 9 Jul 2013 14:41:56 -0400 Subject: Metadata: fixed long line --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 67fd4c93d..e8962d707 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1166,8 +1166,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, try: bundles.update(self.groups[group].bundles) except KeyError: - self.logger.warning("%s: %s is a member of undefined group %s" % - (self.name, client, group)) + self.logger.warning("%s: %s is a member of undefined group %s" + % (self.name, client, group)) aliases = self.raliases.get(client, set()) addresses = self.raddresses.get(client, set()) -- cgit v1.2.3-1-g7c22 From 6b0b45842b138a928f7d5e395f7a21f9ad932f2a Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Wed, 10 Jul 2013 10:09:24 -0500 Subject: Copy xml elements before appending to tree. This prevents modified entries from being added --- src/lib/Bcfg2/Client/Frame.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 3254da9e9..1b26450a6 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -1,6 +1,7 @@ """ Frame is the Client Framework that verifies and installs entries, and generates statistics. """ +import copy import time import fnmatch import logging @@ -522,7 +523,7 @@ class Frame(object): container = Bcfg2.Client.XML.SubElement(stats, ename) for item in data: item.set('qtext', '') - container.append(item) + container.append(copy.deepcopy(item)) item.text = None timeinfo = Bcfg2.Client.XML.Element("OpStamps") -- cgit v1.2.3-1-g7c22 From d1a6f9dcc97e6fe05707b40364b4bedb3c54d14a Mon Sep 17 00:00:00 2001 From: Duncan Hutty Date: Thu, 11 Jul 2013 10:33:54 -0400 Subject: repoint some trac links to github --- doc/appendix/tools.txt | 2 +- doc/server/plugins/generators/nagiosgen.txt | 2 +- doc/server/plugins/index.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/appendix/tools.txt b/doc/appendix/tools.txt index 1d7a8dd90..92bde683b 100644 --- a/doc/appendix/tools.txt +++ b/doc/appendix/tools.txt @@ -11,4 +11,4 @@ can help you to maintain your Bcfg2 configuration, to make the initial setup easier, or to do some other tasks. -http://trac.mcs.anl.gov/projects/bcfg2/browser/tools +https://github.com/Bcfg2/bcfg2/tree/maint/tools diff --git a/doc/server/plugins/generators/nagiosgen.txt b/doc/server/plugins/generators/nagiosgen.txt index ee99b2dc1..0ae922fa3 100644 --- a/doc/server/plugins/generators/nagiosgen.txt +++ b/doc/server/plugins/generators/nagiosgen.txt @@ -8,7 +8,7 @@ NagiosGen This page describes the installation and use of the `NagiosGen`_ plugin. -.. _NagiosGen: http://trac.mcs.anl.gov/projects/bcfg2/browser/src/lib/Server/Plugins/NagiosGen.py +.. _NagiosGen: https://github.com/Bcfg2/bcfg2/blob/maint/src/lib/Bcfg2/Server/Plugins/NagiosGen.py Update ``/etc/bcfg2.conf``, adding NagiosGen to plugins:: diff --git a/doc/server/plugins/index.txt b/doc/server/plugins/index.txt index 4f2b484ac..f3d6daa73 100644 --- a/doc/server/plugins/index.txt +++ b/doc/server/plugins/index.txt @@ -31,7 +31,7 @@ Default Plugins The `Bcfg2 repository`_ contains the all plugins currently distributed with Bcfg2. -.. _Bcfg2 repository: http://trac.mcs.anl.gov/projects/bcfg2/browser/src/lib/Server/Plugins +.. _Bcfg2 repository: https://github.com/Bcfg2/bcfg2/tree/maint/src/lib/Bcfg2/Server/Plugins Metadata (Grouping) ------------------- -- cgit v1.2.3-1-g7c22 From ba6e31eba192d6fcfc31d2f8ab2ad811a1322f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Fri, 12 Jul 2013 00:19:38 -0400 Subject: Don't crash when a file diff contains utf-8 chars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes a crash when the client tries to render the diff of a file which includes unicode characters. This change assumes that utf-8 is the default encoding, which looking at the rest of the code appears to be a safe bet. Signed-off-by: Stéphane Graber --- src/lib/Bcfg2/Client/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 25603186e..6d1cb9d40 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -21,6 +21,9 @@ def prompt(msg): try: ans = input(msg) return ans in ['y', 'Y'] + except UnicodeEncodeError: + ans = input(msg.encode('utf-8')) + return ans in ['y', 'Y'] except EOFError: # handle ^C on rhel-based platforms raise SystemExit(1) -- cgit v1.2.3-1-g7c22 From 156212c6fc294382198840e143f0867e0536601e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 15 Jul 2013 10:33:25 -0400 Subject: Encryption: reduce noise from encryption failures when decrypt=lax --- src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py | 2 +- src/lib/Bcfg2/Server/Plugins/Properties.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index c7b62f352..c0a3036a9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -230,7 +230,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if strict: raise PluginExecutionError(msg) else: - self.logger.warning(msg) + self.logger.info(msg) Index.__doc__ = StructFile.Index.__doc__ def _decrypt(self, element): diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index e97f66675..89f2d21ff 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -223,7 +223,7 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): if strict: raise PluginExecutionError(msg) else: - LOGGER.warning(msg) + LOGGER.info(msg) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ def _decrypt(self, element): -- cgit v1.2.3-1-g7c22 From 2217fa6295070f137988006c4bb00d25dfc0cb5e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 15 Jul 2013 15:32:21 -0400 Subject: Read-only yum cache This makes the yum cache read-only so that bcfg2-yum-helper cannot update the cache on the fly, which should help avoid locking issues with the yum caches that can cause client runs to fail. It also makes the Packages plugin behave more consistently, since use of yum libraries won't cause the cache to be refreshed at random times on the fly, but rather more predictably as with the Apt cache or the yum cache without using yum libraries. Unlike those two cases, though, the caches will not all be downloaded initially, but rather opportunistically as needed. In order for this to work, the Bcfg2 server must not run as root. Root ignores the 'w' permissions bit, so the cache cannot be made read-only. --- .../Bcfg2/Server/Plugins/Packages/Collection.py | 4 + src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 60 ++++++++++----- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 85 +++++++++++++++++++--- src/sbin/bcfg2-yum-helper | 23 ++++++ 4 files changed, 145 insertions(+), 27 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index b25cb0fc4..39c51f351 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -614,6 +614,10 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): self.filter_unknown(unknown) return packages, unknown + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, + list.__repr__(self)) + def get_collection_class(source_type): """ Given a source type, determine the class of Collection object diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 4608bcca5..55787681f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -53,6 +53,7 @@ The Yum Backend import os import re import sys +import stat import copy import errno import socket @@ -282,13 +283,15 @@ class YumCollection(Collection): #: for cached yum metadata self.cachefile = os.path.join(self.cachepath, "cache-%s" % self.cachekey) - if not os.path.exists(self.cachefile): - os.mkdir(self.cachefile) #: The path to the server-side config file used when #: resolving packages with the Python yum libraries self.cfgfile = os.path.join(self.cachefile, "yum.conf") - self.write_config() + + if not os.path.exists(self.cachefile): + self.debug_log("Creating common cache %s" % self.cachefile) + os.mkdir(self.cachefile) + self.setup_data() else: self.cachefile = None @@ -924,6 +927,28 @@ class YumCollection(Collection): "output: %s" % err) raise + def _set_cache_writeable(self, writeable): + """ Set the writeability of the yum cache. + + :param writeable: If True, the cache will be made writeable. + If False, the cache will be made read-only. + :type writeable: bool + """ + fmode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH + if writeable: + self.debug_log("Packages: Making cache %s writeable" % + self.cachefile) + fmode |= stat.S_IWUSR + else: + self.debug_log("Packages: Making cache %s read-only" % + self.cachefile) + dmode = fmode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + for root, dirs, files in os.walk(self.cachefile): + for dname in dirs: + os.chmod(os.path.join(root, dname), dmode) + for fname in files: + os.chmod(os.path.join(root, fname), fmode) + def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called when sources are loaded or reloaded by @@ -931,11 +956,11 @@ class YumCollection(Collection): If the builtin yum parsers are in use, this defers to :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.setup_data`. - If using the yum Python libraries, this cleans up cached yum - metadata, regenerates the server-side yum config (in order to - catch any new sources that have been added to this server), - and then cleans up cached yum metadata again, in case the new - config has any preexisting cache. + If using the yum Python libraries, this makes the cache + writeable, cleans up cached yum metadata, regenerates the + server-side yum config (in order to catch any new sources that + have been added to this server), regenerates the yum cache, + and then sets the cache back to read-only. :param force_update: Ignore all local cache and setup data from its original upstream sources (i.e., @@ -945,24 +970,25 @@ class YumCollection(Collection): if not self.use_yum: return Collection.setup_data(self, force_update) + self._set_cache_writeable(True) if force_update: - # we call this twice: one to clean up data from the old - # config, and once to clean up data from the new config + # clean up data from the old config try: self.call_helper("clean") except ValueError: # error reported by call_helper pass - os.unlink(self.cfgfile) + if os.path.exists(self.cfgfile): + os.unlink(self.cfgfile) self.write_config() - if force_update: - try: - self.call_helper("clean") - except ValueError: - # error reported by call_helper - pass + try: + self.call_helper("makecache") + except ValueError: + # error reported by call_helper + pass + self._set_cache_writeable(False) class YumSource(Source): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index f93bd0932..aea447520 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,8 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ + MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -22,6 +23,52 @@ APT_CONFIG_DEFAULT = \ "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +class OnDemandDict(MutableMapping): + """ This maps a set of keys to a set of value-getting functions; + the values are populated on-the-fly by the functions as the values + are needed (and not before). This is used by + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data`; + see the docstring for that function for details on why. + + Unlike a dict, you should not specify values for for the righthand + side of this mapping, but functions that get values. E.g.: + + .. code-block:: python + + d = OnDemandDict(foo=load_foo, + bar=lambda: "bar"); + """ + + def __init__(self, **getters): + self._values = dict() + self._getters = dict(**getters) + + def __getitem__(self, key): + if key not in self._values: + self._values[key] = self._getters[key]() + return self._values[key] + + def __setitem__(self, key, getter): + self._getters[key] = getter + + def __delitem__(self, key): + del self._values[key] + del self._getters[key] + + def __len__(self): + return len(self._getters) + + def __iter__(self): + return iter(self._getters.keys()) + + def __str__(self): + rv = dict(self._values) + for key in self._getters.keys(): + if key not in rv: + rv[key] = 'unknown' + return str(rv) + + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -535,20 +582,38 @@ class Packages(Bcfg2.Server.Plugin.Plugin, def get_additional_data(self, metadata): """ Return additional data for the given client. This will be - a dict containing a single key, ``sources``, whose value is a - list of data returned from - :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, - namely, a list of - :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` - data. + an :class:`Bcfg2.Server.Plugins.Packages.OnDemandDict` + containing two keys: + + * ``sources``, whose value is a list of data returned from + :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, + namely, a list of + :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` + data; and + * ``get_config``, whose value is the + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_config` + function, which can be used to get the Packages config for + other systems. + + This uses an OnDemandDict instead of just a normal dict + because loading a source collection can be a fairly + time-consuming process, particularly for the first time. As a + result, when all metadata objects are built at once (such as + after the server is restarted, or far more frequently if + Metadata caching is disabled), this function would be a major + bottleneck if we tried to build all collections at the same + time. Instead, they're merely built on-demand. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ - collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data(), - get_config=self.get_config) + def get_sources(): + return self.get_collection(metadata).get_additional_data + + return OnDemandDict( + sources=get_sources, + get_config=lambda: self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 4ef531d39..473214d89 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -193,6 +193,20 @@ class DepSolver(object): if not msg.startswith("0 "): self.logger.info(msg) + def populate_cache(self): + """ populate the yum cache """ + for repo in self.yumbase.repos.findRepos('*'): + repo.metadata_expire = 0 + repo.mdpolicy = "group:all" + self.yumbase.doRepoSetup() + self.yumbase.repos.doSetup() + for repo in self.yumbase.repos.listEnabled(): + # this populates the cache as a side effect + repo.repoXML # pylint: disable=W0104 + self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) + def main(): parser = OptionParser() @@ -233,6 +247,15 @@ def main(): sys.exc_info()[1], exc_info=1) print(json.dumps(False)) rv = 2 + elif cmd == "makecache": + try: + # this code copied from yumcommands.py + depsolver.populate_cache() + print json.dumps(True) + except yum.Errors.YumBaseError: + logger.error("Unexpected error creating cache: %s" % + sys.exc_info()[1], exc_info=1) + print json.dumps(False) elif cmd == "complete": try: data = json.loads(sys.stdin.read()) -- cgit v1.2.3-1-g7c22 From 4b0934821864538205dabb97974172903f68ba3f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 15 Jul 2013 16:16:19 -0400 Subject: Packages: added missing docstring --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index aea447520..a021cda13 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -609,6 +609,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :return: dict of lists of ``url_map`` data """ def get_sources(): + """ getter for the 'sources' key of the OnDemandDict + returned by this function. This delays calling + get_collection() until it's absolutely necessary. """ return self.get_collection(metadata).get_additional_data return OnDemandDict( -- cgit v1.2.3-1-g7c22 From 2bf9447efac887bbc488bd6ed8606d7968c12339 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Jul 2013 09:15:07 -0400 Subject: Core: handle unknown log handlers without names gracefully --- src/lib/Bcfg2/Server/Core.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 1291584b0..0cd4bea3e 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -1274,9 +1274,14 @@ class BaseCore(object): self.logger.info("Core: debug = %s" % debug) levels = self._loglevels[self.debug_flag] for handler in logging.root.handlers: - level = levels.get(handler.name, levels['default']) - self.logger.debug("Setting %s log handler to %s" % - (handler.name, logging.getLevelName(level))) + try: + level = levels.get(handler.name, levels['default']) + self.logger.debug("Setting %s log handler to %s" % + (handler.name, logging.getLevelName(level))) + except AttributeError: + level = levels['default'] + self.logger.debug("Setting unknown log handler %s to %s" % + (handler, logging.getLevelName(level))) handler.setLevel(level) return self.debug_flag -- cgit v1.2.3-1-g7c22 From 1209f76d4a8b64c157331f2c7dfad1bf60a09ded Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Jul 2013 11:23:00 -0400 Subject: POSIXUsers: remove usermod -m option, which is too new --- src/lib/Bcfg2/Client/Tools/POSIXUsers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 8226392f9..bb684899d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -249,7 +249,6 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get('gid'): cmd.extend(['-g', entry.get('gid')]) elif entry.tag == 'POSIXUser': - cmd.append('-m') if entry.get('uid'): cmd.extend(['-u', entry.get('uid')]) cmd.extend(['-g', entry.get('group')]) -- cgit v1.2.3-1-g7c22 From 5d3f167e88b9537249b76a3ebcbd1c58db657470 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Jul 2013 11:44:07 -0400 Subject: POSIXUsers: updated tests for removal of -m --- testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py index 9478f7071..b855153d0 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py @@ -381,15 +381,15 @@ class TestPOSIXUsers(TestTool): (lxml.etree.Element("POSIXUser", name="test", group="test", home="/home/test", shell="/bin/zsh", gecos="Test McTest"), - ["-m", "-g", "test", "-d", "/home/test", "-s", "/bin/zsh", + ["-g", "test", "-d", "/home/test", "-s", "/bin/zsh", "-c", "Test McTest"]), (lxml.etree.Element("POSIXUser", name="test", group="test", home="/home/test", shell="/bin/zsh", gecos="Test McTest", uid="1001"), - ["-m", "-u", "1001", "-g", "test", "-d", "/home/test", + ["-u", "1001", "-g", "test", "-d", "/home/test", "-s", "/bin/zsh", "-c", "Test McTest"]), (entry, - ["-m", "-g", "test", "-G", "wheel,users", "-d", "/home/test", + ["-g", "test", "-G", "wheel,users", "-d", "/home/test", "-s", "/bin/zsh", "-c", "Test McTest"])] for entry, expected in cases: for action in ["add", "mod", "del"]: -- cgit v1.2.3-1-g7c22 From 610b556a908e6782504aa4ec466f819b31303154 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 17 Jul 2013 15:26:56 +0200 Subject: Bcfg2/Client/Tools: do not get bootstatus directly from the entry Bootstatus is optional and should have the value of status if not specified. This is handled by get_bootstatus. --- src/lib/Bcfg2/Client/Tools/DebInit.py | 2 +- src/lib/Bcfg2/Client/Tools/RcUpdate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index 761c51db7..b544e44d4 100644 --- a/src/lib/Bcfg2/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -108,7 +108,7 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """Install Service entry.""" self.logger.info("Installing Service %s" % (entry.get('name'))) - bootstatus = entry.get('bootstatus') + bootstatus = self.get_bootstatus(entry) # check if init script exists try: diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index 8e9626521..e0c913dcd 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -89,7 +89,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """Install Service entry.""" self.logger.info('Installing Service %s' % entry.get('name')) - bootstatus = entry.get('bootstatus') + bootstatus = self.get_bootstatus(entry) if bootstatus is not None: if bootstatus == 'on': # make sure service is enabled on boot -- cgit v1.2.3-1-g7c22 From e12ada32080cc790d167f79a2bda0cde5721fdfc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 08:55:23 -0400 Subject: Packages: use a separate yum persistdir per cache --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 55787681f..9b08c4e88 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -377,6 +377,7 @@ class YumCollection(Collection): # the rpmdb is so hopelessly intertwined with yum that we # have to totally reinvent the dependency resolver. mainopts = dict(cachedir='/', + persistdir='/', installroot=self.cachefile, keepcache="0", debuglevel="0", -- cgit v1.2.3-1-g7c22 From 7dca37efeaff708fd6db30f498228b94bfafbe8e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 08:55:47 -0400 Subject: Packages: better stringification/representation of OnDemandDict --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index a021cda13..cd1bb70b0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -61,7 +61,7 @@ class OnDemandDict(MutableMapping): def __iter__(self): return iter(self._getters.keys()) - def __str__(self): + def __repr__(self): rv = dict(self._values) for key in self._getters.keys(): if key not in rv: -- cgit v1.2.3-1-g7c22 From 2cfbdf19d62880eed3e80f526bf12f6f98ff6633 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 08:56:32 -0400 Subject: Packages: fixed read-only yum cache Replaced incredibly stupid (mea culpa!) and race-condition-prone system that toggled filesystem permissions (what was I thinking?!?) with judicious application of the yum cacheonly option. --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 33 ++++------------------------ src/sbin/bcfg2-yum-helper | 31 +++++++++++++++++++++----- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 9b08c4e88..9060af150 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -928,28 +928,6 @@ class YumCollection(Collection): "output: %s" % err) raise - def _set_cache_writeable(self, writeable): - """ Set the writeability of the yum cache. - - :param writeable: If True, the cache will be made writeable. - If False, the cache will be made read-only. - :type writeable: bool - """ - fmode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH - if writeable: - self.debug_log("Packages: Making cache %s writeable" % - self.cachefile) - fmode |= stat.S_IWUSR - else: - self.debug_log("Packages: Making cache %s read-only" % - self.cachefile) - dmode = fmode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - for root, dirs, files in os.walk(self.cachefile): - for dname in dirs: - os.chmod(os.path.join(root, dname), dmode) - for fname in files: - os.chmod(os.path.join(root, fname), fmode) - def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called when sources are loaded or reloaded by @@ -957,11 +935,10 @@ class YumCollection(Collection): If the builtin yum parsers are in use, this defers to :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.setup_data`. - If using the yum Python libraries, this makes the cache - writeable, cleans up cached yum metadata, regenerates the - server-side yum config (in order to catch any new sources that - have been added to this server), regenerates the yum cache, - and then sets the cache back to read-only. + If using the yum Python libraries, this cleans up cached yum + metadata, regenerates the server-side yum config (in order to + catch any new sources that have been added to this server), + then regenerates the yum cache. :param force_update: Ignore all local cache and setup data from its original upstream sources (i.e., @@ -971,7 +948,6 @@ class YumCollection(Collection): if not self.use_yum: return Collection.setup_data(self, force_update) - self._set_cache_writeable(True) if force_update: # clean up data from the old config try: @@ -989,7 +965,6 @@ class YumCollection(Collection): except ValueError: # error reported by call_helper pass - self._set_cache_writeable(False) class YumSource(Source): diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 473214d89..414606abb 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -42,8 +42,8 @@ def pkgtup_to_string(package): return ''.join(str(e) for e in rv) -class DepSolver(object): - """ Yum dependency solver """ +class YumHelper(object): + """ Yum helper base object """ def __init__(self, cfgfile, verbose=1): self.cfgfile = cfgfile @@ -57,6 +57,16 @@ class DepSolver(object): self.yumbase._getConfig(cfgfile, debuglevel=verbose) # pylint: enable=E1121,W0212 self.logger = logging.getLogger(self.__class__.__name__) + + +class DepSolver(YumHelper): + """ Yum dependency solver. This is used for operations that only + read from the yum cache, and thus operates in cacheonly mode. """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + # internally, yum uses an integer, not a boolean, for conf.cache + self.yumbase.conf.cache = 1 self._groups = None def get_groups(self): @@ -181,6 +191,14 @@ class DepSolver(object): packages.add(txmbr.pkgtup) return list(packages), list(unknown) + +class CacheManager(YumHelper): + """ Yum cache manager. Unlike :class:`DepSolver`, this can write + to the yum cache, and so is used for operations that muck with the + cache. (Technically, :func:`CacheManager.clean_cache` could be in + either DepSolver or CacheManager, but for consistency I've put it + here.) """ + def clean_cache(self): """ clean the yum cache """ for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", @@ -237,10 +255,10 @@ def main(): # pylint: disable=W0702 rv = 0 - depsolver = DepSolver(options.config, options.verbose) if cmd == "clean": + cachemgr = CacheManager(options.config, options.verbose) try: - depsolver.clean_cache() + cachemgr.clean_cache() print(json.dumps(True)) except: logger.error("Unexpected error cleaning cache: %s" % @@ -248,15 +266,17 @@ def main(): print(json.dumps(False)) rv = 2 elif cmd == "makecache": + cachemgr = CacheManager(options.config, options.verbose) try: # this code copied from yumcommands.py - depsolver.populate_cache() + cachemgr.populate_cache() print json.dumps(True) except yum.Errors.YumBaseError: logger.error("Unexpected error creating cache: %s" % sys.exc_info()[1], exc_info=1) print json.dumps(False) elif cmd == "complete": + depsolver = DepSolver(options.config, options.verbose) try: data = json.loads(sys.stdin.read()) except: @@ -275,6 +295,7 @@ def main(): print(json.dumps(dict(packages=[], unknown=data['packages']))) rv = 2 elif cmd == "get_groups": + depsolver = DepSolver(options.config, options.verbose) try: data = json.loads(sys.stdin.read()) rv = dict() -- cgit v1.2.3-1-g7c22 From c92c307beb7418d69a786adef8d10e709a3d4652 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 09:31:30 -0400 Subject: Yum: Removed unused import --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 9060af150..35f888efa 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -53,7 +53,6 @@ The Yum Backend import os import re import sys -import stat import copy import errno import socket @@ -291,7 +290,7 @@ class YumCollection(Collection): if not os.path.exists(self.cachefile): self.debug_log("Creating common cache %s" % self.cachefile) os.mkdir(self.cachefile) - self.setup_data() + #self.setup_data() else: self.cachefile = None -- cgit v1.2.3-1-g7c22 From a8cbdb38cc7a95a189352bb22e1e61eaf0dcd15e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 09:46:14 -0400 Subject: bcfg2-info: fixed automatch --- src/sbin/bcfg2-info | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index cc9b03b80..539ae3115 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -437,7 +437,7 @@ Bcfg2 client itself.""") pname, client = alist automatch = self.setup.cfp.getboolean("properties", "automatch", default=False) - pfile = self.plugins['Properties'].store.entries[pname] + pfile = self.plugins['Properties'].entries[pname] if (not force and not automatch and pfile.xdata.get("automatch", "false").lower() != "true"): -- cgit v1.2.3-1-g7c22 From 259c84f9ddaaf0233a65e686bd8f3346ab0972b0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 10:16:17 -0400 Subject: Packages: make opportunistic yum cache creation respect metadata=disabled --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 35f888efa..7c950a435 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -290,7 +290,8 @@ class YumCollection(Collection): if not os.path.exists(self.cachefile): self.debug_log("Creating common cache %s" % self.cachefile) os.mkdir(self.cachefile) - #self.setup_data() + if not self.disableMetaData: + self.setup_data() else: self.cachefile = None @@ -313,6 +314,26 @@ class YumCollection(Collection): (certdir, err)) self.pulp_cert_set = PulpCertificateSet(certdir, self.fam) + @property + def disableMetaData(self): + """ Report whether or not metadata processing is enabled. + This duplicates code in Packages/__init__.py, and can probably + be removed in Bcfg2 1.4 when we have a module-level setup + object. """ + if self.setup is None: + return True + try: + return not self.setup.cfp.getboolean("packages", "resolver") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return False + except ValueError: + # for historical reasons we also accept "enabled" and + # "disabled" + return self.setup.cfp.get( + "packages", + "metadata", + default="enabled").lower() == "disabled" + @property def __package_groups__(self): return True -- cgit v1.2.3-1-g7c22