diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Chkconfig.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/DebInit.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/POSIXUsers.py | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/__init__.py | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Core.py | 44 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Collection.py | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 31 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 88 | ||||
-rwxr-xr-x | src/sbin/bcfg2-info | 2 | ||||
-rwxr-xr-x | src/sbin/bcfg2-lint | 2 | ||||
-rwxr-xr-x | src/sbin/bcfg2-test | 2 | ||||
-rwxr-xr-x | src/sbin/bcfg2-yum-helper | 52 |
13 files changed, 191 insertions, 47 deletions
diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index edcc86b85..4833f3f68 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -89,7 +89,7 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): if bootstatus is not None: if bootstatus == 'on': # make sure service is enabled on boot - bootcmd = '/sbin/chkconfig %s %s --level 0123456' % \ + bootcmd = '/sbin/chkconfig %s %s' % \ (entry.get('name'), bootstatus) elif bootstatus == 'off': # make sure service is disabled on boot diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index 761c51db7..b544e44d4 100644 --- a/src/lib/Bcfg2/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -108,7 +108,7 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """Install Service entry.""" self.logger.info("Installing Service %s" % (entry.get('name'))) - bootstatus = entry.get('bootstatus') + bootstatus = self.get_bootstatus(entry) # check if init script exists try: diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 8f6bc5f37..9b40f9cea 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -251,7 +251,6 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get('gid'): cmd.extend(['-g', entry.get('gid')]) elif entry.tag == 'POSIXUser': - cmd.append('-m') if entry.get('uid'): cmd.extend(['-u', entry.get('uid')]) cmd.extend(['-g', entry.get('group')]) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 25603186e..6d1cb9d40 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -21,6 +21,9 @@ def prompt(msg): try: ans = input(msg) return ans in ['y', 'Y'] + except UnicodeEncodeError: + ans = input(msg.encode('utf-8')) + return ans in ['y', 'Y'] except EOFError: # handle ^C on rhel-based platforms raise SystemExit(1) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 34916557a..63e2c2910 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -732,6 +732,26 @@ class BaseCore(object): self.setup.reparse() self.metadata_cache.expire() + def block_for_fam_events(self, handle_events=False): + """ Block until all fam events have been handleed, optionally + handling events as well. (Setting ``handle_events=True`` is + useful for local server cores that don't spawn an event + handling thread.)""" + slept = 0 + log_interval = 3 + if handle_events: + self.fam.handle_events_in_interval(1) + slept += 1 + if self.setup['fam_blocking']: + time.sleep(1) + slept += 1 + while self.fam.pending() != 0: + time.sleep(1) + slept += 1 + if slept % log_interval == 0: + self.logger.debug("Sleeping to handle FAM events...") + self.logger.debug("Slept %s seconds while handling FAM events" % slept) + def run(self): """ Run the server core. This calls :func:`_daemonize`, :func:`_run`, starts the :attr:`fam_thread`, and calls @@ -781,13 +801,9 @@ class BaseCore(object): self.shutdown() raise - if self.setup['fam_blocking']: - time.sleep(1) - while self.fam.pending() != 0: - time.sleep(1) - if self.debug_flag: self.set_debug(None, self.debug_flag) + self.block_for_fam_events() self._block() def _daemonize(self): @@ -888,7 +904,12 @@ class BaseCore(object): imd = self.metadata_cache.get(client_name, None) if not imd: self.logger.debug("Building metadata for %s" % client_name) - imd = self.metadata.get_initial_metadata(client_name) + try: + imd = self.metadata.get_initial_metadata(client_name) + except MetadataConsistencyError: + self.critical_error( + "Client metadata resolution error for %s: %s" % + (client_name, sys.exc_info()[1])) connectors = self.plugins_by_type(Connector) for conn in connectors: grps = conn.get_additional_groups(imd) @@ -1301,9 +1322,14 @@ class BaseCore(object): self.logger.info("Core: debug = %s" % debug) levels = self._loglevels[self.debug_flag] for handler in logging.root.handlers: - level = levels.get(handler.name, levels['default']) - self.logger.debug("Setting %s log handler to %s" % - (handler.name, logging.getLevelName(level))) + try: + level = levels.get(handler.name, levels['default']) + self.logger.debug("Setting %s log handler to %s" % + (handler.name, logging.getLevelName(level))) + except AttributeError: + level = levels['default'] + self.logger.debug("Setting unknown log handler %s to %s" % + (handler, logging.getLevelName(level))) handler.setLevel(level) return self.debug_flag diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 9fb0b07cc..f37e0d97c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1142,9 +1142,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, require_public=False) profile = _add_group(pgroup) else: - msg = "Cannot add new client %s; no default group set" % client - self.logger.error(msg) - raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError( + "Cannot add new client %s; no default group set" % client) for cgroup in self.clientgroups.get(client, []): if cgroup in groups: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 59eefe143..4d05f9d97 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -585,6 +585,10 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): self.filter_unknown(unknown) return packages, unknown + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, + list.__repr__(self)) + def get_collection_class(source_type): """ Given a source type, determine the class of Collection object diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 98add2ccf..55f1007bc 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -281,14 +281,16 @@ class YumCollection(Collection): #: for cached yum metadata self.cachefile = os.path.join(self.cachepath, "cache-%s" % self.cachekey) - if not os.path.exists(self.cachefile): - os.mkdir(self.cachefile) #: The path to the server-side config file used when #: resolving packages with the Python yum libraries self.cfgfile = os.path.join(self.cachefile, "yum.conf") - self.write_config() - self.cmd = Executor() + + if not os.path.exists(self.cachefile): + self.debug_log("Creating common cache %s" % self.cachefile) + os.mkdir(self.cachefile) + if not self.disableMetaData: + self.setup_data() else: self.cachefile = None self.cmd = None @@ -373,6 +375,7 @@ class YumCollection(Collection): # the rpmdb is so hopelessly intertwined with yum that we # have to totally reinvent the dependency resolver. mainopts = dict(cachedir='/', + persistdir='/', installroot=self.cachefile, keepcache="0", debuglevel="0", @@ -921,8 +924,7 @@ class YumCollection(Collection): If using the yum Python libraries, this cleans up cached yum metadata, regenerates the server-side yum config (in order to catch any new sources that have been added to this server), - and then cleans up cached yum metadata again, in case the new - config has any preexisting cache. + then regenerates the yum cache. :param force_update: Ignore all local cache and setup data from its original upstream sources (i.e., @@ -933,23 +935,22 @@ class YumCollection(Collection): return Collection.setup_data(self, force_update) if force_update: - # we call this twice: one to clean up data from the old - # config, and once to clean up data from the new config + # clean up data from the old config try: self.call_helper("clean") except ValueError: # error reported by call_helper pass - os.unlink(self.cfgfile) + if os.path.exists(self.cfgfile): + os.unlink(self.cfgfile) self.write_config() - if force_update: - try: - self.call_helper("clean") - except ValueError: - # error reported by call_helper - pass + try: + self.call_helper("makecache") + except ValueError: + # error reported by call_helper + pass class YumSource(Source): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 8e14b0dfb..03047b046 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,8 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ + MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -23,6 +24,52 @@ APT_CONFIG_DEFAULT = \ "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +class OnDemandDict(MutableMapping): + """ This maps a set of keys to a set of value-getting functions; + the values are populated on-the-fly by the functions as the values + are needed (and not before). This is used by + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data`; + see the docstring for that function for details on why. + + Unlike a dict, you should not specify values for for the righthand + side of this mapping, but functions that get values. E.g.: + + .. code-block:: python + + d = OnDemandDict(foo=load_foo, + bar=lambda: "bar"); + """ + + def __init__(self, **getters): + self._values = dict() + self._getters = dict(**getters) + + def __getitem__(self, key): + if key not in self._values: + self._values[key] = self._getters[key]() + return self._values[key] + + def __setitem__(self, key, getter): + self._getters[key] = getter + + def __delitem__(self, key): + del self._values[key] + del self._getters[key] + + def __len__(self): + return len(self._getters) + + def __iter__(self): + return iter(self._getters.keys()) + + def __repr__(self): + rv = dict(self._values) + for key in self._getters.keys(): + if key not in rv: + rv[key] = 'unknown' + return str(rv) + + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -520,20 +567,41 @@ class Packages(Bcfg2.Server.Plugin.Plugin, def get_additional_data(self, metadata): """ Return additional data for the given client. This will be - a dict containing a single key, ``sources``, whose value is a - list of data returned from - :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, - namely, a list of - :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` - data. + an :class:`Bcfg2.Server.Plugins.Packages.OnDemandDict` + containing two keys: + + * ``sources``, whose value is a list of data returned from + :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, + namely, a list of + :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` + data; and + * ``get_config``, whose value is the + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_config` + function, which can be used to get the Packages config for + other systems. + + This uses an OnDemandDict instead of just a normal dict + because loading a source collection can be a fairly + time-consuming process, particularly for the first time. As a + result, when all metadata objects are built at once (such as + after the server is restarted, or far more frequently if + Metadata caching is disabled), this function would be a major + bottleneck if we tried to build all collections at the same + time. Instead, they're merely built on-demand. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ - collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data(), - get_config=self.get_config) + def get_sources(): + """ getter for the 'sources' key of the OnDemandDict + returned by this function. This delays calling + get_collection() until it's absolutely necessary. """ + return self.get_collection(metadata).get_additional_data + + return OnDemandDict( + sources=get_sources, + get_config=lambda: self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index c01a1210b..5eef72350 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -704,7 +704,7 @@ set of packages""" def run(self, args): # pylint: disable=W0221 try: self.load_plugins() - self.fam.handle_events_in_interval(1) + self.block_for_fam_events(handle_events=True) if args: self.onecmd(" ".join(args)) else: diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 4987b8034..27d8d4291 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -73,7 +73,7 @@ def load_server(): """ load server """ core = Bcfg2.Server.Core.BaseCore() core.load_plugins() - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) return core diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 4aa495c98..564ddec49 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -158,7 +158,7 @@ def get_core(setup): """ Get a server core, with events handled """ core = Bcfg2.Server.Core.BaseCore(setup) core.load_plugins() - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) return core diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 4ef531d39..414606abb 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -42,8 +42,8 @@ def pkgtup_to_string(package): return ''.join(str(e) for e in rv) -class DepSolver(object): - """ Yum dependency solver """ +class YumHelper(object): + """ Yum helper base object """ def __init__(self, cfgfile, verbose=1): self.cfgfile = cfgfile @@ -57,6 +57,16 @@ class DepSolver(object): self.yumbase._getConfig(cfgfile, debuglevel=verbose) # pylint: enable=E1121,W0212 self.logger = logging.getLogger(self.__class__.__name__) + + +class DepSolver(YumHelper): + """ Yum dependency solver. This is used for operations that only + read from the yum cache, and thus operates in cacheonly mode. """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + # internally, yum uses an integer, not a boolean, for conf.cache + self.yumbase.conf.cache = 1 self._groups = None def get_groups(self): @@ -181,6 +191,14 @@ class DepSolver(object): packages.add(txmbr.pkgtup) return list(packages), list(unknown) + +class CacheManager(YumHelper): + """ Yum cache manager. Unlike :class:`DepSolver`, this can write + to the yum cache, and so is used for operations that muck with the + cache. (Technically, :func:`CacheManager.clean_cache` could be in + either DepSolver or CacheManager, but for consistency I've put it + here.) """ + def clean_cache(self): """ clean the yum cache """ for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", @@ -193,6 +211,20 @@ class DepSolver(object): if not msg.startswith("0 "): self.logger.info(msg) + def populate_cache(self): + """ populate the yum cache """ + for repo in self.yumbase.repos.findRepos('*'): + repo.metadata_expire = 0 + repo.mdpolicy = "group:all" + self.yumbase.doRepoSetup() + self.yumbase.repos.doSetup() + for repo in self.yumbase.repos.listEnabled(): + # this populates the cache as a side effect + repo.repoXML # pylint: disable=W0104 + self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) + def main(): parser = OptionParser() @@ -223,17 +255,28 @@ def main(): # pylint: disable=W0702 rv = 0 - depsolver = DepSolver(options.config, options.verbose) if cmd == "clean": + cachemgr = CacheManager(options.config, options.verbose) try: - depsolver.clean_cache() + cachemgr.clean_cache() print(json.dumps(True)) except: logger.error("Unexpected error cleaning cache: %s" % sys.exc_info()[1], exc_info=1) print(json.dumps(False)) rv = 2 + elif cmd == "makecache": + cachemgr = CacheManager(options.config, options.verbose) + try: + # this code copied from yumcommands.py + cachemgr.populate_cache() + print json.dumps(True) + except yum.Errors.YumBaseError: + logger.error("Unexpected error creating cache: %s" % + sys.exc_info()[1], exc_info=1) + print json.dumps(False) elif cmd == "complete": + depsolver = DepSolver(options.config, options.verbose) try: data = json.loads(sys.stdin.read()) except: @@ -252,6 +295,7 @@ def main(): print(json.dumps(dict(packages=[], unknown=data['packages']))) rv = 2 elif cmd == "get_groups": + depsolver = DepSolver(options.config, options.verbose) try: data = json.loads(sys.stdin.read()) rv = dict() |