From 2217fa6295070f137988006c4bb00d25dfc0cb5e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 15 Jul 2013 15:32:21 -0400 Subject: Read-only yum cache This makes the yum cache read-only so that bcfg2-yum-helper cannot update the cache on the fly, which should help avoid locking issues with the yum caches that can cause client runs to fail. It also makes the Packages plugin behave more consistently, since use of yum libraries won't cause the cache to be refreshed at random times on the fly, but rather more predictably as with the Apt cache or the yum cache without using yum libraries. Unlike those two cases, though, the caches will not all be downloaded initially, but rather opportunistically as needed. In order for this to work, the Bcfg2 server must not run as root. Root ignores the 'w' permissions bit, so the cache cannot be made read-only. --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 60 ++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 17 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Yum.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 4608bcca5..55787681f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -53,6 +53,7 @@ The Yum Backend import os import re import sys +import stat import copy import errno import socket @@ -282,13 +283,15 @@ class YumCollection(Collection): #: for cached yum metadata self.cachefile = os.path.join(self.cachepath, "cache-%s" % self.cachekey) - if not os.path.exists(self.cachefile): - os.mkdir(self.cachefile) #: The path to the server-side config file used when #: resolving packages with the Python yum libraries self.cfgfile = os.path.join(self.cachefile, "yum.conf") - self.write_config() + + if not os.path.exists(self.cachefile): + self.debug_log("Creating common cache %s" % self.cachefile) + os.mkdir(self.cachefile) + self.setup_data() else: self.cachefile = None @@ -924,6 +927,28 @@ class YumCollection(Collection): "output: %s" % err) raise + def _set_cache_writeable(self, writeable): + """ Set the writeability of the yum cache. + + :param writeable: If True, the cache will be made writeable. + If False, the cache will be made read-only. + :type writeable: bool + """ + fmode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH + if writeable: + self.debug_log("Packages: Making cache %s writeable" % + self.cachefile) + fmode |= stat.S_IWUSR + else: + self.debug_log("Packages: Making cache %s read-only" % + self.cachefile) + dmode = fmode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + for root, dirs, files in os.walk(self.cachefile): + for dname in dirs: + os.chmod(os.path.join(root, dname), dmode) + for fname in files: + os.chmod(os.path.join(root, fname), fmode) + def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called when sources are loaded or reloaded by @@ -931,11 +956,11 @@ class YumCollection(Collection): If the builtin yum parsers are in use, this defers to :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.setup_data`. - If using the yum Python libraries, this cleans up cached yum - metadata, regenerates the server-side yum config (in order to - catch any new sources that have been added to this server), - and then cleans up cached yum metadata again, in case the new - config has any preexisting cache. + If using the yum Python libraries, this makes the cache + writeable, cleans up cached yum metadata, regenerates the + server-side yum config (in order to catch any new sources that + have been added to this server), regenerates the yum cache, + and then sets the cache back to read-only. :param force_update: Ignore all local cache and setup data from its original upstream sources (i.e., @@ -945,24 +970,25 @@ class YumCollection(Collection): if not self.use_yum: return Collection.setup_data(self, force_update) + self._set_cache_writeable(True) if force_update: - # we call this twice: one to clean up data from the old - # config, and once to clean up data from the new config + # clean up data from the old config try: self.call_helper("clean") except ValueError: # error reported by call_helper pass - os.unlink(self.cfgfile) + if os.path.exists(self.cfgfile): + os.unlink(self.cfgfile) self.write_config() - if force_update: - try: - self.call_helper("clean") - except ValueError: - # error reported by call_helper - pass + try: + self.call_helper("makecache") + except ValueError: + # error reported by call_helper + pass + self._set_cache_writeable(False) class YumSource(Source): -- cgit v1.2.3-1-g7c22 From e12ada32080cc790d167f79a2bda0cde5721fdfc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 08:55:23 -0400 Subject: Packages: use a separate yum persistdir per cache --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Yum.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 55787681f..9b08c4e88 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -377,6 +377,7 @@ class YumCollection(Collection): # the rpmdb is so hopelessly intertwined with yum that we # have to totally reinvent the dependency resolver. mainopts = dict(cachedir='/', + persistdir='/', installroot=self.cachefile, keepcache="0", debuglevel="0", -- cgit v1.2.3-1-g7c22 From 2cfbdf19d62880eed3e80f526bf12f6f98ff6633 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 08:56:32 -0400 Subject: Packages: fixed read-only yum cache Replaced incredibly stupid (mea culpa!) and race-condition-prone system that toggled filesystem permissions (what was I thinking?!?) with judicious application of the yum cacheonly option. --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 33 ++++------------------------ 1 file changed, 4 insertions(+), 29 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Yum.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 9b08c4e88..9060af150 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -928,28 +928,6 @@ class YumCollection(Collection): "output: %s" % err) raise - def _set_cache_writeable(self, writeable): - """ Set the writeability of the yum cache. - - :param writeable: If True, the cache will be made writeable. - If False, the cache will be made read-only. - :type writeable: bool - """ - fmode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH - if writeable: - self.debug_log("Packages: Making cache %s writeable" % - self.cachefile) - fmode |= stat.S_IWUSR - else: - self.debug_log("Packages: Making cache %s read-only" % - self.cachefile) - dmode = fmode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - for root, dirs, files in os.walk(self.cachefile): - for dname in dirs: - os.chmod(os.path.join(root, dname), dmode) - for fname in files: - os.chmod(os.path.join(root, fname), fmode) - def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called when sources are loaded or reloaded by @@ -957,11 +935,10 @@ class YumCollection(Collection): If the builtin yum parsers are in use, this defers to :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.setup_data`. - If using the yum Python libraries, this makes the cache - writeable, cleans up cached yum metadata, regenerates the - server-side yum config (in order to catch any new sources that - have been added to this server), regenerates the yum cache, - and then sets the cache back to read-only. + If using the yum Python libraries, this cleans up cached yum + metadata, regenerates the server-side yum config (in order to + catch any new sources that have been added to this server), + then regenerates the yum cache. :param force_update: Ignore all local cache and setup data from its original upstream sources (i.e., @@ -971,7 +948,6 @@ class YumCollection(Collection): if not self.use_yum: return Collection.setup_data(self, force_update) - self._set_cache_writeable(True) if force_update: # clean up data from the old config try: @@ -989,7 +965,6 @@ class YumCollection(Collection): except ValueError: # error reported by call_helper pass - self._set_cache_writeable(False) class YumSource(Source): -- cgit v1.2.3-1-g7c22 From c92c307beb7418d69a786adef8d10e709a3d4652 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 09:31:30 -0400 Subject: Yum: Removed unused import --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Yum.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 9060af150..35f888efa 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -53,7 +53,6 @@ The Yum Backend import os import re import sys -import stat import copy import errno import socket @@ -291,7 +290,7 @@ class YumCollection(Collection): if not os.path.exists(self.cachefile): self.debug_log("Creating common cache %s" % self.cachefile) os.mkdir(self.cachefile) - self.setup_data() + #self.setup_data() else: self.cachefile = None -- cgit v1.2.3-1-g7c22 From 259c84f9ddaaf0233a65e686bd8f3346ab0972b0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 10:16:17 -0400 Subject: Packages: make opportunistic yum cache creation respect metadata=disabled --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Yum.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 35f888efa..7c950a435 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -290,7 +290,8 @@ class YumCollection(Collection): if not os.path.exists(self.cachefile): self.debug_log("Creating common cache %s" % self.cachefile) os.mkdir(self.cachefile) - #self.setup_data() + if not self.disableMetaData: + self.setup_data() else: self.cachefile = None @@ -313,6 +314,26 @@ class YumCollection(Collection): (certdir, err)) self.pulp_cert_set = PulpCertificateSet(certdir, self.fam) + @property + def disableMetaData(self): + """ Report whether or not metadata processing is enabled. + This duplicates code in Packages/__init__.py, and can probably + be removed in Bcfg2 1.4 when we have a module-level setup + object. """ + if self.setup is None: + return True + try: + return not self.setup.cfp.getboolean("packages", "resolver") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return False + except ValueError: + # for historical reasons we also accept "enabled" and + # "disabled" + return self.setup.cfp.get( + "packages", + "metadata", + default="enabled").lower() == "disabled" + @property def __package_groups__(self): return True -- cgit v1.2.3-1-g7c22