diff options
author | Zac Medico <zmedico@gentoo.org> | 2009-03-31 20:14:03 +0000 |
---|---|---|
committer | Zac Medico <zmedico@gentoo.org> | 2009-03-31 20:14:03 +0000 |
commit | 98961815eda7c7ff44cfca3643d8b280f3873d47 (patch) | |
tree | 9d86d7c131d8a6c64ca6f33f4c428dc32f0835ff /pym/portage | |
parent | 4bed54db77464c13f3fc059997286cd9c06562da (diff) | |
download | portage-98961815eda7c7ff44cfca3643d8b280f3873d47.tar.gz portage-98961815eda7c7ff44cfca3643d8b280f3873d47.tar.bz2 portage-98961815eda7c7ff44cfca3643d8b280f3873d47.zip |
Add a new egencache --rsync option which enables a stat collision workaround
for cases in which the content of a cache entry changes and neither the file
mtime nor size changes (preventing rsync from detecting changes). See bug
#139134. This option should only be needed for distribution via something
like rsync, which relies on timestamps and file sizes to detect changes. It's
not needed with git since that uses a more thorough mechanism which allows it
to detect changed inode numbers (described in racy-git.txt in the git
technical docs).
svn path=/main/trunk/; revision=13262
Diffstat (limited to 'pym/portage')
-rw-r--r-- | pym/portage/cache/cache_errors.py | 22 | ||||
-rw-r--r-- | pym/portage/cache/metadata.py | 49 |
2 files changed, 63 insertions, 8 deletions
diff --git a/pym/portage/cache/cache_errors.py b/pym/portage/cache/cache_errors.py index f63e5994b..e1e8eead0 100644 --- a/pym/portage/cache/cache_errors.py +++ b/pym/portage/cache/cache_errors.py @@ -39,3 +39,25 @@ class ReadOnlyRestriction(CacheError): self.info = info def __str__(self): return "cache is non-modifiable"+str(self.info) + +class StatCollision(CacheError): + """ + If the content of a cache entry changes and neither the file mtime nor + size changes, it will prevent rsync from detecting changes. Cache backends + may raise this exception from _setitem() if they detect this type of stat + collision. See bug #139134. + """ + def __init__(self, key, filename, mtime, size): + self.key = key + self.filename = filename + self.mtime = mtime + self.size = size + + def __str__(self): + return "%s has stat collision with size %s and mtime %s" % \ + (self.key, self.size, self.mtime) + + def __repr__(self): + return "portage.cache.cache_errors.StatCollision(%s)" % \ + (', '.join((repr(self.key), repr(self.filename), + repr(self.mtime), repr(self.size))),) diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py index 5222223c6..a8be01095 100644 --- a/pym/portage/cache/metadata.py +++ b/pym/portage/cache/metadata.py @@ -3,7 +3,7 @@ # License: GPL2 # $Id$ -import errno, os, re +import errno, os, re, sys from portage.cache import cache_errors, flat_hash import portage.eclass_cache from portage.cache.template import reconstruct_eclasses @@ -30,6 +30,7 @@ class database(flat_hash.database): super(database, self).__init__(location, *args, **config) self.location = os.path.join(loc, "metadata","cache") self.ec = portage.eclass_cache.cache(loc) + self.raise_stat_collision = False def _parse_data(self, data, cpv): _hashed_re_match = self._hashed_re.match @@ -73,31 +74,63 @@ class database(flat_hash.database): values = ProtectedDict(values) values["INHERITED"] = ' '.join(sorted(values["_eclasses_"])) + new_content = [] + for k in self.auxdbkey_order: + new_content.append(unicode(values.get(k, ''), errors='replace')) + new_content.append(u'\n') + for i in xrange(magic_line_count - len(self.auxdbkey_order)): + new_content.append(u'\n') + new_content = u''.join(new_content) + new_content = new_content.encode( + sys.getdefaultencoding(), 'backslashreplace') + + new_fp = os.path.join(self.location, cpv) + try: + f = open(new_fp, 'rb') + except EnvironmentError: + pass + else: + try: + try: + existing_st = os.fstat(f.fileno()) + existing_content = f.read() + finally: + f.close() + except EnvironmentError: + pass + else: + existing_mtime = long(existing_st.st_mtime) + if values['_mtime_'] == existing_mtime and \ + existing_content == new_content: + return + + if self.raise_stat_collision and \ + values['_mtime_'] == existing_mtime and \ + len(new_content) == existing_st.st_size: + raise cache_errors.StatCollision(cpv, new_fp, + existing_mtime, existing_st.st_size) + s = cpv.rfind("/") fp = os.path.join(self.location,cpv[:s], ".update.%i.%s" % (os.getpid(), cpv[s+1:])) try: - myf = open(fp, "w") + myf = open(fp, 'wb') except EnvironmentError, e: if errno.ENOENT == e.errno: try: self._ensure_dirs(cpv) - myf = open(fp, "w") + myf = open(fp, 'wb') except EnvironmentError, e: raise cache_errors.CacheCorruption(cpv, e) else: raise cache_errors.CacheCorruption(cpv, e) try: - for k in self.auxdbkey_order: - myf.write(values.get(k, "") + "\n") - for i in xrange(magic_line_count - len(self.auxdbkey_order)): - myf.write("\n") + myf.write(new_content) finally: myf.close() self._ensure_access(fp, mtime=values["_mtime_"]) - new_fp = os.path.join(self.location, cpv) try: os.rename(fp, new_fp) except EnvironmentError, e: |