Bug #197965 - When fetch() encounters a checksum failure,

look for an identical file from a previous checksum failure
in order to avoid consuming unnecessary disk space with
duplicate files.

svn path=/main/trunk/; revision=8401

1 files changed, 46 insertions, 12 deletions

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index d609d2b6e..3597fa3f8 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -2717,6 +2717,45 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero
 		return retval >> 8
 	return retval
 
+def _checksum_failure_temp_file(distdir, basename):
+	"""
+	First try to find a duplicate temp file with the same checksum and return
+	that filename if available. Otherwise, use mkstemp to create a new unique
+	filename._checksum_failure_.$RANDOM, rename the given file, and return the
+	new filename. In any case, filename will be renamed or removed before this
+	function returns a temp filename.
+	"""
+
+	filename = os.path.join(distdir, basename)
+	size = os.stat(filename).st_size
+	checksum = None
+	tempfile_re = re.compile(re.escape(basename) + r'\._checksum_failure_\..*')
+	for temp_filename in os.listdir(distdir):
+		if not tempfile_re.match(temp_filename):
+			continue
+		temp_filename = os.path.join(distdir, temp_filename)
+		try:
+			if size != os.stat(temp_filename).st_size:
+				continue
+		except OSError:
+			continue
+		try:
+			temp_checksum = portage.checksum.perform_md5(temp_filename)
+		except portage.exception.FileNotFound:
+			# Apparently the temp file disappeared. Let it go.
+			continue
+		if checksum is None:
+			checksum = portage.checksum.perform_md5(filename)
+		if checksum == temp_checksum:
+			os.unlink(filename)
+			return temp_filename
+
+	from tempfile import mkstemp
+	fd, temp_filename = mkstemp("", basename + "._checksum_failure_.", distdir)
+	os.close(fd)
+	os.rename(filename, temp_filename)
+	return temp_filename
+
 def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
 	"fetch files.  Will use digest file if available."
 
@@ -2888,6 +2927,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 					noiselevel=-1)
 				return 0
 			del distlocks_subdir
+
 	for myfile in filedict:
 		"""
 		fetched  status
@@ -2976,12 +3016,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 								if reason[0] == "Insufficient data for checksum verification":
 									return 0
 								if can_fetch and not restrict_fetch:
-									from tempfile import mkstemp
-									fd, temp_filename = mkstemp("",
-										myfile + "._checksum_failure_.",
-										mysettings["DISTDIR"])
-									os.close(fd)
-									os.rename(myfile_path, temp_filename)
+									temp_filename = \
+										_checksum_failure_temp_file(
+										mysettings["DISTDIR"], myfile)
 									writemsg_stdout("Refetching... " + \
 										"File renamed to '%s'\n\n" % \
 										temp_filename, noiselevel=-1)
@@ -3158,12 +3195,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 										(reason[1], reason[2]), noiselevel=-1)
 									if reason[0] == "Insufficient data for checksum verification":
 										return 0
-									from tempfile import mkstemp
-									fd, temp_filename = mkstemp("",
-										myfile + "._checksum_failure_.",
-										mysettings["DISTDIR"])
-									os.close(fd)
-									os.rename(myfile_path, temp_filename)
+									temp_filename = \
+										_checksum_failure_temp_file(
+										mysettings["DISTDIR"], myfile)
 									writemsg_stdout("Refetching... " + \
 										"File renamed to '%s'\n\n" % \
 										temp_filename, noiselevel=-1)