Implement a PORTAGE_FETCH_RESUME_MIN_SIZE variable that is useful for

helping to ensure that small garbage files such as html 404 pages are
properly discarded. (trunk r9687)

svn path=/main/branches/2.1.2/; revision=9688

1 files changed, 57 insertions, 3 deletions

diff --git a/pym/portage.py b/pym/portage.py
index 56c4dab12..d1cf85cc7 100644
--- a/pym/portage.py
+++ b/pym/portage.py
@@ -1073,7 +1073,8 @@ class config:
 		"PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES",
 		"PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT",
 		"PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM",
-		"PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_GPG_DIR",
+		"PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE",
+		"PORTAGE_GPG_DIR",
 		"PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT",
 		"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
 		"PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR",
@@ -2986,6 +2987,20 @@ def _checksum_failure_temp_file(distdir, basename):
 	os.rename(filename, temp_filename)
 	return temp_filename
 
+_fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
+
+_size_suffix_map = {
+	''  : 0,
+	'K' : 10,
+	'M' : 20,
+	'G' : 30,
+	'T' : 40,
+	'P' : 50,
+	'E' : 60,
+	'Z' : 70,
+	'Y' : 80,
+}
+
 def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
 	"fetch files.  Will use digest file if available."
 
@@ -3026,6 +3041,26 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 	checksum_failure_max_tries = v
 	del v
 
+	fetch_resume_size_default = "350K"
+	fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE")
+	if fetch_resume_size is not None:
+		fetch_resume_size = "".join(fetch_resume_size.split())
+		match = _fetch_resume_size_re.match(fetch_resume_size)
+		if match is None or \
+			(match.group(2).upper() not in _size_suffix_map):
+			writemsg("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE" + \
+				" contains an unrecognized format: '%s'\n" % \
+				mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1)
+			writemsg("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE " + \
+				"default value: %s\n" % fetch_resume_size_default,
+				noiselevel=-1)
+			fetch_resume_size = None
+	if fetch_resume_size is None:
+		fetch_resume_size = fetch_resume_size_default
+		match = _fetch_resume_size_re.match(fetch_resume_size)
+	fetch_resume_size = int(match.group(1)) * \
+		2 ** _size_suffix_map[match.group(2)]
+
 	# Behave like the package has RESTRICT="primaryuri" after a
 	# couple of checksum failures, to increase the probablility
 	# of success before checksum_failure_max_tries is reached.
@@ -3381,8 +3416,27 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 
 				if fetched != 2 and has_space:
 					#we either need to resume or start the download
-					#you can't use "continue" when you're inside a "try" block
-					if fetched==1:
+					if fetched == 1:
+						try:
+							mystat = os.stat(myfile_path)
+						except OSError, e:
+							if e.errno != errno.ENOENT:
+								raise
+							del e
+							fetched = 0
+						else:
+							if mystat.st_size < fetch_resume_size:
+								writemsg((">>> Deleting distfile with size " + \
+									"%d (smaller than " "PORTAGE_FETCH_RESU" + \
+									"ME_MIN_SIZE)\n") % mystat.st_size)
+								try:
+									os.unlink(myfile_path)
+								except OSError, e:
+									if e.errno != errno.ENOENT:
+										raise
+									del e
+								fetched = 0
+					if fetched == 1:
 						#resume mode:
 						writemsg(">>> Resuming download...\n")
 						locfetch=resumecommand