Add support for a PORTAGE_RO_DISTDIRS variable. When a given file does not

exist in DISTDIR, search for the file in this list of directories. Search
order is from left to right. Note that the current implementation works by
creating a symlink inside DISTDIR, but that may change in the future.

svn path=/main/trunk/; revision=10547

2 files changed, 138 insertions, 1 deletions

diff --git a/man/make.conf.5 b/man/make.conf.5
index 2eb77cc69..2949d3806 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -451,6 +451,12 @@ emerge is running at.  In other words, this will not set the nice level,
 it will increment it.  For more information about nice levels and what
 are acceptable ranges, see \fBnice\fR(1).
 .TP
+\fBPORTAGE_RO_DISTDIRS\fR = \fI[space delimited list of directories]\fR
+When a given file does not exist in \fBDISTDIR\fR, search for the file
+in this list of directories. Search order is from left to right. Note
+that the current implementation works by creating a symlink inside
+\fBDISTDIR\fR, but that may change in the future.
+.TP
 \fBPORTAGE_RSYNC_INITIAL_TIMEOUT\fR = \fIinteger\fR
 Used by \fBemerge \-\-sync\fR as a timeout for the initial connection to an
 rsync server.
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index ed3000155..51cb281a5 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -988,6 +988,7 @@ class config(object):
 		"PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE",
 		"PORTAGE_GPG_DIR",
 		"PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT",
+		"PORTAGE_RO_DISTDIRS",
 		"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
 		"PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR",
 		"QUICKPKG_DEFAULT_OPTS",
@@ -3096,6 +3097,54 @@ def _checksum_failure_temp_file(distdir, basename):
 	os.rename(filename, temp_filename)
 	return temp_filename
 
+def _check_digests(filename, digests):
+	"""
+	Check digests and displey a message if an error occurs.
+	@return True if all digests match, False otherwise.
+	"""
+	verified_ok, reason = portage.checksum.verify_all(filename, digests)
+	if not verified_ok:
+		writemsg("!!! Previously fetched" + \
+			" file: '%s'\n" % filename, noiselevel=-1)
+		writemsg("!!! Reason: %s\n" % reason[0],
+			noiselevel=-1)
+		writemsg(("!!! Got:      %s\n" + \
+			"!!! Expected: %s\n") % \
+			(reason[1], reason[2]), noiselevel=-1)
+		return False
+	return True
+
+def _check_distfile(filename, digests, eout):
+	"""
+	@return a tuple of (match, stat_obj) where match is True if filename
+	matches all given digests (if any) and stat_obj is a stat result, or
+	None if the file does not exist.
+	"""
+	if digests is None:
+		digests = {}
+	size = digests.get("size")
+	if size is not None and len(digests) == 1:
+		digests = None
+
+	try:
+		st = os.stat(filename)
+	except OSError:
+		return (False, None)
+	if size is not None and size != st.st_size:
+		return (False, st)
+	if not digests:
+		if size is not None:
+			eout.ebegin("%s %s ;-)" % (os.path.basename(filename), "size"))
+			eout.eend(0)
+	else:
+		if _check_digests(filename, digests):
+			eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
+				" ".join(sorted(digests))))
+			eout.eend(0)
+		else:
+			return (False, st)
+	return (True, st)
+
 _fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
 
 _size_suffix_map = {
@@ -3229,6 +3278,11 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 		# no digests because fetch was not called for a specific package
 		mydigests = {}
 
+	import shlex
+	ro_distdirs = [x for x in \
+		shlex.split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
+		if os.path.isdir(x)]
+
 	fsmirrors = []
 	for x in range(len(mymirrors)-1,-1,-1):
 		if mymirrors[x] and mymirrors[x][0]=='/':
@@ -3356,8 +3410,16 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 		1        partially downloaded
 		2        completely downloaded
 		"""
+		fetched = 0
+
+		digests = mydigests.get(myfile, {})
+		size = digests.get("size")
+		if parallel_fetchonly:
+			digests.clear()
+			if size is not None:
+				digests["size"] = size
+
 		myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
-		fetched=0
 		has_space = True
 		file_lock = None
 		if listonly:
@@ -3403,6 +3465,75 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 						waiting_msg=waiting_msg)
 		try:
 			if not listonly:
+
+				eout = portage.output.EOutput()
+				eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
+				match, mystat = _check_distfile(myfile_path, digests, eout)
+				if match:
+					if can_fetch and not fetch_to_ro:
+						try:
+							apply_secpass_permissions(myfile_path,
+								gid=portage_gid, mode=0664, mask=02,
+								stat_cached=mystat)
+						except portage.exception.PortageException, e:
+							if not os.access(myfile_path, os.R_OK):
+								writemsg("!!! Failed to adjust permissions:" + \
+									" %s\n" % str(e), noiselevel=-1)
+							del e
+					continue
+
+				if can_fetch and mystat is None:
+					# Remove broken symlinks if necessary.
+					try:
+						os.unlink(myfile_path)
+					except OSError:
+						pass
+
+				if mystat is not None:
+					if mystat.st_size == 0:
+						if can_fetch:
+							try:
+								os.unlink(myfile_path)
+							except OSError:
+								pass
+					elif can_fetch:
+						if mystat.st_size < fetch_resume_size and \
+							mystat.st_size < size:
+							writemsg((">>> Deleting distfile with size " + \
+								"%d (smaller than " "PORTAGE_FETCH_RESU" + \
+								"ME_MIN_SIZE)\n") % mystat.st_size)
+							try:
+								os.unlink(myfile_path)
+							except OSError, e:
+								if e.errno != errno.ENOENT:
+									raise
+								del e
+						elif mystat.st_size >= size:
+							temp_filename = \
+								_checksum_failure_temp_file(
+								mysettings["DISTDIR"], myfile)
+							writemsg_stdout("Refetching... " + \
+								"File renamed to '%s'\n\n" % \
+								temp_filename, noiselevel=-1)
+
+				if can_fetch and ro_distdirs:
+					readonly_file = None
+					for x in ro_distdirs:
+						filename = os.path.join(x, myfile)
+						match, mystat = _check_distfile(filename, digests, eout)
+						if match:
+							readonly_file = filename
+							break
+					if readonly_file is not None:
+						try:
+							os.unlink(myfile_path)
+						except OSError, e:
+							if e.errno != errno.ENOENT:
+								raise
+							del e
+						os.symlink(readonly_file, myfile_path)
+						continue
+
 				if fsmirrors and not os.path.exists(myfile_path) and has_space:
 					for mydir in fsmirrors:
 						mirror_file = os.path.join(mydir, myfile)