Implement SRC_URI arrows for EAPI 2. The portdbapi.getfetchlist() method

is now deprecated and there is a new getFetchMap() method that returns
a dict which maps each file name to a set of alternative URIs. The
portage.fetch() function uses introspection to detect when such a dict
is passed in and handles it appropriately, while maintaining backward
compatibility if a list of uris is passed in.

svn path=/main/trunk/; revision=11522

3 files changed, 160 insertions, 67 deletions

diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index 5c5aaa731..6cfe7eb66 100644
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -431,7 +431,7 @@ class search(object):
 			pass
 		self.portdb = fake_portdb
 		for attrib in ("aux_get", "cp_all",
-			"xmatch", "findname", "getfetchlist"):
+			"xmatch", "findname", "getFetchMap"):
 			setattr(fake_portdb, attrib, getattr(self, "_"+attrib))
 
 		self._dbs = []
@@ -478,14 +478,14 @@ class search(object):
 					return value
 		return None
 
-	def _getfetchlist(self, *args, **kwargs):
+	def _getFetchMap(self, *args, **kwargs):
 		for db in self._dbs:
-			func = getattr(db, "getfetchlist", None)
+			func = getattr(db, "getFetchMap", None)
 			if func:
 				value = func(*args, **kwargs)
 				if value:
 					return value
-		return [], []
+		return {}
 
 	def _visible(self, db, cpv, metadata):
 		installed = db is self.vartree.dbapi
@@ -684,8 +684,7 @@ class search(object):
 						from portage import manifest
 						mf = manifest.Manifest(
 							pkgdir, self.settings["DISTDIR"])
-						fetchlist = self.portdb.getfetchlist(mycpv,
-							mysettings=self.settings, all=True)[1]
+						fetchlist = self.portdb.getFetchMap(mycpv)
 						try:
 							mysum[0] = mf.getDistfilesSize(fetchlist)
 						except KeyError, e:
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 199317c28..207432f35 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -3487,11 +3487,19 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 	else:
 		locations = mymirrors
 
+	file_uri_tuples = []
+	if isinstance(myuris, dict):
+		for myfile, uri_set in myuris.iteritems():
+			for myuri in uri_set:
+				file_uri_tuples.append((myfile, myuri))
+	else:
+		for myuri in myuris:
+			file_uri_tuples.append((os.path.basename(myuri), myuri))
+
 	filedict={}
 	primaryuri_indexes={}
 	primaryuri_dict = {}
-	for myuri in myuris:
-		myfile=os.path.basename(myuri)
+	for myfile, myuri in file_uri_tuples:
 		if myfile not in filedict:
 			filedict[myfile]=[]
 			for y in range(0,len(locations)):
@@ -4150,11 +4158,8 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None
 						doebuild_environment(myebuild, "fetch",
 							mysettings["ROOT"], fetch_settings,
 							debug, 1, myportdb)
-						alluris, aalist = myportdb.getfetchlist(
-							cpv, mytree=mytree, all=True,
-							mysettings=fetch_settings)
-						myuris = [uri for uri in alluris \
-							if os.path.basename(uri) == myfile]
+						uri_map = myportdb.getFetchMap(cpv, mytree=mytree)
+						myuris = {myfile:uri_map[myfile]}
 						fetch_settings["A"] = myfile # for use by pkg_nofetch()
 						if fetch(myuris, fetch_settings):
 							success = True
@@ -4191,8 +4196,7 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None
 				writemsg_stdout("  digest.assumed" + portage.output.colorize("WARN",
 					str(len(auto_assumed)).rjust(18)) + "\n")
 				for pkg_key in pkgs:
-					fetchlist = myportdb.getfetchlist(pkg_key,
-						mysettings=mysettings, all=True, mytree=mytree)[1]
+					fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree)
 					pv = pkg_key.split("/")[1]
 					for filename in auto_assumed:
 						if filename in fetchlist:
@@ -5580,11 +5584,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
 		# Make sure we get the correct tree in case there are overlays.
 		mytree = os.path.realpath(
 			os.path.dirname(os.path.dirname(mysettings["O"])))
+		useflags = mysettings["PORTAGE_USE"].split()
 		try:
-			newuris, alist = mydbapi.getfetchlist(
-				mycpv, mytree=mytree, mysettings=mysettings)
-			alluris, aalist = mydbapi.getfetchlist(
-				mycpv, mytree=mytree, all=True, mysettings=mysettings)
+			alist = mydbapi.getFetchMap(mycpv, useflags=useflags, mytree=mytree)
+			aalist = mydbapi.getFetchMap(mycpv, mytree=mytree)
 		except portage.exception.InvalidDependString, e:
 			writemsg("!!! %s\n" % str(e), noiselevel=-1)
 			writemsg("!!! Invalid SRC_URI for '%s'.\n" % mycpv, noiselevel=-1)
@@ -5593,26 +5596,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
 		mysettings["A"] = " ".join(alist)
 		mysettings["AA"] = " ".join(aalist)
 		if ("mirror" in features) or fetchall:
-			fetchme = alluris[:]
-			checkme = aalist[:]
-		elif mydo == "digest":
-			fetchme = alluris[:]
-			checkme = aalist[:]
-			# Skip files that we already have digests for.
-			mf = Manifest(mysettings["O"], mysettings["DISTDIR"])
-			mydigests = mf.getTypeDigests("DIST")
-			required_hash_types = set()
-			required_hash_types.add("size")
-			required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH)
-			for filename, hashes in mydigests.iteritems():
-				if not required_hash_types.difference(hashes):
-					checkme = [i for i in checkme if i != filename]
-					fetchme = [i for i in fetchme \
-						if os.path.basename(i) != filename]
-				del filename, hashes
+			fetchme = aalist
+			checkme = aalist
 		else:
-			fetchme = newuris[:]
-			checkme = alist[:]
+			fetchme = alist
+			checkme = alist
 
 		if mydo == "fetch":
 			# Files are already checked inside fetch(),
@@ -6871,8 +6859,7 @@ class FetchlistDict(UserDict.DictMixin):
 		self.portdb = mydbapi
 	def __getitem__(self, pkg_key):
 		"""Returns the complete fetch list for a given package."""
-		return self.portdb.getfetchlist(pkg_key, mysettings=self.settings,
-			all=True, mytree=self.mytree)[1]
+		return self.portdb.getFetchMap(pkg_key, mytree=self.mytree).keys()
 	def __contains__(self, cpv):
 		return cpv in self.keys()
 	def has_key(self, pkg_key):
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index a1165907c..feade6fc8 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -25,6 +25,52 @@ from portage import eclass_cache, auxdbkeys, doebuild, flatten, \
 import os, stat
 from itertools import izip
 
+def _src_uri_validate(cpv, eapi, src_uri):
+	"""
+	Take a SRC_URI structure as returned by paren_reduce or use_reduce
+	and validate it. Raises InvalidDependString if a problem is detected,
+	such as missing operand for a -> operator.
+	"""
+	uri = None
+	operator = None
+	for x in src_uri:
+		if isinstance(x, list):
+			if operator is not None:
+				raise portage.exception.InvalidDependString(
+					("getFetchMap(): '%s' SRC_URI arrow missing " + \
+					"right operand") % (cpv,))
+			uri = None
+			_src_uri_validate(cpv, eapi, x)
+			continue
+		if x[:-1] == "?":
+			if operator is not None:
+				raise portage.exception.InvalidDependString(
+					("getFetchMap(): '%s' SRC_URI arrow missing " + \
+					"right operand") % (cpv,))
+			uri = None
+			continue
+		if uri is None:
+			if x == "->":
+				raise portage.exception.InvalidDependString(
+					("getFetchMap(): '%s' SRC_URI arrow missing " + \
+					"left operand") % (cpv,))
+			uri = x
+			continue
+		if x == "->":
+			if eapi in ("0", "1"):
+				raise portage.exception.InvalidDependString(
+					("getFetchMap(): '%s' SRC_URI arrows are not " + \
+					"supported with EAPI='%s'") % (cpv, eapi))
+			operator = x
+			continue
+		uri = None
+		operator = None
+
+	if operator is not None:
+		raise portage.exception.InvalidDependString(
+			"getFetchMap(): '%s' SRC_URI arrow missing right operand" % \
+			(cpv,))
+
 class portdbapi(dbapi):
 	"""this tree will scan a portage directory located at root (passed to init)"""
 	portdbapi_instances = []
@@ -450,9 +496,24 @@ class portdbapi(dbapi):
 
 		return returnme
 
-	def getfetchlist(self, mypkg, useflags=None, mysettings=None, all=0, mytree=None):
-		if mysettings is None:
-			mysettings = self.doebuild_settings
+	def getFetchMap(self, mypkg, useflags=None, mytree=None):
+		"""
+		Get the SRC_URI metadata as a dict which maps each file name to a
+		set of alternative URIs.
+
+		@param mypkg: cpv for an ebuild
+		@type mypkg: String
+		@param useflags: a collection of enabled USE flags, for evaluation of
+			conditionals
+		@type useflags: set, or None to enable all conditionals
+		@param mytree: The canonical path of the tree in which the ebuild
+			is located, or None for automatic lookup
+		@type mypkg: String
+		@returns: A dict which maps each file name to a set of alternative
+			URIs.
+		@rtype: dict
+		"""
+
 		try:
 			eapi, myuris = self.aux_get(mypkg,
 				["EAPI", "SRC_URI"], mytree=mytree)
@@ -460,33 +521,80 @@ class portdbapi(dbapi):
 			# Convert this to an InvalidDependString exception since callers
 			# already handle it.
 			raise portage.exception.InvalidDependString(
-				"getfetchlist(): aux_get() error reading "+mypkg+"; aborting.")
+				"getFetchMap(): aux_get() error reading "+mypkg+"; aborting.")
 
 		if not eapi_is_supported(eapi):
 			# Convert this to an InvalidDependString exception
 			# since callers already handle it.
 			raise portage.exception.InvalidDependString(
-				"getfetchlist(): '%s' has unsupported EAPI: '%s'" % \
+				"getFetchMap(): '%s' has unsupported EAPI: '%s'" % \
 				(mypkg, eapi.lstrip("-")))
 
-		if not all and useflags is None:
+		myuris = paren_reduce(myuris)
+		_src_uri_validate(mypkg, eapi, myuris)
+		myuris = use_reduce(myuris, uselist=useflags,
+			matchall=(useflags is None))
+		myuris = flatten(myuris)
+
+		uri_map = {}
+
+		uri = None
+		operator = None
+		myuris.reverse()
+		while myuris:
+			token = myuris.pop()
+			if uri is None:
+				uri = token
+				if myuris:
+					continue
+			if token == "->":
+				if eapi in ("0", "1"):
+					raise portage.exception.InvalidDependString(
+						("getFetchMap(): '%s' SRC_URI arrows are not " + \
+						"supported with EAPI='%s'") % (mypkg, eapi))
+
+				operator = token
+				continue
+			if operator is None:
+				distfile = os.path.basename(uri)
+				if not distfile:
+					raise portage.exception.InvalidDependString(
+						("getFetchMap(): '%s' SRC_URI has no file " + \
+						"name: '%s'") % (mypkg, uri))
+			else:
+				distfile = token
+				if "/" in distfile:
+					raise portage.exception.InvalidDependString(
+						("getFetchMap(): '%s' SRC_URI '/' character in " + \
+						"file name: '%s'") % (mypkg, distfile))
+			uri_set = uri_map.get(distfile)
+			if uri_set is None:
+				uri_set = set()
+				uri_map[distfile] = uri_set
+			uri_set.add(uri)
+			uri = None
+			operator = None
+
+		return uri_map
+
+	def getfetchlist(self, mypkg, useflags=None, mysettings=None,
+		all=0, mytree=None):
+
+		writemsg("!!! pordbapi.getfetchlist() is deprecated, " + \
+			"use getFetchMap() instead.\n", noiselevel=-1)
+
+		if all:
+			useflags = None
+		elif useflags is None:
+			if mysettings is None:
+				mysettings = self.doebuild_settings
 			mysettings.setcpv(mypkg, mydb=self)
 			useflags = mysettings["PORTAGE_USE"].split()
-
-		myurilist = paren_reduce(myuris)
-		myurilist = use_reduce(myurilist, uselist=useflags, matchall=all)
-		newuris = flatten(myurilist)
-
-		myfiles = []
-		for x in newuris:
-			mya = os.path.basename(x)
-			if not mya:
-				raise portage.exception.InvalidDependString(
-					"getfetchlist(): '%s' SRC_URI has no file name: '%s'" % \
-					(mypkg, x))
-			if not mya in myfiles:
-				myfiles.append(mya)
-		return [newuris, myfiles]
+		uri_map = self.getFetchMap(mypkg, useflags=useflags, mytree=mytree)
+		uris = set()
+		for uri_set in uri_map.itervalues():
+			uris.update(uri_set)
+		return [list(uris), uri_map.keys()]
 
 	def getfetchsizes(self, mypkg, useflags=None, debug=0):
 		# returns a filename:size dictionnary of remaining downloads
@@ -499,10 +607,7 @@ class portdbapi(dbapi):
 				print "[empty/missing/bad digest]: "+mypkg
 			return None
 		filesdict={}
-		if useflags is None:
-			myuris, myfiles = self.getfetchlist(mypkg,all=1)
-		else:
-			myuris, myfiles = self.getfetchlist(mypkg,useflags=useflags)
+		myfiles = self.getFetchMap(mypkg, useflags=useflags)
 		#XXX: maybe this should be improved: take partial downloads
 		# into account? check checksums?
 		for myfile in myfiles:
@@ -530,10 +635,12 @@ class portdbapi(dbapi):
 		return filesdict
 
 	def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False):
-		if not useflags:
+		if all:
+			useflags = None
+		elif useflags is None:
 			if mysettings:
 				useflags = mysettings["USE"].split()
-		myuri, myfiles = self.getfetchlist(mypkg, useflags=useflags, mysettings=mysettings, all=all)
+		myfiles = self.getFetchMap(mypkg, useflags=useflags)
 		myebuild = self.findname(mypkg)
 		pkgdir = os.path.dirname(myebuild)
 		mf = Manifest(pkgdir, self.mysettings["DISTDIR"])