portage.dep: Integrate the functionality of flatten into use_reduce

9 files changed, 153 insertions, 54 deletions

diff --git a/bin/quickpkg b/bin/quickpkg
index 8734a2bf5..042e98cc2 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -21,7 +21,7 @@ except ImportError:
 from portage import os
 from portage import xpak
 from portage.dbapi.dep_expand import dep_expand
-from portage.dep import flatten, use_reduce
+from portage.dep import use_reduce
 from portage.exception import InvalidAtom, InvalidData, InvalidDependString, PackageSetNotFound
 from portage.util import ConfigProtect, ensure_dirs, shlex_split
 from portage.dbapi.vartree import dblink, tar_contents
@@ -78,7 +78,7 @@ def quickpkg_atom(options, infos, arg, eout):
 			iuse = [ x.lstrip("+-") for x in iuse.split() ]
 			use = use.split()
 			try:
-				restrict = flatten(use_reduce(restrict, uselist=use))
+				restrict = use_reduce(restrict, uselist=use, flat=True)
 			except InvalidDependString as e:
 				eout.eerror("Invalid RESTRICT metadata " + \
 					"for '%s': %s; skipping" % (cpv, str(e)))
diff --git a/bin/repoman b/bin/repoman
index 97566d3f1..740b92e78 100755
--- a/bin/repoman
+++ b/bin/repoman
@@ -986,7 +986,9 @@ check_ebuild_notadded = not \
 	(vcs == "svn" and repolevel < 3 and options.mode != "commit")
 
 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
-thirdpartymirrors = portage.flatten(list(repoman_settings.thirdpartymirrors().values()))
+thirdpartymirrors = []
+for v in repoman_settings.thirdpartymirrors().values():
+	thirdpartymirrors.extend(v)
 
 class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder):
 	"""
@@ -1448,9 +1450,9 @@ for x in scanlist:
 
 		if not src_uri_error:
 			# Check that URIs don't reference a server from thirdpartymirrors.
-			for uri in portage.flatten(portage.dep.use_reduce( \
+			for uri in portage.dep.use_reduce( \
 				myaux["SRC_URI"], matchall=True, is_src_uri=True, \
-				allow_src_uri_file_renames=eapi_has_src_uri_arrows(eapi))):
+				allow_src_uri_file_renames=eapi_has_src_uri_arrows(eapi), flat=True):
 				contains_mirror = False
 				for mirror in thirdpartymirrors:
 					if uri.startswith(mirror):
@@ -1464,7 +1466,7 @@ for x in scanlist:
 					"%s: '%s' found in thirdpartymirrors" % \
 					(relative_path, mirror))
 
-		provide = portage.flatten(portage.dep.use_reduce(pkg.metadata['PROVIDE'], matchall=1))
+		provide = portage.dep.use_reduce(pkg.metadata['PROVIDE'], matchall=1, flat=True)
 		provide_cps = []
 
 		# The Package class automatically evaluates USE conditionals.
@@ -1731,8 +1733,7 @@ for x in scanlist:
 			myuse = myaux["LICENSE"]
 			# Parse the LICENSE variable, remove USE conditions and
 			# flatten it.
-			myuse=portage.dep.use_reduce(myuse, matchall=1)
-			myuse=portage.flatten(myuse)
+			myuse=portage.dep.use_reduce(myuse, matchall=1, flat=True)
 			# Check each entry to ensure that it exists in PORTDIR's
 			# license directory.
 			for mypos in range(0,len(myuse)):
@@ -1761,14 +1762,14 @@ for x in scanlist:
 		#restrict checks
 		myrestrict = None
 		try:
-			myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1)
+			myrestrict = portage.dep.use_reduce(myaux["RESTRICT"], matchall=1, flat=True)
 		except portage.exception.InvalidDependString as e:
 			stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
 			fails["RESTRICT.syntax"].append(
 				"%s: RESTRICT: %s" % (relative_path, e))
 			del e
 		if myrestrict:
-			myrestrict = set(portage.flatten(myrestrict))
+			myrestrict = set(myrestrict)
 			mybadrestrict = myrestrict.difference(valid_restrict)
 			if mybadrestrict:
 				stats["RESTRICT.invalid"] += len(mybadrestrict)
diff --git a/pym/_emerge/Package.py b/pym/_emerge/Package.py
index 0cc36e18f..1671c4ef7 100644
--- a/pym/_emerge/Package.py
+++ b/pym/_emerge/Package.py
@@ -344,12 +344,12 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase):
 			getattr(self, "_set_" + k.lower())(k, v)
 		elif k in self._use_conditional_keys:
 			try:
-				reduced = use_reduce(v, matchall=1)
+				reduced = use_reduce(v, matchall=1, flat=True)
 			except portage.exception.InvalidDependString as e:
 				self._pkg._invalid_metadata(k + ".syntax", "%s: %s" % (k, e))
 			else:
 				if reduced and k == 'PROVIDE':
-					for x in portage.flatten(reduced):
+					for x in reduced:
 						if not isvalidatom(x):
 							self._pkg._invalid_metadata(k + ".syntax",
 								"%s: %s" % (k, x))
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 4b266b48b..a8866794c 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -10,7 +10,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.checksum',
 	'portage.data:portage_gid,secpass',
 	'portage.dbapi.dep_expand:dep_expand',
-	'portage.dep:dep_getkey,flatten,match_from_list,use_reduce',
+	'portage.dep:dep_getkey,match_from_list,use_reduce',
 	'portage.env.loaders:KeyValuePairFileLoader',
 	'portage.package.ebuild.doebuild:doebuild',
 	'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
@@ -658,7 +658,6 @@ class portdbapi(dbapi):
 
 		myuris = use_reduce(myuris, uselist=useflags, matchall=(useflags is None), \
 			is_src_uri=True, allow_src_uri_file_renames=eapi_has_src_uri_arrows(eapi))
-		myuris = flatten(myuris)
 
 		uri_map = OrderedDict()
 
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 6f587f7b4..ccb48dc76 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -10,7 +10,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.checksum:_perform_md5_merge@perform_md5',
 	'portage.data:portage_gid,portage_uid,secpass',
 	'portage.dbapi.dep_expand:dep_expand',
-	'portage.dep:dep_getkey,isjustname,flatten,match_from_list,' + \
+	'portage.dep:dep_getkey,isjustname,match_from_list,' + \
 	 	'use_reduce,_slot_re',
 	'portage.elog:elog_process',
 	'portage.locks:lockdir,unlockdir',
@@ -1795,7 +1795,7 @@ class vartree(object):
 			mylines, myuse = self.dbapi.aux_get(mycpv, ["PROVIDE", "USE"])
 			if mylines:
 				myuse = myuse.split()
-				mylines = flatten(use_reduce(mylines, uselist=myuse))
+				mylines = use_reduce(mylines, uselist=myuse, flat=True)
 				for myprovide in mylines:
 					mys = catpkgsplit(myprovide)
 					if not mys:
diff --git a/pym/portage/dep/__init__.py b/pym/portage/dep/__init__.py
index 82c0c4e71..4b6c2d0b0 100644
--- a/pym/portage/dep/__init__.py
+++ b/pym/portage/dep/__init__.py
@@ -221,7 +221,7 @@ def paren_enclose(mylist):
 	return " ".join(mystrparts)
 
 def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], is_src_uri=False, \
-	allow_src_uri_file_renames=False, opconvert=False):
+	allow_src_uri_file_renames=False, opconvert=False, flat=False):
 	"""
 	Takes a dep string and reduces the use? conditionals out, leaving an array
 	with subarrays. All redundant brackets are removed.
@@ -234,8 +234,16 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i
 	@type masklist: List
 	@param matchall: Treat all conditionals as active. Used by repoman. 
 	@type matchall: Bool
-	@param matchall: List of flags for which negated conditionals are always treated as inactive.
-	@type matchall: List
+	@param excludeall: List of flags for which negated conditionals are always treated as inactive.
+	@type excludeall: List
+	@param is_src_uri: Indicates if depstr represents a SRC_URI
+	@type is_src_uri: Bool
+	@param allow_src_uri_file_renames: Indicates if EAPI-2 SRC_URI arrows are allowed when parsing a SRC_URI
+	@type allow_src_uri_file_renames: Bool
+	@param opconvert: Put every operator as first element into it's argument list
+	@type opconvert: Bool
+	@param flat: Create a flat list of all tokens
+	@type flat: Bool
 	@rtype: List
 	@return: The use reduced depend array
 	"""
@@ -244,7 +252,10 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i
 			"Pass the original dep string instead.") % \
 			('portage.dep.use_reduce',), DeprecationWarning, stacklevel=2)
 		depstr = paren_enclose(depstr)
-	
+
+	if opconvert and flat:
+		raise ValueError("portage.dep.use_reduce: 'opconvert' and 'flat' are mutually exclusive")
+
 	def is_active(conditional):
 		if conditional.startswith("!"):
 			flag = conditional[1:-1]
@@ -292,6 +303,17 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i
 				l = stack.pop()
 				ignore = False
 
+				if flat:
+					if stack[level] and stack[level][-1][-1] == "?":
+						if is_active(stack[level][-1]):
+							stack[level].pop()
+							stack[level].extend(l)
+						else:
+							stack[level].pop()
+					else:
+						stack[level].extend(l)
+					continue
+
 				if stack[level]:
 					if stack[level][-1] == "||" and not l:
 						stack[level].pop()
@@ -406,6 +428,9 @@ def flatten(mylist):
 	@rtype: List
 	@return: A single list containing only non-list elements.
 	"""
+	warnings.warn(_("%s is deprecated and will be removed without replacement.") % \
+		('portage.dep.flatten',), DeprecationWarning, stacklevel=2)
+
 	newlist = []
 	for x in mylist:
 		if isinstance(x, list):
diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py
index 081bad3f4..20b3186ec 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -32,7 +32,7 @@ from portage.dbapi import dbapi
 from portage.dbapi.porttree import portdbapi
 from portage.dbapi.vartree import vartree
 from portage.dep import Atom, best_match_to_list, \
-	flatten, isvalidatom, match_from_list, match_to_list, \
+	isvalidatom, match_from_list, match_to_list, \
 	remove_slot, use_reduce
 from portage.eapi import eapi_exports_AA, eapi_supports_prefix, eapi_exports_replace_vars
 from portage.env.loaders import KeyValuePairFileLoader
@@ -1278,7 +1278,7 @@ class config(object):
 			E2BIG errors as in bug #262647.
 			"""
 			try:
-				licenses = set(flatten(use_reduce(settings['LICENSE'],uselist=use)))
+				licenses = set(use_reduce(settings['LICENSE'], uselist=use, flat=True))
 			except InvalidDependString:
 				licenses = set()
 			licenses.discard('||')
@@ -1303,7 +1303,7 @@ class config(object):
 
 		def _restrict(self, use, settings):
 			try:
-				restrict = set(flatten(use_reduce(settings['RESTRICT'],uselist=use)))
+				restrict = set(use_reduce(settings['RESTRICT'], uselist=use, flat=True))
 			except InvalidDependString:
 				restrict = set()
 			return ' '.join(sorted(restrict))
@@ -1917,7 +1917,7 @@ class config(object):
 		"""
 
 
-		licenses = set(flatten(use_reduce(metadata["LICENSE"], matchall=1)))
+		licenses = set(use_reduce(metadata["LICENSE"], matchall=1, flat=True))
 		licenses.discard('||')
 
 		acceptable_licenses = set()
@@ -1948,17 +1948,17 @@ class config(object):
 			for element in license_struct[1:]:
 				if isinstance(element, list):
 					if element:
-						ret.append(self._getMaskedLicenses(
-							element, acceptable_licenses))
-						if not ret[-1]:
+						tmp = self._getMaskedLicenses(element, acceptable_licenses)
+						if not tmp:
 							return []
+						ret.extend(tmp)
 				else:
 					if element in acceptable_licenses:
 						return []
 					ret.append(element)
 			# Return all masked licenses, since we don't know which combination
 			# (if any) the user will decide to unmask.
-			return flatten(ret)
+			return ret
 
 		ret = []
 		for element in license_struct:
@@ -2006,7 +2006,7 @@ class config(object):
 				for x in pproperties_list:
 					accept_properties.extend(x)
 
-		properties = set(flatten(use_reduce(metadata["PROPERTIES"], matchall=1)))
+		properties = set(use_reduce(metadata["PROPERTIES"], matchall=1, flat=True))
 		properties.discard('||')
 
 		acceptable_properties = set()
@@ -2037,17 +2037,18 @@ class config(object):
 			for element in properties_struct[1:]:
 				if isinstance(element, list):
 					if element:
-						ret.append(self._getMaskedProperties(
-							element, acceptable_properties))
-						if not ret[-1]:
+						tmp = self._getMaskedProperties(
+							element, acceptable_properties)
+						if not tmp:
 							return []
+						ret.extend(tmp)
 				else:
 					if element in acceptable_properties:
 						return[]
 					ret.append(element)
 			# Return all masked properties, since we don't know which combination
 			# (if any) the user will decide to unmask
-			return flatten(ret)
+			return ret
 
 		ret = []
 		for element in properties_struct:
@@ -2117,7 +2118,7 @@ class config(object):
 			myuse = mydbapi["USE"]
 		else:
 			myuse = mydbapi.aux_get(mycpv, ["USE"])[0]
-		virts = flatten(use_reduce(provides, uselist=myuse.split()))
+		virts = use_reduce(provides, uselist=myuse.split(), flat=True)
 
 		modified = False
 		cp = Atom(cpv_getkey(mycpv))
@@ -2681,8 +2682,11 @@ class config(object):
 		return self._thirdpartymirrors
 
 	def archlist(self):
-		return flatten([[myarch, "~" + myarch] \
-			for myarch in self["PORTAGE_ARCHLIST"].split()])
+		_archlist = []
+		for myarch in self["PORTAGE_ARCHLIST"].split():
+			_archlist.append(myarch)
+			_archlist.append("~" + myarch)
+		return _archlist
 
 	def selinux_enabled(self):
 		if getattr(self, "_selinux_enabled", None) is None:
diff --git a/pym/portage/tests/dep/testStandalone.py b/pym/portage/tests/dep/testStandalone.py
index a5d661cc3..e9f01df03 100644
--- a/pym/portage/tests/dep/testStandalone.py
+++ b/pym/portage/tests/dep/testStandalone.py
@@ -2,7 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 from portage.tests import TestCase
-from portage.dep import cpvequal, flatten
+from portage.dep import cpvequal
 from portage.exception import PortageException
 
 class TestStandalone(TestCase):
@@ -34,17 +34,3 @@ class TestStandalone(TestCase):
 		for cpv1, cpv2 in test_cases_xfail:
 			self.assertRaisesMsg("cpvequal("+cpv1+", "+cpv2+")", \
 				PortageException, cpvequal, cpv1, cpv2)
-
-
-	def testFlattenl(self):
-
-		test_cases = (
-			( [], [] ),
-			( [[]], [] ),
-			( [[], 1], [1] ),
-			( [1, [2, 3, [4]]], [1, 2, 3, 4] ),
-			( [1, [2], 3, [4]], [1, 2, 3, 4] ),
-		)
-
-		for not_flat, flat in test_cases:
-			self.assertEqual(flatten(not_flat), flat)
diff --git a/pym/portage/tests/dep/test_use_reduce.py b/pym/portage/tests/dep/test_use_reduce.py
index e952e93d0..d95663663 100644
--- a/pym/portage/tests/dep/test_use_reduce.py
+++ b/pym/portage/tests/dep/test_use_reduce.py
@@ -8,7 +8,7 @@ from portage.dep import use_reduce
 class UseReduceTestCase(object):
 	def __init__(self, deparray, uselist=[], masklist=[], \
 		matchall=0, excludeall=[], is_src_uri=False, \
-		allow_src_uri_file_renames=False, opconvert=False, expected_result=None):
+		allow_src_uri_file_renames=False, opconvert=False, flat=False, expected_result=None):
 		self.deparray = deparray
 		self.uselist = uselist
 		self.masklist = masklist
@@ -17,11 +17,13 @@ class UseReduceTestCase(object):
 		self.is_src_uri = is_src_uri
 		self.allow_src_uri_file_renames = allow_src_uri_file_renames
 		self.opconvert = opconvert
+		self.flat = flat
 		self.expected_result = expected_result
 
 	def run(self):
 		return use_reduce(self.deparray, self.uselist, self.masklist, \
-			self.matchall, self.excludeall, self.is_src_uri, self.allow_src_uri_file_renames, self.opconvert)
+			self.matchall, self.excludeall, self.is_src_uri, self.allow_src_uri_file_renames, \
+				self.opconvert, self.flat)
 				
 class UseReduce(TestCase):
 
@@ -318,6 +320,83 @@ class UseReduce(TestCase):
 				uselist = ["foo"],
 				opconvert = True,
 				expected_result = ["A", "B"]),
+			
+			#flat test
+			UseReduceTestCase(
+				"A",
+				flat = True,
+				expected_result = ["A"]),
+			UseReduceTestCase(
+				"( A )",
+				flat = True,
+				expected_result = ["A"]),
+			UseReduceTestCase(
+				"|| ( A B )",
+				flat = True,
+				expected_result = [ "||", "A", "B" ] ),
+			UseReduceTestCase(
+				"|| ( A || ( B C ) )",
+				flat = True,
+				expected_result = [ "||", "A", "||", "B", "C" ]),
+			UseReduceTestCase(
+				"|| ( A || ( B C D ) )",
+				flat = True,
+				expected_result = [ "||", "A", "||", "B", "C", "D" ]),
+			UseReduceTestCase(
+				"|| ( A || ( B || ( C D ) E ) )",
+				flat = True,
+				expected_result = [ "||", "A", "||", "B", "||", "C", "D", "E" ]),
+			UseReduceTestCase(
+				"( || ( ( ( A ) B ) ) )",
+				flat = True,
+				expected_result = [ "||", "A", "B"] ),
+			UseReduceTestCase(
+				"( || ( || ( ( A ) B ) ) )",
+				flat = True,
+				expected_result = [ "||", "||", "A", "B" ]),
+			UseReduceTestCase(
+				"( || ( || ( ( A ) B ) ) )",
+				flat = True,
+				expected_result = [ "||", "||", "A", "B" ]),
+			UseReduceTestCase(
+				"|| ( A )",
+				flat = True,
+				expected_result = ["||", "A"]),
+			UseReduceTestCase(
+				"( || ( || ( || ( A ) foo? ( B ) ) ) )",
+				expected_result = ["A"]),
+			UseReduceTestCase(
+				"( || ( || ( || ( A ) foo? ( B ) ) ) )",
+				uselist = ["foo"],
+				flat = True,
+				expected_result = [ "||", "||","||", "A", "B" ]),
+			UseReduceTestCase(
+				"( || ( || ( bar? ( A ) || ( foo? ( B ) ) ) ) )",
+				flat = True,
+				expected_result = ["||", "||","||"]),
+			UseReduceTestCase(
+				"( || ( || ( bar? ( A ) || ( foo? ( B ) ) ) ) )",
+				uselist = ["foo", "bar"],
+				flat = True,
+				expected_result = [ "||", "||", "A", "||", "B" ]),
+			UseReduceTestCase(
+				"A || ( ) foo? ( ) B",
+				flat = True,
+				expected_result = ["A", "||", "B"]),
+			UseReduceTestCase(
+				"|| ( A ) || ( B )",
+				flat = True,
+				expected_result = ["||", "A", "||", "B"]),
+			UseReduceTestCase(
+				"foo? ( A ) foo? ( B )",
+				flat = True,
+				expected_result = []),
+			UseReduceTestCase(
+				"foo? ( A ) foo? ( B )",
+				uselist = ["foo"],
+				flat = True,
+				expected_result = ["A", "B"]),
+			
 		)
 		
 		test_cases_xfail = (
@@ -348,10 +427,15 @@ class UseReduce(TestCase):
 			UseReduceTestCase("http://foo/bar -> foo? ( http://foo.com/foo )", is_src_uri = True, allow_src_uri_file_renames = True),
 			UseReduceTestCase("foo? ( http://foo/bar -> ) blah.tbz2", is_src_uri = True, allow_src_uri_file_renames = True),
 			UseReduceTestCase("http://foo/bar -> foo/blah.tbz2", is_src_uri = True, allow_src_uri_file_renames = True),
+
+			UseReduceTestCase(
+				"A",
+				opconvert = True,
+				flat = True),
 		)
 
 		for test_case in test_cases:
 			self.assertEqual(test_case.run(), test_case.expected_result)
 
 		for test_case in test_cases_xfail:
-			self.assertRaisesMsg(test_case.deparray, InvalidDependString, test_case.run)
+			self.assertRaisesMsg(test_case.deparray, (InvalidDependString, ValueError), test_case.run)