Add portdbapi support for a metadata/layout.conf file which

specifies information about the repository layout. Currently,
only a single "masters" attribute is supported, which is used
to specify names of repositories which satisfy dependencies
on eclasses and/or ebuilds. Each repository name should
correspond the value of a repo_name entry from one of the
repositories that is configured via the PORTDIR or
PORTDIR_OVERLAY variables.

Since layout.conf is now used to control eclass inheritance,
it is now safer to use overlays which contain forked eclasses
have names identical to those from the main tree. Such
eclasses will only apply to their containing repository and
any other repositories which reference their containing
repository via layout.conf. This solves bug #124041 by
containing eclass overrides so that they don't necessarily
apply to all ebuilds.

Thanks to Alistair Bush <ali_bush@g.o> for his initial patch
for layout.conf support in repoman (will be merged later).
See the "QA Overlay Layout support" thread on the gentoo-dev
mailing list for more information:

http://archives.gentoo.org/gentoo-dev/msg_33c61550b4ed2b7b25dd5a4110e1ec81.xml

svn path=/main/trunk/; revision=13291

7 files changed, 131 insertions, 53 deletions

diff --git a/man/portage.5 b/man/portage.5
index fac8c90eb..0ab4ec344 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -59,6 +59,11 @@ categories
 .BR /etc/portage/profile/
 site-specific overrides of \fB/etc/make.profile/\fR
 .TP
+.BR /usr/portage/metadata/
+.nf
+layout.conf
+.fi
+.TP
 .BR /usr/portage/profiles/
 .nf
 arch.list
@@ -543,6 +548,23 @@ media\-other
 .fi
 .RE
 .TP
+.BR /usr/portage/metadata/
+.RS
+.TP
+.BR layout.conf
+Specifies information about the repository layout. Currently, only a single
+"masters" attribute is supported, which is used to specify names of
+repositories which satisfy dependencies on eclasses and/or ebuilds. Each
+repository name should correspond the value of a \fBrepo_name\fR entry
+from one of the repositories that is configured via the \fBPORTDIR\fR or
+\fBPORTDIR_OVERLAY\fR variables (see \fBmake.conf\fR(5)).
+
+.I Example:
+.nf
+masters = gentoo java-overlay
+.fi
+.RE
+.TP
 .BR /usr/portage/profiles/
 Global Gentoo settings that are controlled by the developers.  To override 
 these settings, you can use the files in \fB/etc/portage/\fR.
diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index ec160fd02..772a97de4 100644
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -15464,41 +15464,12 @@ def emerge_main():
 		repo_name_check(trees)
 		config_protect_check(trees)
 
-	eclasses_overridden = {}
 	for mytrees in trees.itervalues():
 		mydb = mytrees["porttree"].dbapi
 		# Freeze the portdbapi for performance (memoize all xmatch results).
 		mydb.freeze()
-		eclasses_overridden.update(mydb.eclassdb._master_eclasses_overridden)
 	del mytrees, mydb
 
-	if eclasses_overridden and \
-		settings.get("PORTAGE_ECLASS_WARNING_ENABLE") != "0":
-		prefix = bad(" * ")
-		if len(eclasses_overridden) == 1:
-			writemsg(prefix + "Overlay eclass overrides " + \
-				"eclass from PORTDIR:\n", noiselevel=-1)
-		else:
-			writemsg(prefix + "Overlay eclasses override " + \
-				"eclasses from PORTDIR:\n", noiselevel=-1)
-		writemsg(prefix + "\n", noiselevel=-1)
-		for eclass_name in sorted(eclasses_overridden):
-			writemsg(prefix + "  '%s/%s.eclass'\n" % \
-				(eclasses_overridden[eclass_name], eclass_name),
-				noiselevel=-1)
-		writemsg(prefix + "\n", noiselevel=-1)
-		msg = "It is best to avoid overriding eclasses from PORTDIR " + \
-		"because it will trigger invalidation of cached ebuild metadata " + \
-		"that is distributed with the portage tree. If you must " + \
-		"override eclasses from PORTDIR then you are advised to add " + \
-		"FEATURES=\"metadata-transfer\" to /etc/make.conf and to run " + \
-		"`emerge --regen` after each time that you run `emerge --sync`. " + \
-		"Set PORTAGE_ECLASS_WARNING_ENABLE=\"0\" in /etc/make.conf if " + \
-		"you would like to disable this warning."
-		from textwrap import wrap
-		for line in wrap(msg, 72):
-			writemsg("%s%s\n" % (prefix, line), noiselevel=-1)
-
 	if "moo" in myfiles:
 		print """
 
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index f549ad5c1..4e07a2b90 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -1105,7 +1105,7 @@ class config(object):
 		"PORTAGE_BACKGROUND",
 		"PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_CALLER",
 		"PORTAGE_COUNTER_HASH",
-		"PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES",
+		"PORTAGE_ELOG_CLASSES",
 		"PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT",
 		"PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM",
 		"PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE",
@@ -5160,6 +5160,12 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m
 	mysettings["FILESDIR"] = pkg_dir+"/files"
 	mysettings["PF"]       = mypv
 
+	if hasattr(mydbapi, '_repo_info'):
+		mytree = os.path.dirname(os.path.dirname(pkg_dir))
+		repo_info = mydbapi._repo_info[mytree]
+		mysettings['PORTDIR'] = repo_info.portdir
+		mysettings['PORTDIR_OVERLAY'] = repo_info.portdir_overlay
+
 	mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
 	mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
 	mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py
index a8be01095..e3f6fc571 100644
--- a/pym/portage/cache/metadata.py
+++ b/pym/portage/cache/metadata.py
@@ -29,7 +29,7 @@ class database(flat_hash.database):
 		loc = location
 		super(database, self).__init__(location, *args, **config)
 		self.location = os.path.join(loc, "metadata","cache")
-		self.ec = portage.eclass_cache.cache(loc)
+		self.ec = None
 		self.raise_stat_collision = False
 
 	def _parse_data(self, data, cpv):
@@ -53,9 +53,11 @@ class database(flat_hash.database):
 
 		if "_eclasses_" not in d:
 			if "INHERITED" in d:
+				if self.ec is None:
+					self.ec = portage.eclass_cache.cache(self.location)
 				try:
 					d["_eclasses_"] = self.ec.get_eclass_data(
-						d["INHERITED"].split(), from_master_only=True)
+						d["INHERITED"].split())
 				except KeyError, e:
 					# INHERITED contains a non-existent eclass.
 					raise cache_errors.CacheCorruption(cpv, e)
diff --git a/pym/portage/cache/util.py b/pym/portage/cache/util.py
index 0f2685c0a..bdee52ddc 100644
--- a/pym/portage/cache/util.py
+++ b/pym/portage/cache/util.py
@@ -102,8 +102,7 @@ def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None,
 				# Even if _eclasses_ already exists, replace it with data from
 				# eclass_cache, in order to insert local eclass paths.
 				try:
-					eclasses = eclass_cache.get_eclass_data(inherited,
-						from_master_only=True)
+					eclasses = eclass_cache.get_eclass_data(inherited)
 				except KeyError:
 					# INHERITED contains a non-existent eclass.
 					noise.eclass_stale(x)
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 9d4b1523d..1a896b754 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -8,7 +8,8 @@ import portage
 portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.checksum',
 	'portage.dep:dep_getkey,match_from_list,paren_reduce,use_reduce',
-	'portage.util:ensure_dirs,writemsg',
+	'portage.env.loaders:KeyValuePairFileLoader',
+	'portage.util:ensure_dirs,writemsg,writemsg_level',
 	'portage.versions:best,catpkgsplit,pkgsplit,ver_regexp',
 )
 
@@ -25,7 +26,7 @@ from portage import eclass_cache, auxdbkeys, doebuild, flatten, \
 	listdir, dep_expand, eapi_is_supported, key_expand, dep_check, \
 	_eapi_is_deprecated
 
-import codecs, os, stat
+import codecs, logging, os, stat
 from itertools import izip
 
 def _src_uri_validate(cpv, eapi, src_uri):
@@ -95,6 +96,15 @@ def _src_uri_validate(cpv, eapi, src_uri):
 			"getFetchMap(): '%s' SRC_URI arrow missing right operand" % \
 			(cpv,))
 
+class _repo_info(object):
+	__slots__ = ('name', 'path', 'eclass_db', 'portdir', 'portdir_overlay')
+	def __init__(self, name, path, eclass_db):
+		self.name = name
+		self.path = path
+		self.eclass_db = eclass_db
+		self.portdir = eclass_db.porttrees[0]
+		self.portdir_overlay = ' '.join(eclass_db.porttrees[1:])
+
 class portdbapi(dbapi):
 	"""this tree will scan a portage directory located at root (passed to init)"""
 	portdbapi_instances = []
@@ -116,10 +126,8 @@ class portdbapi(dbapi):
 		# instance that is passed in.
 		self.doebuild_settings = config(clone=self.mysettings)
 
-		#self.root=settings["PORTDIR"]
+		porttree_root = os.path.realpath(porttree_root)
 		self.porttree_root = porttree_root
-		if porttree_root:
-			self.porttree_root = os.path.realpath(porttree_root)
 
 		self.depcachedir = os.path.realpath(self.mysettings.depcachedir)
 
@@ -132,8 +140,7 @@ class portdbapi(dbapi):
 				os.environ["SANDBOX_WRITE"] = \
 					":".join(filter(None, sandbox_write))
 
-		self.eclassdb = eclass_cache.cache(self.porttree_root,
-			overlays=self.mysettings["PORTDIR_OVERLAY"].split())
+		self.eclassdb = eclass_cache.cache(porttree_root)
 
 		# This is used as sanity check for aux_get(). If there is no
 		# root eclass dir, we assume that PORTDIR is invalid or
@@ -163,7 +170,43 @@ class portdbapi(dbapi):
 				# don't want to see a warning every time the portage module is
 				# imported.
 				pass
-		
+
+		self._repo_info = {}
+		eclass_dbs = {porttree_root : self.eclassdb}
+		for path in self.porttrees:
+			if path in self._repo_info:
+				continue
+
+			layout_filename = os.path.join(path, "metadata/layout.conf")
+			layout_file = KeyValuePairFileLoader(layout_filename, None, None)
+			layout_data, layout_errors = layout_file.load()
+			porttrees = []
+			for master_name in layout_data.get('masters', '').split():
+				master_path = self.treemap.get(master_name)
+				if master_path is None:
+					writemsg_level(("Unavailable repository '%s' " + \
+						"referenced by masters entry in '%s'\n") % \
+						(master_name, layout_filename),
+						level=logging.ERROR, noiselevel=-1)
+				else:
+					porttrees.append(master_path)
+			if not porttrees:
+				porttrees.append(porttree_root)
+			porttrees.append(path)
+
+			eclass_db = None
+			for porttree in porttrees:
+				tree_db = eclass_dbs.get(porttree)
+				if tree_db is None:
+					tree_db = eclass_cache.cache(porttree)
+				if eclass_db is None:
+					eclass_db = tree_db.copy()
+				else:
+					eclass_db.append(tree_db)
+
+			self._repo_info[path] = _repo_info(self._repository_map.get(path),
+				path, eclass_db)
+
 		self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule")
 		self.auxdb = {}
 		self._pregen_auxdb = {}
@@ -191,6 +234,10 @@ class portdbapi(dbapi):
 				if os.path.isdir(os.path.join(x, "metadata", "cache")):
 					self._pregen_auxdb[x] = self.metadbmodule(
 						x, "metadata/cache", filtered_auxdbkeys, readonly=True)
+					try:
+						self._pregen_auxdb[x].ec = self._repo_info[x].eclass_db
+					except AttributeError:
+						pass
 		# Selectively cache metadata in order to optimize dep matching.
 		self._aux_cache_keys = set(
 			["DEPEND", "EAPI", "INHERITED", "IUSE", "KEYWORDS", "LICENSE",
diff --git a/pym/portage/eclass_cache.py b/pym/portage/eclass_cache.py
index 670e9fb24..15602fc02 100644
--- a/pym/portage/eclass_cache.py
+++ b/pym/portage/eclass_cache.py
@@ -5,6 +5,7 @@
 
 __all__ = ["cache"]
 
+import warnings
 from portage.util import normalize_path, writemsg
 import errno, os, sys
 from portage.data import portage_gid
@@ -15,18 +16,47 @@ class cache(object):
 	Maintains the cache information about eclasses used in ebuild.
 	"""
 	def __init__(self, porttree_root, overlays=[]):
-		self.porttree_root = porttree_root
 
 		self.eclasses = {} # {"Name": ("location","_mtime_")}
 		self._eclass_locations = {}
 
 		# screw with the porttree ordering, w/out having bash inherit match it, and I'll hurt you.
 		# ~harring
-		self.porttrees = [self.porttree_root]+overlays
-		self.porttrees = tuple(map(normalize_path, self.porttrees))
-		self._master_eclass_root = os.path.join(self.porttrees[0],"eclass")
-		self._master_eclasses_overridden = {}
-		self.update_eclasses()
+		if porttree_root:
+			self.porttree_root = porttree_root
+			self.porttrees = [self.porttree_root] + overlays
+			self.porttrees = tuple(map(normalize_path, self.porttrees))
+			self._master_eclass_root = os.path.join(self.porttrees[0], "eclass")
+			self.update_eclasses()
+		else:
+			self.porttree_root = None
+			self.porttrees = ()
+			self._master_eclass_root = None
+
+	def copy(self):
+		return self.__copy__()
+
+	def __copy__(self):
+		result = self.__class__(None)
+		result.eclasses = self.eclasses.copy()
+		result._eclass_locations = self._eclass_locations.copy()
+		result.porttree_root = self.porttree_root
+		result.porttrees = self.porttrees
+		result._master_eclass_root = self._master_eclass_root
+		return result
+
+	def append(self, other):
+		"""
+		Append another instance to this instance. This will cause eclasses
+		from the other instance to override and eclases from this instance
+		that have the same name.
+		"""
+		if not isinstance(other, self.__class__):
+			raise TypeError(
+				"expected type %s, got %s" % (self.__class__, type(other)))
+		self.porttrees = self.porttrees + other.porttrees
+		self.eclasses.update(other.eclasses)
+		self._eclass_locations.update(other._eclass_locations)
 
 	def close_caches(self):
 		import traceback
@@ -77,8 +107,7 @@ class cache(object):
 						# It appears to be identical to the master,
 						# so prefer the master entry.
 						continue
-					else:
-						self._master_eclasses_overridden[ys] = x
+
 				self.eclasses[ys] = (x, mtime)
 				self._eclass_locations[ys] = x
 
@@ -100,8 +129,10 @@ class cache(object):
 		ec_dict = {}
 		for x in inherits:
 			ec_dict[x] = self.eclasses[x]
-			if from_master_only and \
-				self._eclass_locations[x] != self._master_eclass_root:
-				return None
+
+		if from_master_only is not False:
+			warnings.warn("portage.eclass_cache.cache.get_eclass_data(): " + \
+				"ignoring deprecated 'from_master_only' parameter",
+				DeprecationWarning)
 
 		return ec_dict