add profile-formats portage-1 awareness and enforcement

Currently it just spits warnings for implicit portage-1 format usage;
down the line, this will be converted into an explicit error.

For repo's that configure themselves as portage-1 or pms explicitly, enforce
it.

5 files changed, 78 insertions, 54 deletions

diff --git a/pym/portage/package/ebuild/_config/KeywordsManager.py b/pym/portage/package/ebuild/_config/KeywordsManager.py
index c2b834344..704ddca70 100644
--- a/pym/portage/package/ebuild/_config/KeywordsManager.py
+++ b/pym/portage/package/ebuild/_config/KeywordsManager.py
@@ -20,7 +20,7 @@ class KeywordsManager(object):
 				global_accept_keywords=""):
 		self._pkeywords_list = []
 		rawpkeywords = [grabdict_package(
-			os.path.join(x, "package.keywords"), recursive=1,
+			os.path.join(x[0], "package.keywords"), recursive=x[1],
 			verify_eapi=True) \
 			for x in profiles]
 		for pkeyworddict in rawpkeywords:
@@ -35,7 +35,7 @@ class KeywordsManager(object):
 
 		self._p_accept_keywords = []
 		raw_p_accept_keywords = [grabdict_package(
-			os.path.join(x, "package.accept_keywords"), recursive=1,
+			os.path.join(x[0], "package.accept_keywords"), recursive=x[1],
 			verify_eapi=True) \
 			for x in profiles]
 		for d in raw_p_accept_keywords:
diff --git a/pym/portage/package/ebuild/_config/LocationsManager.py b/pym/portage/package/ebuild/_config/LocationsManager.py
index a465935f2..92c838cc1 100644
--- a/pym/portage/package/ebuild/_config/LocationsManager.py
+++ b/pym/portage/package/ebuild/_config/LocationsManager.py
@@ -121,8 +121,8 @@ class LocationsManager(object):
 				if os.path.isdir(os.path.join(currentPath, x)))
 			if offenders:
 				writemsg((_("Profile '%(profile_path)s' in repository "
-					"'%(repo_name)s' is using 'portage-1' profile format, but "
-					"the repository is not configured for it.  This will break "
+					"'%(repo_name)s' is implicitly using 'portage-1' profile format, but "
+					"the repository profiles are not marked as that format.  This will break "
 					"in the future.  Please either convert the following paths "
 					"to files, or add\nprofile-format = portage-1\nto the "
 					"repositories layout.conf.  Files: '%(files)s'\n")
diff --git a/pym/portage/package/ebuild/_config/MaskManager.py b/pym/portage/package/ebuild/_config/MaskManager.py
index 08acf92ef..a2bfb19b1 100644
--- a/pym/portage/package/ebuild/_config/MaskManager.py
+++ b/pym/portage/package/ebuild/_config/MaskManager.py
@@ -33,22 +33,33 @@ class MaskManager(object):
 		# repo may be often referenced by others as the master.
 		pmask_cache = {}
 
-		def grab_pmask(loc):
+		def grab_pmask(loc, repo_config):
 			if loc not in pmask_cache:
-				pmask_cache[loc] = grabfile_package(
-						os.path.join(loc, "profiles", "package.mask"),
-						recursive=1, remember_source_file=True, verify_eapi=True)
+				path = os.path.join(loc, 'profiles', 'package.mask')
+				pmask_cache[loc] = grabfile_package(path,
+						recursive=repo_config.portage1_profiles,
+						remember_source_file=True, verify_eapi=True)
+				if repo_config.portage1_profiles_compat and os.path.isdir(path):
+					writemsg((_("Repository '%(repo_name)s' is implicitly using "
+						"'portage-1' profile format in its profiles/package.mask, but "
+						"the repository profiles are not marked as that format.  This will break "
+						"in the future.  Please either convert the following paths "
+						"to files, or add\nprofile-format = portage-1\nto the "
+						"repositories layout.conf.\n")
+						% dict(repo_name=repo.name)),
+						noiselevel=-1)
+
 			return pmask_cache[loc]
 
 		repo_pkgmasklines = []
 		for repo in repositories.repos_with_profiles():
 			lines = []
-			repo_lines = grab_pmask(repo.location)
+			repo_lines = grab_pmask(repo.location, repo)
 			removals = frozenset(line[0][1:] for line in repo_lines
 				if line[0][:1] == "-")
 			matched_removals = set()
 			for master in repo.masters:
-				master_lines = grab_pmask(master.location)
+				master_lines = grab_pmask(master.location, master)
 				for line in master_lines:
 					if line[0] in removals:
 						matched_removals.add(line[0])
@@ -91,6 +102,8 @@ class MaskManager(object):
 
 		repo_pkgunmasklines = []
 		for repo in repositories.repos_with_profiles():
+			if not repo.portage1_profiles:
+				continue
 			repo_lines = grabfile_package(os.path.join(repo.location, "profiles", "package.unmask"), \
 				recursive=1, remember_source_file=True, verify_eapi=True)
 			lines = stack_lists([repo_lines], incremental=1, \
@@ -102,11 +115,12 @@ class MaskManager(object):
 		#to allow profiles to override masks from their parent profiles.
 		profile_pkgmasklines = []
 		profile_pkgunmasklines = []
-		for x in profiles:
+		for x, portage1_mode in profiles:
 			profile_pkgmasklines.append(grabfile_package(
-				os.path.join(x, "package.mask"), recursive=1, remember_source_file=True, verify_eapi=True))
-			profile_pkgunmasklines.append(grabfile_package(
-				os.path.join(x, "package.unmask"), recursive=1, remember_source_file=True, verify_eapi=True))
+				os.path.join(x, "package.mask"), recursive=portage1_mode, remember_source_file=True, verify_eapi=True))
+			if portage1_mode:
+				profile_pkgunmasklines.append(grabfile_package(
+					os.path.join(x, "package.unmask"), recursive=1, remember_source_file=True, verify_eapi=True))
 		profile_pkgmasklines = stack_lists(profile_pkgmasklines, incremental=1, \
 			remember_source_file=True, warn_for_unmatched_removal=True,
 			strict_warn_for_unmatched_removal=strict_umatched_removal)
diff --git a/pym/portage/package/ebuild/_config/UseManager.py b/pym/portage/package/ebuild/_config/UseManager.py
index d7ef0f6b0..bf06bd79d 100644
--- a/pym/portage/package/ebuild/_config/UseManager.py
+++ b/pym/portage/package/ebuild/_config/UseManager.py
@@ -65,9 +65,9 @@ class UseManager(object):
 
 		self.repositories = repositories
 	
-	def _parse_file_to_tuple(self, file_name):
+	def _parse_file_to_tuple(self, file_name, recursive=True):
 		ret = []
-		lines = grabfile(file_name, recursive=1)
+		lines = grabfile(file_name, recursive=recursive)
 		eapi = read_corresponding_eapi_file(file_name)
 		useflag_re = _get_useflag_re(eapi)
 		for prefixed_useflag in lines:
@@ -82,10 +82,10 @@ class UseManager(object):
 				ret.append(prefixed_useflag)
 		return tuple(ret)
 
-	def _parse_file_to_dict(self, file_name, juststrings=False):
+	def _parse_file_to_dict(self, file_name, juststrings=False, recursive=True):
 		ret = {}
 		location_dict = {}
-		file_dict = grabdict_package(file_name, recursive=1, verify_eapi=True)
+		file_dict = grabdict_package(file_name, recursive=recursive, verify_eapi=True)
 		eapi = read_corresponding_eapi_file(file_name)
 		useflag_re = _get_useflag_re(eapi)
 		for k, v in file_dict.items():
@@ -132,10 +132,12 @@ class UseManager(object):
 		return ret
 
 	def _parse_profile_files_to_tuple_of_tuples(self, file_name, locations):
-		return tuple(self._parse_file_to_tuple(os.path.join(profile, file_name)) for profile in locations)
+		return tuple(self._parse_file_to_tuple(os.path.join(profile[0], file_name), recursive=profile[1])
+			for profile in locations)
 
 	def _parse_profile_files_to_tuple_of_dicts(self, file_name, locations, juststrings=False):
-		return tuple(self._parse_file_to_dict(os.path.join(profile, file_name), juststrings) for profile in locations)
+		return tuple(self._parse_file_to_dict(os.path.join(profile[0], file_name), juststrings, recursive=profile[1])
+			for profile in locations)
 
 	def getUseMask(self, pkg=None):
 		if pkg is None:
diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py
index c4c5e660f..cfbbad49a 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -210,6 +210,7 @@ class config(object):
 			self.module_priority = clone.module_priority
 			self.profile_path = clone.profile_path
 			self.profiles = clone.profiles
+			self.profiles_complex = clone.profiles_complex
 			self.packages = clone.packages
 			self.repositories = clone.repositories
 			self._iuse_implicit_match = clone._iuse_implicit_match
@@ -276,13 +277,9 @@ class config(object):
 			locations_manager = LocationsManager(config_root=config_root,
 				config_profile_path=config_profile_path, eprefix=eprefix,
 				local_config=local_config, target_root=target_root)
-			locations_manager.load_profiles([])
 
 			eprefix = locations_manager.eprefix
 			config_root = locations_manager.config_root
-			self.profiles = locations_manager.profiles
-			self.profile_path = locations_manager.profile_path
-			self.user_profile_dir = locations_manager.user_profile_dir
 			abs_user_config = locations_manager.abs_user_config
 
 			make_conf = getconfig(
@@ -301,6 +298,13 @@ class config(object):
 			eroot = locations_manager.eroot
 			self.global_config_path = locations_manager.global_config_path
 
+			make_globals = getconfig(os.path.join(self.global_config_path, 'make.globals'))
+			if make_globals is None:
+				make_globals = {}
+
+			for k, v in self._default_globals.items():
+				make_globals.setdefault(k, v)
+
 			if config_incrementals is None:
 				self.incrementals = INCREMENTALS
 			else:
@@ -342,19 +346,6 @@ class config(object):
 			self.configlist.append({})
 			self.configdict["pkginternal"] = self.configlist[-1]
 
-			self.packages_list = [grabfile_package(os.path.join(x, "packages"), verify_eapi=True) for x in self.profiles]
-			self.packages      = tuple(stack_lists(self.packages_list, incremental=1))
-			del self.packages_list
-			#self.packages = grab_stacked("packages", self.profiles, grabfile, incremental_lines=1)
-
-			# revmaskdict
-			self.prevmaskdict={}
-			for x in self.packages:
-				# Negative atoms are filtered by the above stack_lists() call.
-				if not isinstance(x, Atom):
-					x = Atom(x.lstrip('*'))
-				self.prevmaskdict.setdefault(x.cp, []).append(x)
-
 			# The expand_map is used for variable substitution
 			# in getconfig() calls, and the getconfig() calls
 			# update expand_map with the value of each variable
@@ -404,23 +395,39 @@ class config(object):
 
 			self.configdict["env"] = LazyItemsDict(self.backupenv)
 
-			for x in (self.global_config_path,):
-				self.mygcfg = getconfig(os.path.join(x, "make.globals"),
-					expand=expand_map)
-				if self.mygcfg:
-					break
-
-			if self.mygcfg is None:
-				self.mygcfg = {}
-
-			for k, v in self._default_globals.items():
-				self.mygcfg.setdefault(k, v)
-
-			self.configlist.append(self.mygcfg)
+			self.configlist.append(make_globals)
 			self.configdict["globals"]=self.configlist[-1]
 
 			self.make_defaults_use = []
 			self.mygcfg = {}
+
+			known_repos = []
+			for confs in [make_globals, make_conf, self.configdict["env"]]:
+				known_repos.extend(confs.get("PORTDIR", '').split())
+				known_repos.extend(confs.get("PORTDIR_OVERLAY", '').split())
+			known_repos = set(known_repos)
+
+			locations_manager.load_profiles(known_repos)
+
+			self.profiles = locations_manager.profiles
+			self.profiles_complex = locations_manager.profiles_complex
+			self.profile_path = locations_manager.profile_path
+			self.user_profile_dir = locations_manager.user_profile_dir
+
+			self.packages_list = [grabfile_package(os.path.join(x, "packages"), verify_eapi=True) for x in self.profiles]
+			self.packages      = tuple(stack_lists(self.packages_list, incremental=1))
+			del self.packages_list
+			#self.packages = grab_stacked("packages", self.profiles, grabfile, incremental_lines=1)
+
+			# revmaskdict
+			self.prevmaskdict={}
+			for x in self.packages:
+				# Negative atoms are filtered by the above stack_lists() call.
+				if not isinstance(x, Atom):
+					x = Atom(x.lstrip('*'))
+				self.prevmaskdict.setdefault(x.cp, []).append(x)
+
+
 			if self.profiles:
 				mygcfg_dlists = [getconfig(os.path.join(x, "make.defaults"),
 					expand=expand_map) for x in self.profiles]
@@ -550,11 +557,11 @@ class config(object):
 				self._repo_make_defaults[repo.name] = d
 
 			#Read package.keywords and package.accept_keywords.
-			self._keywords_manager = KeywordsManager(self.profiles, abs_user_config, \
+			self._keywords_manager = KeywordsManager(self.profiles_complex, abs_user_config, \
 				local_config, global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", ""))
 
 			#Read all USE related files from profiles and optionally from user config.
-			self._use_manager = UseManager(self.repositories, self.profiles, abs_user_config, user_config=local_config)
+			self._use_manager = UseManager(self.repositories, self.profiles_complex, abs_user_config, user_config=local_config)
 			#Initialize all USE related variables we track ourselves.
 			self.usemask = self._use_manager.getUseMask()
 			self.useforce = self._use_manager.getUseForce()
@@ -571,7 +578,7 @@ class config(object):
 					self.configdict["conf"].get("ACCEPT_LICENSE", ""))
 
 			#Read package.mask and package.unmask from profiles and optionally from user config
-			self._mask_manager = MaskManager(self.repositories, self.profiles,
+			self._mask_manager = MaskManager(self.repositories, self.profiles_complex,
 				abs_user_config, user_config=local_config,
 				strict_umatched_removal=_unmatched_removal)
 
@@ -630,7 +637,8 @@ class config(object):
 			archlist = stack_lists(archlist, incremental=1)
 			self.configdict["conf"]["PORTAGE_ARCHLIST"] = " ".join(archlist)
 
-			pkgprovidedlines = [grabfile(os.path.join(x, "package.provided"), recursive=1) for x in self.profiles]
+			pkgprovidedlines = [grabfile(os.path.join(loc, "package.provided"), recursive=recursive)
+				for loc, recursive in self.profiles_complex]
 			pkgprovidedlines = stack_lists(pkgprovidedlines, incremental=1)
 			has_invalid_data = False
 			for x in range(len(pkgprovidedlines)-1, -1, -1):