From 90cfde34e4e74d4c11e0eaa053234e9a7b44ce53 Mon Sep 17 00:00:00 2001 From: Sebastian Luther Date: Thu, 30 Sep 2010 12:36:37 +0200 Subject: More repository handling cleanup --- pym/_emerge/MergeListItem.py | 2 +- pym/_emerge/main.py | 2 +- pym/portage/dbapi/porttree.py | 179 ++++++---------------- pym/portage/package/ebuild/_config/MaskManager.py | 12 +- pym/portage/repository/config.py | 140 ++++++++++++++--- 5 files changed, 170 insertions(+), 165 deletions(-) diff --git a/pym/_emerge/MergeListItem.py b/pym/_emerge/MergeListItem.py index 0eadb7d99..1dcc1780a 100644 --- a/pym/_emerge/MergeListItem.py +++ b/pym/_emerge/MergeListItem.py @@ -59,7 +59,7 @@ class MergeListItem(CompositeTask): colorize("GOOD", pkg.cpv)) portdb = pkg.root_config.trees["porttree"].dbapi - portdir_repo_name = portdb._repository_map.get(portdb.porttree_root) + portdir_repo_name = portdb.getRepositoryName(portdb.porttree_root) if portdir_repo_name: pkg_repo_name = pkg.repo if pkg_repo_name != portdir_repo_name: diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 48b0955d0..fe48763d0 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -1207,7 +1207,7 @@ def repo_name_duplicate_check(trees): if 'porttree' in root_trees: portdb = root_trees['porttree'].dbapi if portdb.settings.get('PORTAGE_REPO_DUPLICATE_WARN') != '0': - for repo_name, paths in portdb._ignored_repos: + for repo_name, paths in portdb.getIgnoredRepos(): k = (root, repo_name, portdb.getRepositoryPath(repo_name)) ignored_repos.setdefault(k, []).extend(paths) diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index c401c4745..8d71b2325 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -11,7 +11,6 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.data:portage_gid,secpass', 'portage.dbapi.dep_expand:dep_expand', 'portage.dep:dep_getkey,match_from_list,use_reduce', - 'portage.env.loaders:KeyValuePairFileLoader', 'portage.package.ebuild.doebuild:doebuild', 'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level', 'portage.util.listdir:listdir', @@ -20,7 +19,6 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage.cache.cache_errors import CacheError from portage.cache.mappings import Mapping -from portage.const import REPO_NAME_LOC from portage.dbapi import dbapi from portage.exception import PortageException, \ FileNotFound, InvalidDependString, InvalidPackageName @@ -66,7 +64,15 @@ class portdbapi(dbapi): def _categories(self): return self.settings.categories - def __init__(self, _unused_param=None, mysettings=None): + @property + def porttrees(self): + return list(reversed(self.settings.repositories.repoLocationList())) + + @property + def porttree_root(self): + return self.settings.repositories.mainRepoLocation() + + def __init__(self, mysettings=None): """ @param _unused_param: deprecated, use mysettings['PORTDIR'] instead @type _unused_param: None @@ -82,14 +88,8 @@ class portdbapi(dbapi): from portage import settings self.settings = config(clone=settings) - porttree_root = self.settings['PORTDIR'] - - if _unused_param is not None and _unused_param != porttree_root: - warnings.warn("The first parameter of the " + \ - "portage.dbapi.porttree.portdbapi" + \ - " constructor is now unused. " + \ - "mysettings['PORTDIR'] will be used instead.", - DeprecationWarning, stacklevel=2) + self.repositories = self.settings.repositories + self.treemap = self.repositories.treemap # This is strictly for use in aux_get() doebuild calls when metadata # is generated by the depend phase. It's safest to use a clone for @@ -97,7 +97,7 @@ class portdbapi(dbapi): # instance that is passed in. self.doebuild_settings = config(clone=self.settings) self.depcachedir = os.path.realpath(self.settings.depcachedir) - + if os.environ.get("SANDBOX_ON") == "1": # Make api consumers exempt from sandbox violations # when doing metadata cache updates. @@ -107,35 +107,14 @@ class portdbapi(dbapi): os.environ["SANDBOX_WRITE"] = \ ":".join(filter(None, sandbox_write)) - #adding porttress from repositories - porttrees = [os.path.realpath(x) for x in \ - self.settings.repositories.repoLocationList()] - self._missing_repo_names = self.settings.repositories.missing_repo_names - - # Ensure that each repo_name is unique. Later paths override - # earlier ones that correspond to the same name. - self._ignored_repos = self.settings.repositories.ignored_repos - - self._repository_map = {} - for k, v in self.settings.repositories.location_map.items(): - self._repository_map[os.path.realpath(k)] = v - self.treemap = {} - for k, v in self.settings.repositories.treemap.items(): - self.treemap[k] = os.path.realpath(v) - - self.porttrees = porttrees - porttree_root = os.path.realpath( - self.settings.repositories.mainRepoLocation()) - self.porttree_root = porttree_root - - self.eclassdb = eclass_cache.cache(porttree_root) + self.eclassdb = eclass_cache.cache(self.settings.repositories.mainRepoLocation()) # This is used as sanity check for aux_get(). If there is no # root eclass dir, we assume that PORTDIR is invalid or # missing. This check allows aux_get() to detect a missing # portage tree and return early by raising a KeyError. self._have_root_eclass_dir = os.path.isdir( - os.path.join(self.porttree_root, "eclass")) + os.path.join(self.settings.repositories.mainRepoLocation(), "eclass")) self.metadbmodule = self.settings.load_best_module("portdbapi.metadbmodule") @@ -143,102 +122,28 @@ class portdbapi(dbapi): self.xcache = {} self.frozen = 0 + #Create eclass dbs self._repo_info = {} - eclass_dbs = {porttree_root : self.eclassdb} - local_repo_configs = self.settings.repositories.prepos - default_loc_repo_config = None - repo_aliases = {} - if local_repo_configs is not None: - default_loc_repo_config = local_repo_configs.get('DEFAULT') - for repo_name, loc_repo_conf in local_repo_configs.items(): - if loc_repo_conf.aliases is not None: - for alias in loc_repo_conf.aliases: - overridden_alias = repo_aliases.get(alias) - if overridden_alias is not None: - writemsg_level(_("!!! Alias '%s' " \ - "created for '%s' overrides " \ - "'%s' alias in " \ - "'%s'\n") % (alias, repo_name, - overridden_alias, - 'repos.conf'), - level=logging.WARNING, noiselevel=-1) - repo_aliases[alias] = repo_name - - for path in self.porttrees: - if path in self._repo_info: + eclass_dbs = {self.settings.repositories.mainRepoLocation() : self.eclassdb} + for repo in self.repositories: + if repo.location in self._repo_info: continue - repo_name = self._repository_map.get(path) - - loc_repo_conf = None - if local_repo_configs is not None: - if repo_name is not None: - loc_repo_conf = local_repo_configs.get(repo_name) - if loc_repo_conf is None: - loc_repo_conf = default_loc_repo_config - - layout_filename = os.path.join(path, "metadata/layout.conf") - layout_file = KeyValuePairFileLoader(layout_filename, None, None) - layout_data, layout_errors = layout_file.load() - porttrees = [] - - masters = None - if loc_repo_conf is not None and \ - loc_repo_conf.masters is not None: - masters = loc_repo_conf.masters - else: - masters = layout_data.get('masters', '').split() - - for master_name in masters: - master_name = repo_aliases.get(master_name, master_name) - master_path = self.treemap.get(master_name) - if master_path is None: - writemsg_level(_("Unavailable repository '%s' " \ - "referenced by masters entry in '%s'\n") % \ - (master_name, layout_filename), - level=logging.ERROR, noiselevel=-1) - else: - porttrees.append(master_path) - - if not porttrees and path != porttree_root: - # Make PORTDIR the default master, but only if our - # heuristics suggest that it's necessary. - profiles_desc = os.path.join(path, 'profiles', 'profiles.desc') - eclass_dir = os.path.join(path, 'eclass') - if not os.path.isfile(profiles_desc) or \ - not os.path.isdir(eclass_dir): - porttrees.append(porttree_root) - - porttrees.append(path) - - if loc_repo_conf is not None and \ - loc_repo_conf.eclass_overrides is not None: - for other_name in loc_repo_conf.eclass_overrides: - other_path = self.treemap.get(other_name) - if other_path is None: - writemsg_level(_("Unavailable repository '%s' " \ - "referenced by eclass-overrides entry in " \ - "'%s'\n") % (other_name, - 'repos.conf'), - level=logging.ERROR, noiselevel=-1) - continue - porttrees.append(other_path) - eclass_db = None - for porttree in porttrees: - tree_db = eclass_dbs.get(porttree) + for eclass_location in repo.eclass_locations: + tree_db = eclass_dbs.get(eclass_location) if tree_db is None: - tree_db = eclass_cache.cache(porttree) - eclass_dbs[porttree] = tree_db + tree_db = eclass_cache.cache(eclass_location) + eclass_dbs[eclass_location] = tree_db if eclass_db is None: eclass_db = tree_db.copy() else: eclass_db.append(tree_db) - self._repo_info[path] = _repo_info(repo_name, path, eclass_db) + self._repo_info[repo.location] = _repo_info(repo.name, repo.location, eclass_db) #Keep a list of repo names, sorted by priority (highest priority first). - self._ordered_repo_name_list = tuple(self._repo_info[path].name for path in reversed(self.porttrees)) + self._ordered_repo_name_list = tuple(reversed(self.repositories.prepos_order)) self.auxdbmodule = self.settings.load_best_module("portdbapi.auxdbmodule") self.auxdb = {} @@ -261,7 +166,7 @@ class portdbapi(dbapi): filtered_auxdbkeys.sort() from portage.cache import metadata_overlay, volatile if not depcachedir_w_ok: - for x in self.porttrees: + for x in reversed(self.repositories.repoLocationList()): db_ro = self.auxdbmodule(self.depcachedir, x, filtered_auxdbkeys, gid=portage_gid, readonly=True) self.auxdb[x] = metadata_overlay.database( @@ -269,7 +174,7 @@ class portdbapi(dbapi): gid=portage_gid, db_rw=volatile.database, db_ro=db_ro) else: - for x in self.porttrees: + for x in reversed(self.repositories.repoLocationList()): if x in self.auxdb: continue # location, label, auxdbkeys @@ -278,7 +183,7 @@ class portdbapi(dbapi): if self.auxdbmodule is metadata_overlay.database: self.auxdb[x].db_ro.ec = self._repo_info[x].eclass_db if "metadata-transfer" not in self.settings.features: - for x in self.porttrees: + for x in reversed(self.repositories.repoLocationList()): if x in self._pregen_auxdb: continue if os.path.isdir(os.path.join(x, "metadata", "cache")): @@ -324,9 +229,7 @@ class portdbapi(dbapi): x.sync() def findLicensePath(self, license_name): - mytrees = self.porttrees[:] - mytrees.reverse() - for x in mytrees: + for x in reversed(self.repositories.repoLocationList()): license_path = os.path.join(x, "licenses", license_name) if os.access(license_path, os.R_OK): return license_path @@ -355,7 +258,10 @@ class portdbapi(dbapi): if the path does not correspond a known repository @rtype: String or None """ - return self._repository_map.get(canonical_repo_path) + try: + return self.repositories.get_name_for_location(canonical_repo_path) + except KeyError: + return None def getRepositories(self): """ @@ -369,7 +275,14 @@ class portdbapi(dbapi): """ Returns a list of repository paths that lack profiles/repo_name. """ - return self._missing_repo_names + return self.settings.repositories.missing_repo_names + + def getIgnoredRepos(self): + """ + Returns a list of repository paths that have been ignored, because + another repo with the same name exists. + """ + return self.settings.repositories.ignored_repos def findname2(self, mycpv, mytree=None, myrepo = None): """ @@ -402,8 +315,7 @@ class portdbapi(dbapi): if mytree: mytrees = [mytree] else: - mytrees = self.porttrees[:] - mytrees.reverse() + mytrees = reversed(self.repositories.repoLocationList()) relative_path = mysplit[0] + _os.sep + psplit[0] + _os.sep + \ mysplit[1] + ".ebuild" @@ -590,8 +502,7 @@ class portdbapi(dbapi): mydata = proc.metadata # do we have a origin repository name for the current package - mydata["repository"] = self._repository_map.get(mylocation, "") - + mydata["repository"] = self.repositories.get_name_for_location(mylocation) mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", [])) mydata["_mtime_"] = st[stat.ST_MTIME] @@ -770,7 +681,7 @@ class portdbapi(dbapi): if categories is None: categories = self.settings.categories if trees is None: - trees = self.porttrees + trees = reversed(self.repositories.repoLocationList()) for x in categories: for oroot in trees: for y in listdir(oroot+"/"+x, EmptyOnError=1, ignorecvs=1, dirsonly=1): @@ -803,7 +714,7 @@ class portdbapi(dbapi): # assume it's iterable mytrees = mytree else: - mytrees = self.porttrees + mytrees = reversed(self.repositories.repoLocationList()) for oroot in mytrees: try: file_list = os.listdir(os.path.join(oroot, mycp)) @@ -899,7 +810,7 @@ class portdbapi(dbapi): mylist = self.cp_list(mykey) else: mylist = match_from_list(mydep, self.cp_list(mykey, - mytree=self._repository_map.get(mydep.repo))) + mytree=self.repositories.get_location_for_name(mydep.repo))) myval = "" settings = self.settings local_config = settings.local_config diff --git a/pym/portage/package/ebuild/_config/MaskManager.py b/pym/portage/package/ebuild/_config/MaskManager.py index 70ded2dae..c68a7a421 100644 --- a/pym/portage/package/ebuild/_config/MaskManager.py +++ b/pym/portage/package/ebuild/_config/MaskManager.py @@ -29,18 +29,16 @@ class MaskManager(object): lines = [] repo_lines = grabfile_package(os.path.join(repo.location, "profiles", "package.mask"), \ recursive=1, remember_source_file=True, verify_eapi=True) - masters = repo.masters - if masters is None: - masters = [] - main_repo = repositories.mainRepo() - if main_repo is not None: - masters.append(main_repo) - for master in masters: + for master in repo.masters: master_lines = grabfile_package(os.path.join(master.location, "profiles", "package.mask"), \ recursive=1, remember_source_file=True, verify_eapi=True) lines.append(stack_lists([master_lines, repo_lines], incremental=1, remember_source_file=True, warn_for_unmatched_removal=True, strict_warn_for_unmatched_removal=strict_umatched_removal)) + if not repo.masters: + lines.append(stack_lists([repo_lines], incremental=1, + remember_source_file=True, warn_for_unmatched_removal=True, + strict_warn_for_unmatched_removal=strict_umatched_removal)) repo_pkgmasklines.extend(append_repo(stack_lists(lines), repo.name, remember_source_file=True)) repo_pkgunmasklines = [] diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index 73cddff42..022c0c8d8 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -1,23 +1,27 @@ # Copyright 2010 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import codecs +import logging + try: from configparser import SafeConfigParser except ImportError: from ConfigParser import SafeConfigParser from portage import os from portage.const import USER_CONFIG_PATH, GLOBAL_CONFIG_PATH, REPO_NAME_LOC -from portage.util import normalize_path, writemsg, shlex_split +from portage.env.loaders import KeyValuePairFileLoader +from portage.util import normalize_path, writemsg, writemsg_level, shlex_split from portage.localization import _ from portage import _unicode_encode from portage import _encodings -import codecs - class RepoConfig(object): """Stores config of one repository""" - __slots__ = ['aliases', 'eclass_overrides', 'location', 'masters', 'main_repo', + + __slots__ = ['aliases', 'eclass_overrides', 'eclass_locations', 'location', 'user_location', 'masters', 'main_repo', 'missing_repo_name', 'name', 'priority', 'sync', 'format'] + def __init__(self, name, repo_opts): """Build a RepoConfig with options in repo_opts Try to read repo_name in repository location, but if @@ -31,12 +35,13 @@ class RepoConfig(object): if eclass_overrides is not None: eclass_overrides = tuple(eclass_overrides.split()) self.eclass_overrides = eclass_overrides + #Locations are computed later. + self.eclass_locations = None - masters = repo_opts.get('masters') - if masters is not None: - masters = tuple(masters.split()) - self.masters = masters + #Masters are only read from layout.conf. + self.masters = None + #The main-repo key makes only sense for the 'DEFAULT' section. self.main_repo = repo_opts.get('main-repo') priority = repo_opts.get('priority') @@ -57,18 +62,21 @@ class RepoConfig(object): format = format.strip() self.format = format - self.missing_repo_name = False - location = repo_opts.get('location') + self.user_location = location if location is not None: - location = normalize_path(location) if os.path.isdir(location): - repo_name = self._get_repo_name(location) - if repo_name: - name = repo_name - self.name = name + location = os.path.realpath(location) self.location = location + missing = True + if self.location is not None: + name, missing = self._read_repo_name(self.location) + elif name == "DEFAULT": + missing = False + self.name = name + self.missing_repo_name = missing + def update(self, new_repo): """Update repository with options in another RepoConfig""" if new_repo.aliases is not None: @@ -86,18 +94,20 @@ class RepoConfig(object): if new_repo.sync is not None: self.sync = new_repo.sync - def _get_repo_name(self, repo_path): - """Read repo_name from repo_path""" + def _read_repo_name(self, repo_path): + """ + Read repo_name from repo_path. + Returns repo_name, missing. + """ repo_name_path = os.path.join(repo_path, REPO_NAME_LOC) try: return codecs.open( _unicode_encode(repo_name_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace').readline().strip() + errors='replace').readline().strip(), False except EnvironmentError: - self.missing_repo_name = True - return "x-" + os.path.basename(repo_path) + return "x-" + os.path.basename(repo_path), True class RepoConfigLoader(object): """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf""" @@ -145,7 +155,6 @@ class RepoConfigLoader(object): for ov in overlays: if os.path.isdir(ov): repo = RepoConfig(None, {'location' : ov}) - if repo.name in prepos: old_location = prepos[repo.name].location if old_location is not None and old_location != repo.location: @@ -187,6 +196,35 @@ class RepoConfigLoader(object): self.missing_repo_names = frozenset(repo.location for repo in prepos.values() if repo.missing_repo_name) + #Parse layout.conf and read masters key. + for repo in prepos.values(): + if not repo.location: + continue + layout_filename = os.path.join(repo.location, "metadata", "layout.conf") + layout_file = KeyValuePairFileLoader(layout_filename, None, None) + layout_data, layout_errors = layout_file.load() + + masters = layout_data.get('masters') + if masters: + master = masters.plit() + repo.masters = masters + + #Take aliases into account. + new_prepos = {} + for repo_name, repo in prepos.items(): + names = set() + names.add(repo_name) + if repo.aliases: + names.update(repo.aliases) + + for name in names: + if name in new_prepos: + writemsg_level(_("!!! Repository name or alias '%s', " + \ + "defined for repository '%s', overrides " + \ + "existing alias or repository.\n") % (name, repo_name), level=logging.WARNING, noiselevel=-1) + new_prepos[name] = repo + prepos = new_prepos + for (name, r) in prepos.items(): if r.location is not None: location_map[r.location] = name @@ -221,6 +259,49 @@ class RepoConfigLoader(object): self._prepos_changed = True self._repo_location_list = [] + #The 'masters' key currently contains repo names. Replace them with the matching RepoConfig. + for repo_name, repo in prepos.items(): + if repo_name == "DEFAULT": + continue + if repo.masters is None: + if self.mainRepo() and repo_name != self.mainRepo().name: + repo.masters = self.mainRepo(), + else: + repo.masters = () + else: + master_repos = [] + for master_name in repo.masters: + if master_name not in prepos: + writemsg_level(_("Unavailable repository '%s' " \ + "referenced by masters entry in '%s'\n") % \ + (master_name, layout_filename), + level=logging.ERROR, noiselevel=-1) + else: + master_repos.append(prepos[master_name]) + repo.masters = tuple(master_repos) + + #The 'eclass_overrides' key currently contains repo names. Replace them with the matching repo paths. + for repo_name, repo in prepos.items(): + if repo_name == "DEFAULT": + continue + + eclass_locations = [] + eclass_locations.extend(master_repo.location for master_repo in repo.masters) + eclass_locations.append(repo.location) + + if repo.eclass_overrides: + for other_repo_name in eclass_overrides: + if other_repo_name in self.prepos: + eclass_locations.append(self.get_location_for_name(other_repo_name)) + else: + writemsg_level(_("Unavailable repository '%s' " \ + "referenced by eclass-overrides entry for " \ + "'%s'\n") % (other_name, repo_name), level=logging.ERROR, noiselevel=-1) + repo.eclass_locations = tuple(eclass_locations) + + self._prepos_changed = True + self._repo_location_list = [] + self._check_locations() def repoLocationList(self): @@ -231,7 +312,7 @@ class RepoConfigLoader(object): if self.prepos[repo].location is not None: _repo_location_list.append(self.prepos[repo].location) self._repo_location_list = tuple(_repo_location_list) - + self._prepos_changed = False return self._repo_location_list @@ -268,6 +349,21 @@ class RepoConfigLoader(object): if repo.format != "unavailable": yield repo + def get_name_for_location(self, location): + return self.location_map[location] + + def get_location_for_name(self, repo_name): + if repo_name is None: + # This simplifies code in places where + # we want to be able to pass in Atom.repo + # even if it is None. + return None + return self.treemap[repo_name] + + def __iter__(self): + for repo_name in self.prepos_order: + yield self.prepos[repo_name] + def load_repository_config(settings): #~ repoconfigpaths = [os.path.join(settings.global_config_path, "repos.conf")] #~ repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], -- cgit v1.2.3-1-g7c22