From e4ba8f36e6a4624f4fec61c7ce8bed0e3bd2fa01 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Fri, 22 Jun 2012 02:59:53 -0700 Subject: Add experimental EAPI 4-slot-abi support. Refer to 4-slot-abi.docbook for a full description. --- pym/_emerge/Dependency.py | 2 +- pym/_emerge/FakeVartree.py | 32 +- pym/_emerge/Package.py | 26 +- pym/_emerge/Scheduler.py | 5 +- pym/_emerge/create_depgraph_params.py | 14 + pym/_emerge/depgraph.py | 312 +++++++++++++++++- pym/_emerge/main.py | 23 ++ pym/_emerge/resolver/backtracking.py | 17 +- pym/portage/__init__.py | 2 +- pym/portage/dbapi/__init__.py | 4 +- pym/portage/dbapi/porttree.py | 4 +- pym/portage/dbapi/vartree.py | 5 + pym/portage/dep/__init__.py | 126 +++++++- pym/portage/dep/_slot_abi.py | 92 ++++++ pym/portage/eapi.py | 6 +- pym/portage/package/ebuild/doebuild.py | 24 +- pym/portage/tests/dep/testAtom.py | 19 ++ pym/portage/tests/dep/test_isvalidatom.py | 9 + pym/portage/tests/dep/test_match_from_list.py | 19 +- pym/portage/tests/emerge/test_emerge_slot_abi.py | 198 ++++++++++++ pym/portage/tests/resolver/test_complete_graph.py | 4 +- pym/portage/tests/resolver/test_slot_abi.py | 376 ++++++++++++++++++++++ pym/portage/versions.py | 15 +- 23 files changed, 1287 insertions(+), 47 deletions(-) create mode 100644 pym/portage/dep/_slot_abi.py create mode 100644 pym/portage/tests/emerge/test_emerge_slot_abi.py create mode 100644 pym/portage/tests/resolver/test_slot_abi.py (limited to 'pym') diff --git a/pym/_emerge/Dependency.py b/pym/_emerge/Dependency.py index c2d36b2dc..2ec860f83 100644 --- a/pym/_emerge/Dependency.py +++ b/pym/_emerge/Dependency.py @@ -6,7 +6,7 @@ from _emerge.DepPriority import DepPriority class Dependency(SlotObject): __slots__ = ("atom", "blocker", "child", "depth", - "parent", "onlydeps", "priority", "root", + "parent", "onlydeps", "priority", "root", "want_update", "collapsed_parent", "collapsed_priority") def __init__(self, **kwargs): SlotObject.__init__(self, **kwargs) diff --git a/pym/_emerge/FakeVartree.py b/pym/_emerge/FakeVartree.py index e8454e8e9..e62058540 100644 --- a/pym/_emerge/FakeVartree.py +++ b/pym/_emerge/FakeVartree.py @@ -10,11 +10,17 @@ from _emerge.Package import Package from _emerge.PackageVirtualDbapi import PackageVirtualDbapi from portage.const import VDB_PATH from portage.dbapi.vartree import vartree +from portage.dep._slot_abi import find_built_slot_abi_atoms +from portage.eapi import _get_eapi_attrs +from portage.exception import InvalidDependString from portage.repository.config import _gen_valid_repo from portage.update import grab_updates, parse_updates, update_dbentries if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode class FakeVardbapi(PackageVirtualDbapi): """ @@ -39,9 +45,10 @@ class FakeVartree(vartree): is not a matching ebuild in the tree). Instances of this class are not populated until the sync() method is called.""" def __init__(self, root_config, pkg_cache=None, pkg_root_config=None, - dynamic_deps=True): + dynamic_deps=True, ignore_built_slot_abi_deps=False): self._root_config = root_config self._dynamic_deps = dynamic_deps + self._ignore_built_slot_abi_deps = ignore_built_slot_abi_deps if pkg_root_config is None: pkg_root_config = self._root_config self._pkg_root_config = pkg_root_config @@ -101,7 +108,18 @@ class FakeVartree(vartree): self._aux_get_history.add(pkg) # We need to check the EAPI, and this also raises # a KeyError to the caller if appropriate. - installed_eapi, repo = self._aux_get(pkg, ["EAPI", "repository"]) + pkg_obj = self.dbapi._cpv_map[pkg] + installed_eapi = pkg_obj.metadata['EAPI'] + repo = pkg_obj.metadata['repository'] + eapi_attrs = _get_eapi_attrs(installed_eapi) + built_slot_abi_atoms = None + + if eapi_attrs.slot_abi and not self._ignore_built_slot_abi_deps: + try: + built_slot_abi_atoms = find_built_slot_abi_atoms(pkg_obj) + except InvalidDependString: + pass + try: # Use the live ebuild metadata if possible. repo = _gen_valid_repo(repo) @@ -118,6 +136,16 @@ class FakeVartree(vartree): if not (portage.eapi_is_supported(live_metadata["EAPI"]) and \ portage.eapi_is_supported(installed_eapi)): raise KeyError(pkg) + + # preserve built SLOT/ABI := operator deps + if built_slot_abi_atoms: + live_eapi_attrs = _get_eapi_attrs(live_metadata["EAPI"]) + if not live_eapi_attrs.slot_abi: + raise KeyError(pkg) + for k, v in built_slot_abi_atoms.items(): + live_metadata[k] += (" " + + " ".join(_unicode(atom) for atom in v)) + self.dbapi.aux_update(pkg, live_metadata) except (KeyError, portage.exception.PortageException): if self._global_updates is None: diff --git a/pym/_emerge/Package.py b/pym/_emerge/Package.py index 18bc2014f..7bf7181ba 100644 --- a/pym/_emerge/Package.py +++ b/pym/_emerge/Package.py @@ -26,8 +26,9 @@ class Package(Task): "root_config", "type_name", "category", "counter", "cp", "cpv_split", "inherited", "iuse", "mtime", - "pf", "root", "slot", "slot_atom", "version") + \ - ("_invalid", "_raw_metadata", "_masks", "_use", "_visible") + "pf", "root", "slot", "slot_abi", "slot_atom", "version") + \ + ("_invalid", "_raw_metadata", "_masks", "_use", + "_validated_atoms", "_visible") metadata_keys = [ "BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "EAPI", @@ -58,6 +59,7 @@ class Package(Task): "SLOT: invalid value: '%s'" % self.metadata["SLOT"]) self.cp = self.cpv.cp self.slot = self.cpv.slot + self.slot_abi = self.cpv.slot_abi # sync metadata with validated repo (may be UNKNOWN_REPO) self.metadata['repository'] = self.cpv.repo if (self.iuse.enabled or self.iuse.disabled) and \ @@ -107,6 +109,17 @@ class Package(Task): self._visible = self._eval_visiblity(self.masks) return self._visible + @property + def validated_atoms(self): + """ + Returns *all* validated atoms from the deps, regardless + of USE conditionals, with USE conditionals inside + atoms left unevaluated. + """ + if self._validated_atoms is None: + self._validate_deps() + return self._validated_atoms + @classmethod def _gen_hash_key(cls, cpv=None, installed=None, onlydeps=None, operation=None, repo_name=None, root_config=None, @@ -160,16 +173,21 @@ class Package(Task): dep_eapi = None dep_valid_flag = None + validated_atoms = [] for k in self._dep_keys: v = self.metadata.get(k) if not v: continue try: - use_reduce(v, eapi=dep_eapi, matchall=True, - is_valid_flag=dep_valid_flag, token_class=Atom) + validated_atoms.extend(use_reduce(v, eapi=dep_eapi, + matchall=True, is_valid_flag=dep_valid_flag, + token_class=Atom, flat=True)) except InvalidDependString as e: self._metadata_exception(k, e) + self._validated_atoms = frozenset(atom for atom in + validated_atoms if isinstance(atom, Atom)) + k = 'PROVIDE' v = self.metadata.get(k) if v: diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index 30a7e101b..0b72a4cfc 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -328,12 +328,15 @@ class Scheduler(PollScheduler): self._set_graph_config(graph_config) self._blocker_db = {} dynamic_deps = self.myopts.get("--dynamic-deps", "y") != "n" + ignore_built_slot_abi_deps = self.myopts.get( + "--ignore-built-slot-abi-deps", "n") == "y" for root in self.trees: if self._uninstall_only: continue if graph_config is None: fake_vartree = FakeVartree(self.trees[root]["root_config"], - pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps) + pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps, + ignore_built_slot_abi_deps=ignore_built_slot_abi_deps) fake_vartree.sync() else: fake_vartree = graph_config.trees[root]['vartree'] diff --git a/pym/_emerge/create_depgraph_params.py b/pym/_emerge/create_depgraph_params.py index 8f15c6813..33d413aa3 100644 --- a/pym/_emerge/create_depgraph_params.py +++ b/pym/_emerge/create_depgraph_params.py @@ -15,12 +15,22 @@ def create_depgraph_params(myopts, myaction): # complete: completely account for all known dependencies # remove: build graph for use in removing packages # rebuilt_binaries: replace installed packages with rebuilt binaries + # rebuild_if_new_slot_abi: rebuild or reinstall packages when + # SLOT/ABI := operator dependencies can be satisfied by a newer + # SLOT/ABI, so that older packages slots will become eligible for + # removal by the --depclean action as soon as possible + # ignore_built_slot_abi_deps: ignore the SLOT/ABI := operator parts + # of dependencies that have been recorded when packages where built myparams = {"recurse" : True} bdeps = myopts.get("--with-bdeps") if bdeps is not None: myparams["bdeps"] = bdeps + ignore_built_slot_abi_deps = myopts.get("--ignore-built-slot-abi-deps") + if ignore_built_slot_abi_deps is not None: + myparams["ignore_built_slot_abi_deps"] = ignore_built_slot_abi_deps + dynamic_deps = myopts.get("--dynamic-deps") if dynamic_deps is not None: myparams["dynamic_deps"] = dynamic_deps @@ -31,6 +41,10 @@ def create_depgraph_params(myopts, myaction): myparams["selective"] = True return myparams + rebuild_if_new_slot_abi = myopts.get('--rebuild-if-new-slot-abi') + if rebuild_if_new_slot_abi is not None: + myparams['rebuild_if_new_slot_abi'] = rebuild_if_new_slot_abi + if "--update" in myopts or \ "--newuse" in myopts or \ "--reinstall" in myopts or \ diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 0c014bcfd..2547fa4e4 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -22,6 +22,7 @@ from portage.dbapi.dep_expand import dep_expand from portage.dep import Atom, best_match_to_list, extract_affecting_use, \ check_required_use, human_readable_required_use, match_from_list, \ _repo_separator +from portage.dep._slot_abi import ignore_built_slot_abi_deps from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use from portage.exception import InvalidAtom, InvalidDependString, PortageException from portage.output import colorize, create_color_func, \ @@ -110,6 +111,8 @@ class _frozen_depgraph_config(object): self._pkg_cache = {} self._highest_license_masked = {} dynamic_deps = myopts.get("--dynamic-deps", "y") != "n" + ignore_built_slot_abi_deps = myopts.get( + "--ignore-built-slot-abi-deps", "n") == "y" for myroot in trees: self.trees[myroot] = {} # Create a RootConfig instance that references @@ -124,7 +127,8 @@ class _frozen_depgraph_config(object): FakeVartree(trees[myroot]["root_config"], pkg_cache=self._pkg_cache, pkg_root_config=self.roots[myroot], - dynamic_deps=dynamic_deps) + dynamic_deps=dynamic_deps, + ignore_built_slot_abi_deps=ignore_built_slot_abi_deps) self.pkgsettings[myroot] = portage.config( clone=self.trees[myroot]["vartree"].settings) @@ -404,6 +408,7 @@ class _dynamic_depgraph_config(object): self._needed_license_changes = backtrack_parameters.needed_license_changes self._needed_use_config_changes = backtrack_parameters.needed_use_config_changes self._runtime_pkg_mask = backtrack_parameters.runtime_pkg_mask + self._slot_abi_replace_installed = backtrack_parameters.slot_abi_replace_installed self._need_restart = False # For conditions that always require user intervention, such as # unsatisfied REQUIRED_USE (currently has no autounmask support). @@ -413,6 +418,8 @@ class _dynamic_depgraph_config(object): self._autounmask = depgraph._frozen_config.myopts.get('--autounmask') != 'n' self._success_without_autounmask = False self._traverse_ignored_deps = False + self._complete_mode = False + self._slot_abi_deps = {} for myroot in depgraph._frozen_config.trees: self.sets[myroot] = _depgraph_sets() @@ -830,6 +837,7 @@ class depgraph(object): slot_parent_atoms.update(parent_atoms) conflict_pkgs = [] + conflict_atoms = {} for pkg in slot_nodes: if self._dynamic_config._allow_backtracking and \ @@ -860,6 +868,7 @@ class depgraph(object): parent_atoms.add(parent_atom) else: all_match = False + conflict_atoms.setdefault(parent_atom, set()).add(pkg) if not all_match: conflict_pkgs.append(pkg) @@ -867,8 +876,14 @@ class depgraph(object): if conflict_pkgs and \ self._dynamic_config._allow_backtracking and \ not self._accept_blocker_conflicts(): - self._slot_confict_backtrack(root, slot_atom, - slot_parent_atoms, conflict_pkgs) + remaining = [] + for pkg in conflict_pkgs: + if not self._slot_conflict_backtrack_abi(pkg, + slot_nodes, conflict_atoms): + remaining.append(pkg) + if remaining: + self._slot_confict_backtrack(root, slot_atom, + slot_parent_atoms, remaining) def _slot_confict_backtrack(self, root, slot_atom, all_parents, conflict_pkgs): @@ -931,6 +946,219 @@ class depgraph(object): writemsg_level("".join("%s\n" % l for l in msg), noiselevel=-1, level=logging.DEBUG) + def _slot_conflict_backtrack_abi(self, pkg, slot_nodes, conflict_atoms): + """ + If one or more conflict atoms have a SLOT/ABI dep that can be resolved + by rebuilding the parent package, then schedule the rebuild via + backtracking, and return True. Otherwise, return False. + """ + + found_update = False + for parent_atom, conflict_pkgs in conflict_atoms.items(): + parent, atom = parent_atom + if atom.slot_abi_op != "=" or not parent.built: + continue + + if pkg not in conflict_pkgs: + continue + + for other_pkg in slot_nodes: + if other_pkg in conflict_pkgs: + continue + + dep = Dependency(atom=atom, child=other_pkg, + parent=parent, root=pkg.root) + + if self._slot_abi_update_probe(dep): + self._slot_abi_update_backtrack(dep) + found_update = True + + return found_update + + def _slot_abi_update_backtrack(self, dep, new_child_slot=None): + if new_child_slot is None: + child = dep.child + else: + child = new_child_slot + if "--debug" in self._frozen_config.myopts: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking to due missed slot abi update:") + msg.append(" child package: %s" % child) + if new_child_slot is not None: + msg.append(" new child slot package: %s" % new_child_slot) + msg.append(" parent package: %s" % dep.parent) + msg.append(" atom: %s" % dep.atom) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + + # mask unwanted binary packages if necessary + abi_masks = {} + if new_child_slot is None: + if not child.installed: + abi_masks.setdefault(child, {})["slot_abi_mask_built"] = dep + if not dep.parent.installed: + abi_masks.setdefault(dep.parent, {})["slot_abi_mask_built"] = dep + if abi_masks: + config.setdefault("slot_abi_mask_built", {}).update(abi_masks) + + # trigger replacement of installed packages if necessary + abi_reinstalls = set() + if dep.parent.installed: + abi_reinstalls.add((dep.parent.root, dep.parent.slot_atom)) + if new_child_slot is None and child.installed: + abi_reinstalls.add((child.root, child.slot_atom)) + if abi_reinstalls: + config.setdefault("slot_abi_replace_installed", + set()).update(abi_reinstalls) + + self._dynamic_config._need_restart = True + + def _slot_abi_update_probe(self, dep, new_child_slot=False): + """ + SLOT/ABI := operators tend to prevent updates from getting pulled in, + since installed packages pull in packages with the SLOT/ABI that they + were built against. Detect this case so that we can schedule rebuilds + and reinstalls when appropriate. + NOTE: This function only searches for updates that involve upgrades + to higher versions, since the logic required to detect when a + downgrade would be desirable is not implemented. + """ + + debug = "--debug" in self._frozen_config.myopts + + for replacement_parent in self._iter_similar_available(dep.parent, + dep.parent.slot_atom): + + for atom in replacement_parent.validated_atoms: + if not atom.slot_abi_op == "=" or \ + atom.blocker or \ + atom.cp != dep.atom.cp: + continue + + # Discard USE deps, we're only searching for an approximate + # pattern, and dealing with USE states is too complex for + # this purpose. + atom = atom.without_use + + if replacement_parent.built and \ + portage.dep._match_slot(atom, dep.child): + # Our selected replacement_parent appears to be built + # for the existing child selection. So, discard this + # parent and search for another. + break + + for pkg in self._iter_similar_available( + dep.child, atom): + if pkg.slot == dep.child.slot and \ + pkg.slot_abi == dep.child.slot_abi: + # If SLOT/ABI is identical, then there's + # no point in updating. + continue + if new_child_slot: + if pkg.slot == dep.child.slot: + continue + if pkg < dep.child: + # the new slot only matters if the + # package version is higher + continue + else: + if pkg.slot != dep.child.slot: + continue + if pkg < dep.child: + # be careful not to trigger a rebuild when + # the only version available with a + # different slot_abi is an older version + continue + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_abi_update_probe:") + msg.append(" existing child package: %s" % dep.child) + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" new child package: %s" % pkg) + msg.append(" new parent package: %s" % replacement_parent) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return pkg + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_abi_update_probe:") + msg.append(" existing child package: %s" % dep.child) + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" new child package: %s" % None) + msg.append(" new parent package: %s" % None) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return None + + def _iter_similar_available(self, graph_pkg, atom): + """ + Given a package that's in the graph, do a rough check to + see if a similar package is available to install. The given + graph_pkg itself may be yielded only if it's not installed. + """ + for pkg in self._iter_match_pkgs_any( + graph_pkg.root_config, atom): + if pkg.cp != graph_pkg.cp: + # discard old-style virtual match + continue + if pkg.installed: + continue + if pkg in self._dynamic_config._runtime_pkg_mask: + continue + if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + continue + if not self._pkg_visibility_check(pkg): + continue + yield pkg + + def _slot_abi_trigger_reinstalls(self): + """ + Search for packages with slot-abi deps on older slots, and schedule + rebuilds if they can link to a newer slot that's in the graph. + """ + + rebuild_if_new_slot_abi = self._dynamic_config.myparams.get( + "rebuild_if_new_slot_abi", "y") == "y" + + for slot_key, slot_info in self._dynamic_config._slot_abi_deps.items(): + + for dep in slot_info: + if not (dep.child.built and dep.parent and + isinstance(dep.parent, Package) and dep.parent.built): + continue + + # Check for slot update first, since we don't want to + # trigger reinstall of the child package when a newer + # slot will be used instead. + if rebuild_if_new_slot_abi: + new_child = self._slot_abi_update_probe(dep, + new_child_slot=True) + if new_child: + self._slot_abi_update_backtrack(dep, + new_child_slot=new_child) + break + + if dep.want_update: + if self._slot_abi_update_probe(dep): + self._slot_abi_update_backtrack(dep) + break + def _reinstall_for_flags(self, pkg, forced_flags, orig_use, orig_iuse, cur_use, cur_iuse): """Return a set of flags that trigger reinstallation, or None if there @@ -1326,6 +1554,17 @@ class depgraph(object): depth = min(pkg.depth, depth) pkg.depth = depth deep = self._dynamic_config.myparams.get("deep", 0) + update = "--update" in self._frozen_config.myopts + + dep.want_update = (not self._dynamic_config._complete_mode and + (arg_atoms or update) and + not (deep is not True and depth > deep)) + + dep.child = pkg + if (not pkg.onlydeps and pkg.built and + dep.atom and dep.atom.slot_abi_built): + self._add_slot_abi_dep(dep) + recurse = deep is True or depth + 1 <= deep dep_stack = self._dynamic_config._dep_stack if "recurse" not in self._dynamic_config.myparams: @@ -1357,6 +1596,14 @@ class depgraph(object): self._dynamic_config._parent_atoms[pkg] = parent_atoms parent_atoms.add(parent_atom) + def _add_slot_abi_dep(self, dep): + slot_key = (dep.root, dep.child.slot_atom) + slot_info = self._dynamic_config._slot_abi_deps.get(slot_key) + if slot_info is None: + slot_info = [] + self._dynamic_config._slot_abi_deps[slot_key] = slot_info + slot_info.append(dep) + def _add_slot_conflict(self, pkg): self._dynamic_config._slot_collision_nodes.add(pkg) slot_key = (pkg.slot_atom, pkg.root) @@ -1815,9 +2062,14 @@ class depgraph(object): # Yield ~, =*, < and <= atoms first, since those are more likely to # cause slot conflicts, and we want those atoms to be displayed # in the resulting slot conflict message (see bug #291142). + # Give similar treatment to SLOT/ABI atoms. conflict_atoms = [] normal_atoms = [] + abi_atoms = [] for atom in cp_atoms: + if atom.slot_abi_built: + abi_atoms.append(atom) + continue conflict = False for child_pkg in atom_pkg_graph.child_nodes(atom): existing_node, matches = \ @@ -1830,7 +2082,7 @@ class depgraph(object): else: normal_atoms.append(atom) - for atom in chain(conflict_atoms, normal_atoms): + for atom in chain(abi_atoms, conflict_atoms, normal_atoms): child_pkgs = atom_pkg_graph.child_nodes(atom) # if more than one child, yield highest version if len(child_pkgs) > 1: @@ -2256,6 +2508,8 @@ class depgraph(object): atom_list.append((root, '__auto_rebuild__', atom)) for root, atom in self._rebuild.reinstall_list: atom_list.append((root, '__auto_reinstall__', atom)) + for root, atom in self._dynamic_config._slot_abi_replace_installed: + atom_list.append((root, '__auto_slot_abi_replace_installed__', atom)) set_dict = {} for root, set_name, atom in atom_list: @@ -2421,6 +2675,12 @@ class depgraph(object): self._dynamic_config._need_restart = True return False, myfavorites + if "config" in self._dynamic_config._backtrack_infos and \ + ("slot_abi_mask_built" in self._dynamic_config._backtrack_infos["config"] or + "slot_abi_replace_installed" in self._dynamic_config._backtrack_infos["config"]) and \ + self.need_restart(): + return False, myfavorites + # We're true here unless we are missing binaries. return (True, myfavorites) @@ -2575,6 +2835,22 @@ class depgraph(object): """This will raise InvalidDependString if necessary. If trees is None then self._dynamic_config._filtered_trees is used.""" + if not isinstance(depstring, list): + eapi = None + is_valid_flag = None + if parent is not None: + eapi = parent.metadata['EAPI'] + if not parent.installed: + is_valid_flag = parent.iuse.is_valid_flag + depstring = portage.dep.use_reduce(depstring, + uselist=myuse, opconvert=True, token_class=Atom, + is_valid_flag=is_valid_flag, eapi=eapi) + + if (self._dynamic_config.myparams.get( + "ignore_built_slot_abi_deps", "n") == "y" and + parent and parent.built): + ignore_built_slot_abi_deps(depstring) + pkgsettings = self._frozen_config.pkgsettings[root] if trees is None: trees = self._dynamic_config._filtered_trees @@ -4298,8 +4574,14 @@ class depgraph(object): "recurse" not in self._dynamic_config.myparams: return 1 + complete_if_new_ver = self._dynamic_config.myparams.get( + "complete_if_new_ver", "y") == "y" + rebuild_if_new_slot_abi = self._dynamic_config.myparams.get( + "rebuild_if_new_slot_abi", "y") == "y" + complete_if_new_slot = rebuild_if_new_slot_abi + if "complete" not in self._dynamic_config.myparams and \ - self._dynamic_config.myparams.get("complete_if_new_ver", "y") == "y": + (complete_if_new_ver or complete_if_new_slot): # Enable complete mode if an installed package version will change. version_change = False for node in self._dynamic_config.digraph: @@ -4308,10 +4590,19 @@ class depgraph(object): continue vardb = self._frozen_config.roots[ node.root].trees["vartree"].dbapi - inst_pkg = vardb.match_pkgs(node.slot_atom) - if inst_pkg and inst_pkg[0].cp == node.cp: - inst_pkg = inst_pkg[0] - if inst_pkg < node or node < inst_pkg: + + if complete_if_new_ver: + inst_pkg = vardb.match_pkgs(node.slot_atom) + if inst_pkg and inst_pkg[0].cp == node.cp: + inst_pkg = inst_pkg[0] + if inst_pkg < node or node < inst_pkg: + version_change = True + break + + if complete_if_new_slot: + cp_list = vardb.match_pkgs(Atom(node.cp)) + if (cp_list and cp_list[0].cp == node.cp and + not any(node.slot == pkg.slot for pkg in cp_list)): version_change = True break @@ -4329,6 +4620,7 @@ class depgraph(object): # scheduled for replacement. Also, toggle the "deep" # parameter so that all dependencies are traversed and # accounted for. + self._complete_mode = True self._select_atoms = self._select_atoms_from_graph if "remove" in self._dynamic_config.myparams: self._select_package = self._select_pkg_from_installed @@ -4956,6 +5248,8 @@ class depgraph(object): self._process_slot_conflicts() + self._slot_abi_trigger_reinstalls() + if not self._validate_blockers(): self._dynamic_config._skip_restart = True raise self._unknown_internal_error() diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index c52a3eaab..efd954bb9 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -477,6 +477,7 @@ def insert_optional_args(args): '--package-moves' : y_or_n, '--quiet' : y_or_n, '--quiet-build' : y_or_n, + '--rebuild-if-new-slot-abi': y_or_n, '--rebuild-if-new-rev' : y_or_n, '--rebuild-if-new-ver' : y_or_n, '--rebuild-if-unbuilt' : y_or_n, @@ -746,6 +747,16 @@ def parse_opts(tmpcmdline, silent=False): "choices" : true_y_or_n }, + "--ignore-built-slot-abi-deps": { + "help": "Ignore the SLOT/ABI := operator parts of dependencies that have " + "been recorded when packages where built. This option is intended " + "only for debugging purposes, and it only affects built packages " + "that specify SLOT/ABI := operator dependencies using the " + "experimental \"4-slot-abi\" EAPI.", + "type": "choice", + "choices": y_or_n + }, + "--jobs": { "shortopt" : "-j", @@ -859,6 +870,15 @@ def parse_opts(tmpcmdline, silent=False): "choices" : true_y_or_n, }, + "--rebuild-if-new-slot-abi": { + "help" : ("Automatically rebuild or reinstall packages when SLOT/ABI := " + "operator dependencies can be satisfied by a newer slot, so that " + "older packages slots will become eligible for removal by the " + "--depclean action as soon as possible."), + "type" : "choice", + "choices" : true_y_or_n + }, + "--rebuild-if-new-rev": { "help" : "Rebuild packages when dependencies that are " + \ "used at both build-time and run-time are built, " + \ @@ -1100,6 +1120,9 @@ def parse_opts(tmpcmdline, silent=False): if myoptions.quiet_build in true_y: myoptions.quiet_build = 'y' + if myoptions.rebuild_if_new_slot_abi in true_y: + myoptions.rebuild_if_new_slot_abi = 'y' + if myoptions.rebuild_if_new_ver in true_y: myoptions.rebuild_if_new_ver = True else: diff --git a/pym/_emerge/resolver/backtracking.py b/pym/_emerge/resolver/backtracking.py index e3c5c7d78..09df9c822 100644 --- a/pym/_emerge/resolver/backtracking.py +++ b/pym/_emerge/resolver/backtracking.py @@ -7,7 +7,8 @@ class BacktrackParameter(object): __slots__ = ( "needed_unstable_keywords", "runtime_pkg_mask", "needed_use_config_changes", "needed_license_changes", - "rebuild_list", "reinstall_list", "needed_p_mask_changes" + "rebuild_list", "reinstall_list", "needed_p_mask_changes", + "slot_abi_replace_installed" ) def __init__(self): @@ -18,6 +19,7 @@ class BacktrackParameter(object): self.needed_license_changes = {} self.rebuild_list = set() self.reinstall_list = set() + self.slot_abi_replace_installed = set() def __deepcopy__(self, memo=None): if memo is None: @@ -34,6 +36,7 @@ class BacktrackParameter(object): result.needed_license_changes = copy.copy(self.needed_license_changes) result.rebuild_list = copy.copy(self.rebuild_list) result.reinstall_list = copy.copy(self.reinstall_list) + result.slot_abi_replace_installed = copy.copy(self.slot_abi_replace_installed) return result @@ -44,7 +47,8 @@ class BacktrackParameter(object): self.needed_use_config_changes == other.needed_use_config_changes and \ self.needed_license_changes == other.needed_license_changes and \ self.rebuild_list == other.rebuild_list and \ - self.reinstall_list == other.reinstall_list + self.reinstall_list == other.reinstall_list and \ + self.slot_abi_replace_installed == other.slot_abi_replace_installed class _BacktrackNode(object): @@ -114,9 +118,10 @@ class Backtracker(object): before, we revert the mask for other packages (bug 375573). """ - for pkg in runtime_pkg_mask: + for pkg, mask_info in runtime_pkg_mask.items(): - if "missing dependency" in runtime_pkg_mask[pkg]: + if "missing dependency" in mask_info or \ + "slot_abi_mask_built" in mask_info: continue entry_is_valid = False @@ -181,6 +186,10 @@ class Backtracker(object): elif change == "needed_use_config_changes": for pkg, (new_use, new_changes) in data: para.needed_use_config_changes[pkg] = (new_use, new_changes) + elif change == "slot_abi_mask_built": + para.runtime_pkg_mask.update(data) + elif change == "slot_abi_replace_installed": + para.slot_abi_replace_installed.update(data) elif change == "rebuild_list": para.rebuild_list.update(data) elif change == "reinstall_list": diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 84cac7411..46bdc961c 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -408,7 +408,7 @@ def abssymlink(symlink, target=None): _doebuild_manifest_exempt_depend = 0 -_testing_eapis = frozenset(["4-python"]) +_testing_eapis = frozenset(["4-python", "4-slot-abi"]) _deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1"]) def _eapi_is_deprecated(eapi): diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index 2ab7dc5b6..34b1d4705 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -8,7 +8,7 @@ import re import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.dbapi.dep_expand:dep_expand@_dep_expand', - 'portage.dep:match_from_list', + 'portage.dep:match_from_list,_match_slot', 'portage.output:colorize', 'portage.util:cmp_sort_key,writemsg', 'portage.versions:catsplit,catpkgsplit,vercmp,_pkg_str', @@ -173,7 +173,7 @@ class dbapi(object): except (KeyError, InvalidData): pass else: - if pkg_str.slot == atom.slot: + if _match_slot(atom, pkg_str): yield pkg_str def _iter_match_use(self, atom, cpv_iter): diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index df681152c..945c22c3d 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -10,7 +10,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.checksum', 'portage.data:portage_gid,secpass', 'portage.dbapi.dep_expand:dep_expand', - 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce', + 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot', 'portage.package.ebuild.doebuild:doebuild', 'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level', 'portage.util.listdir:listdir', @@ -836,7 +836,7 @@ class portdbapi(dbapi): continue if mydep.slot is not None and \ - mydep.slot != pkg_str.slot: + not _match_slot(mydep, pkg_str): continue if mydep.unevaluated_atom.use is not None and \ diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 5448bd431..b8405d483 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -3465,6 +3465,11 @@ class dblink(object): if not os.path.exists(self.dbcatdir): ensure_dirs(self.dbcatdir) + try: + slot = self.mycpv.slot + except AttributeError: + # discard the sub-slot if necesssary + slot = _pkg_str(self.mycpv, slot=slot).slot cp = self.mysplit[0] slot_atom = "%s:%s" % (cp, slot) diff --git a/pym/portage/dep/__init__.py b/pym/portage/dep/__init__.py index 23bafa892..557c92b42 100644 --- a/pym/portage/dep/__init__.py +++ b/pym/portage/dep/__init__.py @@ -47,7 +47,8 @@ _internal_warnings = False # It must not begin with a hyphen or a dot. _slot_separator = ":" _slot = r'([\w+][\w+.-]*)' -_slot_re = re.compile('^' + _slot + '$', re.VERBOSE) +# loosly match SLOT, which may have an optional ABI part +_slot_loose = r'([\w+./*=-]+)' _use = r'\[.*\]' _op = r'([=~]|[><]=?)' @@ -58,8 +59,49 @@ _repo = r'(?:' + _repo_separator + '(' + _repo_name + ')' + ')?' _extended_cat = r'[\w+*][\w+.*-]*' +_slot_re_cache = {} + def _get_slot_re(eapi_attrs): - return _slot_re + cache_key = eapi_attrs.slot_abi + slot_re = _slot_re_cache.get(cache_key) + if slot_re is not None: + return slot_re + + if eapi_attrs.slot_abi: + slot_re = _slot + r'(/' + _slot + r'=?)?' + else: + slot_re = _slot + + slot_re = re.compile('^' + slot_re + '$', re.VERBOSE) + + _slot_re_cache[cache_key] = slot_re + return slot_re + +_slot_dep_re_cache = {} + +def _get_slot_dep_re(eapi_attrs): + cache_key = eapi_attrs.slot_abi + slot_re = _slot_dep_re_cache.get(cache_key) + if slot_re is not None: + return slot_re + + if eapi_attrs.slot_abi: + slot_re = _slot + r'?(\*|=|/' + _slot + r'=?)?' + else: + slot_re = _slot + + slot_re = re.compile('^' + slot_re + '$', re.VERBOSE) + + _slot_dep_re_cache[cache_key] = slot_re + return slot_re + +def _match_slot(atom, pkg): + if pkg.slot == atom.slot: + if not atom.slot_abi: + return True + elif atom.slot_abi == pkg.slot_abi: + return True + return False _atom_re_cache = {} @@ -80,7 +122,7 @@ def _get_atom_re(eapi_attrs): '(?P' + _op + cpv_re + ')|' + '(?P=' + cpv_re + r'\*)|' + '(?P' + cp_re + '))' + - '(' + _slot_separator + _slot + ')?' + + '(' + _slot_separator + _slot_loose + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE) _atom_re_cache[cache_key] = atom_re @@ -101,7 +143,7 @@ def _get_atom_wildcard_re(eapi_attrs): atom_re = re.compile(r'(?P(' + _extended_cat + r')/(' + pkg_re + - r'))(:(?P' + _slot + r'))?(' + + r'))(:(?P' + _slot_loose + r'))?(' + _repo_separator + r'(?P' + _repo_name + r'))?$') _atom_wildcard_re_cache[cache_key] = atom_re @@ -1256,7 +1298,34 @@ class Atom(_unicode): self.__dict__['cpv'] = cpv self.__dict__['version'] = None self.__dict__['repo'] = repo - self.__dict__['slot'] = slot + if slot is None: + self.__dict__['slot'] = None + self.__dict__['slot_abi'] = None + self.__dict__['slot_abi_op'] = None + else: + slot_re = _get_slot_dep_re(eapi_attrs) + slot_match = slot_re.match(slot) + if slot_match is None: + raise InvalidAtom(self) + if eapi_attrs.slot_abi: + self.__dict__['slot'] = slot_match.group(1) + slot_abi = slot_match.group(2) + if slot_abi is not None: + slot_abi = slot_abi.lstrip("/") + if slot_abi in ("*", "="): + self.__dict__['slot_abi'] = None + self.__dict__['slot_abi_op'] = slot_abi + else: + slot_abi_op = None + if slot_abi is not None and slot_abi[-1:] == "=": + slot_abi_op = slot_abi[-1:] + slot_abi = slot_abi[:-1] + self.__dict__['slot_abi'] = slot_abi + self.__dict__['slot_abi_op'] = slot_abi_op + else: + self.__dict__['slot'] = slot + self.__dict__['slot_abi'] = None + self.__dict__['slot_abi_op'] = None self.__dict__['operator'] = op self.__dict__['extended_syntax'] = extended_syntax @@ -1329,6 +1398,15 @@ class Atom(_unicode): _("Strong blocks are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') + @property + def slot_abi_built(self): + """ + Returns True if slot_abi_op == "=" and slot_abi is not None. + NOTE: foo/bar:2= is unbuilt and returns False, whereas foo/bar:2/2= + is built and returns True. + """ + return self.slot_abi_op == "=" and self.slot_abi is not None + @property def without_repo(self): if self.repo is None: @@ -1338,9 +1416,14 @@ class Atom(_unicode): @property def without_slot(self): - if self.slot is None: + if self.slot is None and self.slot_abi_op is None: return self - return Atom(self.replace(_slot_separator + self.slot, '', 1), + atom = remove_slot(self) + if self.repo is not None: + atom += _repo_separator + self.repo + if self.use is not None: + atom += _unicode(self.use) + return Atom(atom, allow_repo=True, allow_wildcard=True) def with_repo(self, repo): @@ -2077,16 +2160,33 @@ def match_from_list(mydep, candidate_list): else: raise KeyError(_("Unknown operator: %s") % mydep) - if slot is not None and not mydep.extended_syntax: + if mydep.slot is not None and not mydep.extended_syntax: candidate_list = mylist mylist = [] for x in candidate_list: - xslot = getattr(x, "slot", False) - if xslot is False: + x_pkg = None + try: + x.cpv + except AttributeError: xslot = dep_getslot(x) - if xslot is not None and xslot != slot: - continue - mylist.append(x) + if xslot is not None: + try: + x_pkg = _pkg_str(remove_slot(x), slot=xslot) + except InvalidData: + continue + else: + x_pkg = x + + if x_pkg is None: + mylist.append(x) + else: + try: + x_pkg.slot + except AttributeError: + mylist.append(x) + else: + if _match_slot(mydep, x_pkg): + mylist.append(x) if mydep.unevaluated_atom.use: candidate_list = mylist diff --git a/pym/portage/dep/_slot_abi.py b/pym/portage/dep/_slot_abi.py new file mode 100644 index 000000000..3282cafc3 --- /dev/null +++ b/pym/portage/dep/_slot_abi.py @@ -0,0 +1,92 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.dep import Atom, paren_enclose, use_reduce +from portage.exception import InvalidData + +_dep_keys = ('DEPEND', 'PDEPEND', 'RDEPEND') +_runtime_keys = ('PDEPEND', 'RDEPEND') + +def find_built_slot_abi_atoms(pkg): + atoms = {} + for k in _dep_keys: + atom_list = list(_find_built_slot_abi_op(use_reduce(pkg.metadata[k], + uselist=pkg.use.enabled, eapi=pkg.metadata['EAPI'], + token_class=Atom))) + if atom_list: + atoms[k] = atom_list + return atoms + +def _find_built_slot_abi_op(dep_struct): + for x in dep_struct: + if isinstance(x, list): + for atom in _find_slot_abi_equal_op(x): + yield atom + elif isinstance(x, Atom) and x.slot_abi_built: + yield x + +def ignore_built_slot_abi_deps(dep_struct): + for i, x in enumerate(dep_struct): + if isinstance(x, list): + ignore_slot_abi_equal_deps(x) + elif isinstance(x, Atom) and x.slot_abi_built: + # There's no way of knowing here whether the SLOT + # part of the SLOT/ABI pair should be kept, so we + # ignore both parts. + dep_struct[i] = x.without_slot + +def evaluate_slot_abi_equal_deps(settings, use, trees): + + metadata = settings.configdict['pkg'] + eapi = metadata['EAPI'] + running_vardb = trees[trees._running_eroot]["vartree"].dbapi + target_vardb = trees[trees._target_eroot]["vartree"].dbapi + vardbs = [target_vardb] + deps = {} + for k in _dep_keys: + deps[k] = use_reduce(metadata[k], + uselist=use, eapi=eapi, token_class=Atom) + + for k in _runtime_keys: + _eval_deps(deps[k], vardbs) + + if running_vardb is not target_vardb: + vardbs.append(running_vardb) + + _eval_deps(deps["DEPEND"], vardbs) + + result = {} + for k, v in deps.items(): + result[k] = paren_enclose(v) + + return result + +def _eval_deps(dep_struct, vardbs): + for i, x in enumerate(dep_struct): + if isinstance(x, list): + _eval_deps(x, vardbs) + elif isinstance(x, Atom) and x.slot_abi_op == "=": + for vardb in vardbs: + best_version = vardb.match(x) + if best_version: + best_version = best_version[-1] + try: + best_version = \ + vardb._pkg_str(best_version, None) + except (KeyError, InvalidData): + pass + else: + slot_part = "%s/%s=" % \ + (best_version.slot, best_version.slot_abi) + x = x.with_slot(slot_part) + dep_struct[i] = x + break + else: + # this dep could not be resolved, so remove the operator + # (user may be using package.provided and managing rebuilds + # manually) + if x.slot: + x = x.with_slot(x.slot) + else: + x = x.without_slot + dep_struct[i] = x diff --git a/pym/portage/eapi.py b/pym/portage/eapi.py index e36567d16..8b03f830e 100644 --- a/pym/portage/eapi.py +++ b/pym/portage/eapi.py @@ -9,6 +9,9 @@ def eapi_has_iuse_defaults(eapi): def eapi_has_slot_deps(eapi): return eapi != "0" +def eapi_has_slot_abi(eapi): + return eapi in ("4-slot-abi",) + def eapi_has_src_uri_arrows(eapi): return eapi not in ("0", "1") @@ -65,7 +68,7 @@ def eapi_allows_dots_in_use_flags(eapi): _eapi_attrs = collections.namedtuple('_eapi_attrs', 'dots_in_PN dots_in_use_flags iuse_defaults ' - 'repo_deps required_use slot_deps ' + 'repo_deps required_use slot_abi slot_deps ' 'src_uri_arrows strong_blocks use_deps use_dep_defaults') _eapi_attrs_cache = {} @@ -86,6 +89,7 @@ def _get_eapi_attrs(eapi): repo_deps = (eapi is None or eapi_has_repo_deps(eapi)), required_use = (eapi is None or eapi_has_required_use(eapi)), slot_deps = (eapi is None or eapi_has_slot_deps(eapi)), + slot_abi = (eapi is None or eapi_has_slot_abi(eapi)), src_uri_arrows = (eapi is None or eapi_has_src_uri_arrows(eapi)), strong_blocks = (eapi is None or eapi_has_strong_blocks(eapi)), use_deps = (eapi is None or eapi_has_use_deps(eapi)), diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index edc5d0b6e..09062f9f3 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -25,6 +25,8 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.digestcheck:digestcheck', 'portage.package.ebuild.digestgen:digestgen', 'portage.package.ebuild.fetch:fetch', + 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', + 'portage.dep._slot_abi:evaluate_slot_abi_equal_deps', 'portage.package.ebuild._spawn_nofetch:spawn_nofetch', 'portage.util.ExtractKernelVersion:ExtractKernelVersion' ) @@ -43,7 +45,7 @@ from portage.dep import Atom, check_required_use, \ from portage.eapi import eapi_exports_KV, eapi_exports_merge_type, \ eapi_exports_replace_vars, eapi_exports_REPOSITORY, \ eapi_has_required_use, eapi_has_src_prepare_and_src_configure, \ - eapi_has_pkg_pretend + eapi_has_pkg_pretend, _get_eapi_attrs from portage.elog import elog_process, _preload_elog_modules from portage.elog.messages import eerror, eqawarn from portage.exception import DigestException, FileNotFound, \ @@ -1631,6 +1633,8 @@ def _post_src_install_write_metadata(settings): due to local environment settings like in bug #386829. """ + eapi_attrs = _get_eapi_attrs(settings.configdict['pkg']['EAPI']) + build_info_dir = os.path.join(settings['PORTAGE_BUILDDIR'], 'build-info') for k in ('IUSE',): @@ -1664,6 +1668,8 @@ def _post_src_install_write_metadata(settings): continue if k.endswith('DEPEND'): + if eapi_attrs.slot_abi: + continue token_class = Atom else: token_class = None @@ -1682,6 +1688,22 @@ def _post_src_install_write_metadata(settings): errors='strict') as f: f.write(_unicode_decode(v + '\n')) + if eapi_attrs.slot_abi: + deps = evaluate_slot_abi_equal_deps(settings, use, QueryCommand.get_db()) + for k, v in deps.items(): + filename = os.path.join(build_info_dir, k) + if not v: + try: + os.unlink(filename) + except OSError: + pass + continue + with io.open(_unicode_encode(os.path.join(build_info_dir, + k), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write(_unicode_decode(v + '\n')) + _vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND', 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',) diff --git a/pym/portage/tests/dep/testAtom.py b/pym/portage/tests/dep/testAtom.py index 092cacf84..e0cfaabcf 100644 --- a/pym/portage/tests/dep/testAtom.py +++ b/pym/portage/tests/dep/testAtom.py @@ -144,6 +144,25 @@ class TestAtom(TestCase): self.assertRaisesMsg(atom, (InvalidAtom, TypeError), Atom, atom, \ allow_wildcard=allow_wildcard, allow_repo=allow_repo) + def testSlotAbiAtom(self): + tests = ( + ("virtual/ffmpeg:0/53", "4-slot-abi", {"slot": "0", "slot_abi": "53", "slot_abi_op": None}), + ("virtual/ffmpeg:0/53=", "4-slot-abi", {"slot": "0", "slot_abi": "53", "slot_abi_op": "="}), + ("virtual/ffmpeg:=", "4-slot-abi", {"slot": None, "slot_abi": None, "slot_abi_op": "="}), + ("virtual/ffmpeg:0=", "4-slot-abi", {"slot": "0", "slot_abi": None, "slot_abi_op": "="}), + ("virtual/ffmpeg:*", "4-slot-abi", {"slot": None, "slot_abi": None, "slot_abi_op": "*"}), + ("virtual/ffmpeg:0*", "4-slot-abi", {"slot": "0", "slot_abi": None, "slot_abi_op": "*"}), + ("virtual/ffmpeg:0", "4-slot-abi", {"slot": "0", "slot_abi": None, "slot_abi_op": None}), + ("virtual/ffmpeg", "4-slot-abi", {"slot": None, "slot_abi": None, "slot_abi_op": None}), + ) + + for atom, eapi, parts in tests: + a = Atom(atom, eapi=eapi) + for k, v in parts.items(): + self.assertEqual(v, getattr(a, k), + msg="Atom('%s').%s = %s == '%s'" % + (atom, k, getattr(a, k), v )) + def test_intersects(self): test_cases = ( ("dev-libs/A", "dev-libs/A", True), diff --git a/pym/portage/tests/dep/test_isvalidatom.py b/pym/portage/tests/dep/test_isvalidatom.py index 173ab0dec..abcec755e 100644 --- a/pym/portage/tests/dep/test_isvalidatom.py +++ b/pym/portage/tests/dep/test_isvalidatom.py @@ -134,6 +134,15 @@ class IsValidAtom(TestCase): IsValidAtomTestCase("=sys-apps/portage-2.2*:foo::repo[bar?,!baz?,!doc=,build=]", False, allow_repo=False), IsValidAtomTestCase("=sys-apps/portage-2.2*:foo::repo[doc?]", False, allow_repo=False), IsValidAtomTestCase("null/portage::repo", False, allow_repo=False), + + IsValidAtomTestCase("virtual/ffmpeg:0/53", True), + IsValidAtomTestCase("virtual/ffmpeg:0/53=", True), + IsValidAtomTestCase("virtual/ffmpeg:0/53*", False), + IsValidAtomTestCase("virtual/ffmpeg:=", True), + IsValidAtomTestCase("virtual/ffmpeg:0=", True), + IsValidAtomTestCase("virtual/ffmpeg:*", True), + IsValidAtomTestCase("virtual/ffmpeg:0*", True), + IsValidAtomTestCase("virtual/ffmpeg:0", True), ) for test_case in test_cases: diff --git a/pym/portage/tests/dep/test_match_from_list.py b/pym/portage/tests/dep/test_match_from_list.py index fae473e20..d6649ad7f 100644 --- a/pym/portage/tests/dep/test_match_from_list.py +++ b/pym/portage/tests/dep/test_match_from_list.py @@ -16,9 +16,15 @@ class Package(object): def __init__(self, atom): atom = Atom(atom, allow_repo=True) self.cp = atom.cp - self.cpv = _pkg_str(atom.cpv, slot=(atom.slot or '0'), repo=atom.repo) + slot = atom.slot + if atom.slot_abi: + slot = "%s/%s" % (slot, atom.slot_abi) + if not slot: + slot = '0' + self.cpv = _pkg_str(atom.cpv, slot=slot, repo=atom.repo) self.cpv_split = catpkgsplit(self.cpv) - self.slot = atom.slot + self.slot = self.cpv.slot + self.slot_abi = self.cpv.slot_abi self.repo = atom.repo if atom.use: self.use = self._use_class(atom.use.enabled) @@ -93,6 +99,15 @@ class Test_match_from_list(TestCase): ("dev-libs/A::repo2[foo]", [Package("=dev-libs/A-1::repo1[-foo]"), Package("=dev-libs/A-1::repo2[foo]")], ["dev-libs/A-1::repo2"] ), ("dev-libs/A:1::repo2[foo]", [Package("=dev-libs/A-1:1::repo1"), Package("=dev-libs/A-1:2::repo2")], [] ), ("dev-libs/A:1::repo2[foo]", [Package("=dev-libs/A-1:2::repo1"), Package("=dev-libs/A-1:1::repo2[foo]")], ["dev-libs/A-1::repo2"] ), + + ("virtual/ffmpeg:0/53", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:0/53=", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:0/52", [Package("=virtual/ffmpeg-0.10.3:0/53")], [] ), + ("virtual/ffmpeg:=", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:0=", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:*", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:0*", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:0", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), ) for atom, cpv_list, expected_result in tests: diff --git a/pym/portage/tests/emerge/test_emerge_slot_abi.py b/pym/portage/tests/emerge/test_emerge_slot_abi.py new file mode 100644 index 000000000..005c5d354 --- /dev/null +++ b/pym/portage/tests/emerge/test_emerge_slot_abi.py @@ -0,0 +1,198 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import subprocess +import sys + +import portage +from portage import os +from portage import _unicode_decode +from portage.const import (BASH_BINARY, PORTAGE_BIN_PATH, + PORTAGE_PYM_PATH, USER_CONFIG_PATH) +from portage.process import find_binary +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground +from portage.util import ensure_dirs + +class SlotAbiEmergeTestCase(TestCase): + + def testSlotAbiEmerge(self): + + debug = False + + ebuilds = { + "dev-libs/glib-1.2.10" : { + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/glib-2.32.3" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.32" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2=", + "RDEPEND": "dev-libs/glib:2=" + }, + } + installed = { + "dev-libs/glib-1.2.10" : { + "EAPI": "4-slot-abi", + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2/2.30=", + "RDEPEND": "dev-libs/glib:2/2.30=" + }, + } + + world = ["dev-libs/glib:1", "dev-libs/dbus-glib"] + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=debug) + settings = playground.settings + eprefix = settings["EPREFIX"] + eroot = settings["EROOT"] + trees = playground.trees + portdb = trees[eroot]["porttree"].dbapi + vardb = trees[eroot]["vartree"].dbapi + portdir = settings["PORTDIR"] + var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") + + portage_python = portage._python_interpreter + ebuild_cmd = (portage_python, "-Wd", + os.path.join(PORTAGE_BIN_PATH, "ebuild")) + emerge_cmd = (portage_python, "-Wd", + os.path.join(PORTAGE_BIN_PATH, "emerge")) + + test_ebuild = portdb.findname("dev-libs/dbus-glib-0.98") + self.assertFalse(test_ebuild is None) + + test_commands = ( + emerge_cmd + ("--oneshot", "dev-libs/glib",), + (lambda: "dev-libs/glib:2/2.32=" in vardb.aux_get("dev-libs/dbus-glib-0.98", ["RDEPEND"])[0],), + emerge_cmd + ("--oneshot", "=dev-libs/glib-2.30.2", "--ignore-built-slot-abi-deps", "y"), + emerge_cmd + ("--oneshot", "dev-libs/dbus-glib"), + (lambda: "dev-libs/glib:2/2.30=" in vardb.aux_get("dev-libs/dbus-glib-0.98", ["RDEPEND"])[0],), + ) + + distdir = playground.distdir + pkgdir = playground.pkgdir + fake_bin = os.path.join(eprefix, "bin") + portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") + profile_path = settings.profile_path + user_config_dir = os.path.join(os.sep, eprefix, USER_CONFIG_PATH) + + features = [] + if not portage.process.sandbox_capable or \ + os.environ.get("SANDBOX_ON") == "1": + features.append("-sandbox") + + make_conf = ( + "FEATURES=\"%s\"\n" % (" ".join(features),), + "PORTDIR=\"%s\"\n" % (portdir,), + "PORTAGE_GRPNAME=\"%s\"\n" % (os.environ["PORTAGE_GRPNAME"],), + "PORTAGE_USERNAME=\"%s\"\n" % (os.environ["PORTAGE_USERNAME"],), + "PKGDIR=\"%s\"\n" % (pkgdir,), + "PORTAGE_INST_GID=%s\n" % (portage.data.portage_gid,), + "PORTAGE_INST_UID=%s\n" % (portage.data.portage_uid,), + "PORTAGE_TMPDIR=\"%s\"\n" % (portage_tmpdir,), + "CLEAN_DELAY=0\n", + "DISTDIR=\"%s\"\n" % (distdir,), + "EMERGE_WARNING_DELAY=0\n", + ) + + path = os.environ.get("PATH") + if path is not None and not path.strip(): + path = None + if path is None: + path = "" + else: + path = ":" + path + path = fake_bin + path + + pythonpath = os.environ.get("PYTHONPATH") + if pythonpath is not None and not pythonpath.strip(): + pythonpath = None + if pythonpath is not None and \ + pythonpath.split(":")[0] == PORTAGE_PYM_PATH: + pass + else: + if pythonpath is None: + pythonpath = "" + else: + pythonpath = ":" + pythonpath + pythonpath = PORTAGE_PYM_PATH + pythonpath + + env = { + "PORTAGE_OVERRIDE_EPREFIX" : eprefix, + "PATH" : path, + "PORTAGE_PYTHON" : portage_python, + "PYTHONPATH" : pythonpath, + } + + if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: + env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ + os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] + + dirs = [distdir, fake_bin, portage_tmpdir, + user_config_dir, var_cache_edb] + true_symlinks = ["chown", "chgrp"] + true_binary = find_binary("true") + self.assertEqual(true_binary is None, False, + "true command not found") + try: + for d in dirs: + ensure_dirs(d) + with open(os.path.join(user_config_dir, "make.conf"), 'w') as f: + for line in make_conf: + f.write(line) + for x in true_symlinks: + os.symlink(true_binary, os.path.join(fake_bin, x)) + with open(os.path.join(var_cache_edb, "counter"), 'wb') as f: + f.write(b"100") + # non-empty system set keeps --depclean quiet + with open(os.path.join(profile_path, "packages"), 'w') as f: + f.write("*dev-libs/token-system-pkg") + + if debug: + # The subprocess inherits both stdout and stderr, for + # debugging purposes. + stdout = None + else: + # The subprocess inherits stderr so that any warnings + # triggered by python -Wd will be visible. + stdout = subprocess.PIPE + + for i, args in enumerate(test_commands): + + if hasattr(args[0], '__call__'): + self.assertTrue(args[0](), + "callable at index %s failed" % (i,)) + continue + + proc = subprocess.Popen(args, + env=env, stdout=stdout) + + if debug: + proc.wait() + else: + output = proc.stdout.readlines() + proc.wait() + proc.stdout.close() + if proc.returncode != os.EX_OK: + for line in output: + sys.stderr.write(_unicode_decode(line)) + + self.assertEqual(os.EX_OK, proc.returncode, + "emerge failed with args %s" % (args,)) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_complete_graph.py b/pym/portage/tests/resolver/test_complete_graph.py index 07206430e..850b883b4 100644 --- a/pym/portage/tests/resolver/test_complete_graph.py +++ b/pym/portage/tests/resolver/test_complete_graph.py @@ -29,7 +29,7 @@ class CompleteGraphTestCase(TestCase): test_cases = ( ResolverPlaygroundTestCase( [">=sys-libs/x-2"], - options = {"--complete-graph-if-new-ver" : "n"}, + options = {"--complete-graph-if-new-ver" : "n", "--rebuild-if-new-slot-abi": "n"}, mergelist = ["sys-libs/x-2"], success = True, ), @@ -42,7 +42,7 @@ class CompleteGraphTestCase(TestCase): ), ResolverPlaygroundTestCase( ["=sys-libs/db-4:=", + "RDEPEND": ">=sys-libs/db-4:=" + }, + } + binpkgs = { + "sys-libs/db-4.8" : { + "SLOT": "4.8" + }, + "sys-libs/db-4.7" : { + "SLOT": "4.7" + }, + "app-office/libreoffice-3.5.4.2" : { + "EAPI": "4-slot-abi", + "DEPEND": ">=sys-libs/db-4:4.7/4.7=", + "RDEPEND": ">=sys-libs/db-4:4.7/4.7==" + }, + } + installed = { + "sys-libs/db-4.7" : { + "SLOT": "4.7" + }, + "app-office/libreoffice-3.5.4.2" : { + "EAPI": "4-slot-abi", + "DEPEND": ">=sys-libs/db-4:4.7/4.7=", + "RDEPEND": ">=sys-libs/db-4:4.7/4.7=" + }, + } + + world = ["app-office/libreoffice"] + + test_cases = ( + + ResolverPlaygroundTestCase( + ["sys-libs/db"], + options = {"--oneshot": True}, + success = True, + mergelist = ["sys-libs/db-4.8", "app-office/libreoffice-3.5.4.2"]), + + ResolverPlaygroundTestCase( + ["sys-libs/db"], + options = {"--oneshot": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["sys-libs/db-4.8"]), + + ResolverPlaygroundTestCase( + ["sys-libs/db"], + options = {"--oneshot": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]sys-libs/db-4.8", "app-office/libreoffice-3.5.4.2"]), + + ResolverPlaygroundTestCase( + ["sys-libs/db"], + options = {"--oneshot": True, "--usepkgonly": True}, + success = True, + mergelist = ["[binary]sys-libs/db-4.8"]), + + ResolverPlaygroundTestCase( + ["sys-libs/db"], + options = {"--oneshot": True, "--rebuild-if-new-slot-abi": "n"}, + success = True, + mergelist = ["sys-libs/db-4.8"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True}, + success = True, + mergelist = ["sys-libs/db-4.8", "app-office/libreoffice-3.5.4.2"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]sys-libs/db-4.8", "app-office/libreoffice-3.5.4.2"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkg": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["[binary]sys-libs/db-4.8"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True}, + success = True, + mergelist = []), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["[binary]sys-libs/db-4.8"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--rebuild-if-new-slot-abi": "n"}, + success = True, + mergelist = []), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, binpkgs=binpkgs, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() + + def testWholeSlotSubSlotMix(self): + ebuilds = { + "dev-libs/glib-1.2.10" : { + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/glib-2.32.3" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.32" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2=", + "RDEPEND": "dev-libs/glib:2=" + }, + } + binpkgs = { + "dev-libs/glib-1.2.10" : { + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/glib-2.32.3" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.32" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2/2.30=", + "RDEPEND": "dev-libs/glib:2/2.30=" + }, + } + installed = { + "dev-libs/glib-1.2.10" : { + "EAPI": "4-slot-abi", + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2/2.30=", + "RDEPEND": "dev-libs/glib:2/2.30=" + }, + } + + world = ["dev-libs/glib:1", "dev-libs/dbus-glib"] + + test_cases = ( + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True}, + success = True, + mergelist = ["dev-libs/glib-2.32.3", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["dev-libs/glib-2.32.3"]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.32.3", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--usepkgonly": True}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.30.2"]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--usepkgonly": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.32.3"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True}, + success = True, + mergelist = ["dev-libs/glib-2.32.3", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["dev-libs/glib-2.32.3"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.32.3", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True}, + success = True, + mergelist = []), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.32.3"]), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, binpkgs=binpkgs, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/versions.py b/pym/portage/versions.py index a1ded6771..5893096d1 100644 --- a/pym/portage/versions.py +++ b/pym/portage/versions.py @@ -356,13 +356,24 @@ class _pkg_str(_unicode): # for match_from_list introspection self.__dict__['cpv'] = self if slot is not None: - slot_match = _get_slot_re(_get_eapi_attrs(eapi)).match(slot) + eapi_attrs = _get_eapi_attrs(eapi) + slot_match = _get_slot_re(eapi_attrs).match(slot) if slot_match is None: # Avoid an InvalidAtom exception when creating SLOT atoms self.__dict__['slot'] = '0' + self.__dict__['slot_abi'] = '0' self.__dict__['slot_invalid'] = slot else: - self.__dict__['slot'] = slot + if eapi_attrs.slot_abi: + slot_split = slot.split("/") + self.__dict__['slot'] = slot_split[0] + if len(slot_split) > 1: + self.__dict__['slot_abi'] = slot_split[1] + else: + self.__dict__['slot_abi'] = slot_split[0] + else: + self.__dict__['slot'] = slot + self.__dict__['slot_abi'] = slot if repo is not None: repo = _gen_valid_repo(repo) -- cgit v1.2.3-1-g7c22