Instead of doing automatic uninstalls in advance, install conflicting

packages first and then do the uninstall afterwards. This requires
special handling for file collisions occur, but it's preferred
because it ensures that package files remain installed in a usable
state whenever possible.

When file collisions occur between conflicting packages, the contents
entries for those files are removed from the packages that are
scheduled for uninstallation. This prevents uninstallation operations
from removing overlapping files that have been claimed by conflicting
packages.

svn path=/main/trunk/; revision=10225

4 files changed, 233 insertions, 44 deletions

diff --git a/doc/dependency_resolution/task_scheduling.docbook b/doc/dependency_resolution/task_scheduling.docbook
index f879980cd..01953128e 100644
--- a/doc/dependency_resolution/task_scheduling.docbook
+++ b/doc/dependency_resolution/task_scheduling.docbook
@@ -21,7 +21,7 @@
 	</para>
 	<para>
 	In order to avoid a conflict, a package may need to be uninstalled
-	in advance, rather than through replacement. The following constraints
+	rather than replaced. The following constraints
 	protect inappropriate packages from being chosen for automatic
 	uninstallation:
 	<itemizedlist>
@@ -46,6 +46,16 @@
 	</listitem>
 	</itemizedlist>
 	</para>
+	<para>
+	In order to ensure that package files remain installed in a usable state
+	whenever possible, uninstallation operations are not executed
+	until after all associated conflicting packages have been installed.
+	When file collisions occur between conflicting packages, the contents
+	entries for those files are removed from the packages
+	that are scheduled for uninstallation. This prevents
+	uninstallation operations from removing overlapping files that
+	have been claimed by conflicting packages.
+	</para>
 </sect1>
 <sect1 id='dependency-resolution-task-scheduling-circular-dependencies'>
 	<title>Circular Dependencies</title>
diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index cc236633b..a1933d6c1 100644
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -729,7 +729,6 @@ class search(object):
 			result = ""
 		return result
 
-
 class RootConfig(object):
 	"""This is used internally by depgraph to track information about a
 	particular $ROOT."""
@@ -1273,13 +1272,22 @@ class Package(Task):
 	__slots__ = ("built", "cpv", "depth",
 		"installed", "metadata", "onlydeps", "operation",
 		"root", "type_name",
-		"cp", "cpv_slot", "pv_split", "slot_atom")
+		"category", "cp", "cpv_slot", "pf", "pv_split", "slot_atom")
+
+	metadata_keys = [
+		"CHOST", "COUNTER", "DEPEND", "EAPI", "IUSE", "KEYWORDS",
+		"LICENSE", "PDEPEND", "PROVIDE", "RDEPEND",
+		"repository", "RESTRICT", "SLOT", "USE"]
+
 	def __init__(self, **kwargs):
 		Task.__init__(self, **kwargs)
 		self.cp = portage.cpv_getkey(self.cpv)
 		self.slot_atom = "%s:%s" % (self.cp, self.metadata["SLOT"])
 		self.cpv_slot = "%s:%s" % (self.cpv, self.metadata["SLOT"])
-		self.pv_split = portage.catpkgsplit(self.cpv)[1:]
+		cpv_parts = portage.catpkgsplit(self.cpv)
+		self.category = cpv_parts[0]
+		self.pv_split = cpv_parts[1:]
+		self.pf = self.cpv.replace(self.category + "/", "", 1)
 
 	def _get_hash_key(self):
 		hash_key = getattr(self, "_hash_key", None)
@@ -1366,6 +1374,9 @@ class BlockerCache(DictMixin):
 		2) the old-style virtuals have changed
 	"""
 	class BlockerData(object):
+
+		__slots__ = ("__weakref__", "atoms", "counter")
+
 		def __init__(self, counter, atoms):
 			self.counter = counter
 			self.atoms = atoms
@@ -1507,6 +1518,84 @@ class BlockerCache(DictMixin):
 		an AttributeError."""
 		return list(self)
 
+class BlockerDB(object):
+
+	def __init__(self, vartree, portdb):
+		self._vartree = vartree
+		self._portdb = portdb
+		self._blocker_cache = \
+			BlockerCache(self._vartree.root, vartree.dbapi)
+		self._dep_check_trees = { self._vartree.root : {
+			"porttree"    :  self._vartree,
+			"vartree"     :  self._vartree,
+		}}
+		self._installed_pkgs = None
+
+	def findInstalledBlockers(self, new_pkg):
+		self._update_cache()
+		blocker_parents = digraph()
+		blocker_atoms = []
+		for pkg in self._installed_pkgs:
+			for blocker_atom in self._blocker_cache[pkg.cpv].atoms:
+				blocker_atom = blocker_atom[1:]
+				blocker_atoms.append(blocker_atom)
+				blocker_parents.add(blocker_atom, pkg)
+
+		blocker_atoms = InternalPackageSet(initial_atoms=blocker_atoms)
+		blocking_pkgs = set()
+		for atom in blocker_atoms.iterAtomsForPackage(new_pkg):
+			blocking_pkgs.update(blocker_parents.parent_nodes(atom))
+		return blocking_pkgs
+
+	def _update_cache(self):
+		blocker_cache = self._blocker_cache
+		dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"]
+		dep_check_trees = self._dep_check_trees
+		settings = self._vartree.settings
+		stale_cache = set(blocker_cache)
+		fake_vartree = \
+			FakeVartree(self._vartree,
+				self._portdb, Package.metadata_keys, {})
+		vardb = fake_vartree.dbapi
+		self._installed_pkgs = list(vardb)
+
+		for inst_pkg in self._installed_pkgs:
+			stale_cache.discard(inst_pkg.cpv)
+			cached_blockers = blocker_cache.get(inst_pkg.cpv)
+			if cached_blockers is not None and \
+				cached_blockers.counter != long(inst_pkg.metadata["COUNTER"]):
+				cached_blockers = None
+			if cached_blockers is not None:
+				blocker_atoms = cached_blockers.atoms
+			else:
+				myuse = inst_pkg.metadata["USE"].split()
+				# Use aux_get() to trigger FakeVartree global
+				# updates on *DEPEND when appropriate.
+				depstr = " ".join(vardb.aux_get(inst_pkg.cpv, dep_keys))
+				try:
+					portage.dep._dep_check_strict = False
+					success, atoms = portage.dep_check(depstr,
+						vardb, settings, myuse=myuse,
+						trees=dep_check_trees, myroot=inst_pkg.root)
+				finally:
+					portage.dep._dep_check_strict = True
+				if not success:
+					pkg_location = os.path.join(inst_pkg.root,
+						portage.VDB_PATH, inst_pkg.category, inst_pkg.pf)
+					portage.writemsg("!!! %s/*DEPEND: %s\n" % \
+						(pkg_location, atoms), noiselevel=-1)
+					continue
+
+				blocker_atoms = [atom for atom in atoms \
+					if atom.startswith("!")]
+				blocker_atoms.sort()
+				counter = long(inst_pkg.metadata["COUNTER"])
+				blocker_cache[inst_pkg.cpv] = \
+					blocker_cache.BlockerData(counter, blocker_atoms)
+		for cpv in stale_cache:
+			del blocker_cache[cpv]
+		blocker_cache.flush()
+
 def show_invalid_depstring_notice(parent_node, depstring, error_msg):
 
 	from formatter import AbstractFormatter, DumbWriter
@@ -1660,10 +1749,7 @@ class depgraph(object):
 		"binary":"bintree",
 		"installed":"vartree"}
 
-	_mydbapi_keys = [
-		"CHOST", "COUNTER", "DEPEND", "EAPI", "IUSE", "KEYWORDS",
-		"LICENSE", "PDEPEND", "PROVIDE", "RDEPEND",
-		"repository", "RESTRICT", "SLOT", "USE"]
+	_mydbapi_keys = Package.metadata_keys
 
 	_dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"]
 
@@ -3484,6 +3570,9 @@ class depgraph(object):
 			return -1
 		myblocker_uninstalls = self._blocker_uninstalls.copy()
 		retlist=[]
+		# Contains uninstall tasks that have been scheduled to
+		# occur after overlapping blockers have been installed.
+		scheduled_uninstalls = set()
 		# Contains any Uninstall tasks that have been ignored
 		# in order to avoid the circular deps code path. These
 		# correspond to blocker conflicts that could not be
@@ -3698,10 +3787,16 @@ class depgraph(object):
 					selected_nodes = list(selected_nodes)
 				selected_nodes.sort(cmp_circular_bias)
 
+			if not selected_nodes and scheduled_uninstalls:
+				selected_nodes = set()
+				for node in scheduled_uninstalls:
+					if not mygraph.child_nodes(node):
+						selected_nodes.add(node)
+				scheduled_uninstalls.difference_update(selected_nodes)
+
 			if not selected_nodes and not myblocker_uninstalls.is_empty():
 				# An Uninstall task needs to be executed in order to
 				# avoid conflict if possible.
-
 				min_parent_deps = None
 				uninst_task = None
 				for task in myblocker_uninstalls.leaf_nodes():
@@ -3819,7 +3914,20 @@ class depgraph(object):
 						uninst_task = task
 
 				if uninst_task is not None:
-					selected_nodes = [uninst_task]
+					# The uninstall is performed only after blocking
+					# packages have been merged on top of it. File
+					# collisions between blocking packages are detected
+					# and removed from the list of files to be uninstalled.
+					scheduled_uninstalls.add(uninst_task)
+					parent_nodes = mygraph.parent_nodes(uninst_task)
+
+					# Reverse the parent -> uninstall edges since we want
+					# to do the uninstall after blocking packages have
+					# been merged on top of it.
+					mygraph.remove(uninst_task)
+					for blocked_pkg in parent_nodes:
+						mygraph.add(blocked_pkg, uninst_task,
+							priority=BlockerDepPriority.instance)
 				else:
 					# None of the Uninstall tasks are acceptable, so
 					# the corresponding blockers are unresolvable.
@@ -3836,12 +3944,12 @@ class depgraph(object):
 							ignored_uninstall_tasks.add(node)
 							break
 
-					# After dropping an Uninstall task, reset
-					# the state variables for leaf node selection and
-					# continue trying to select leaf nodes.
-					prefer_asap = True
-					accept_root_node = False
-					continue
+				# After dropping an Uninstall task, reset
+				# the state variables for leaf node selection and
+				# continue trying to select leaf nodes.
+				prefer_asap = True
+				accept_root_node = False
+				continue
 
 			if not selected_nodes:
 				self._circular_deps_for_display = mygraph
@@ -4002,6 +4110,8 @@ class depgraph(object):
 			verbosity = ("--quiet" in self.myopts and 1 or \
 				"--verbose" in self.myopts and 3 or 2)
 		favorites_set = InternalPackageSet(favorites)
+		oneshot = "--oneshot" in self.myopts or \
+			"--onlydeps" in self.myopts
 		changelogs=[]
 		p=[]
 		blockers = []
@@ -4558,7 +4668,8 @@ class depgraph(object):
 				try:
 					pkg_system = system_set.findAtomForPackage(pkg_key, metadata)
 					pkg_world  = world_set.findAtomForPackage(pkg_key, metadata)
-					if not pkg_world and myroot == self.target_root and \
+					if not (oneshot or pkg_world) and \
+						myroot == self.target_root and \
 						favorites_set.findAtomForPackage(pkg_key, metadata):
 						# Maybe it will be added to world now.
 						if create_world_atom(pkg_key, metadata,
@@ -5368,12 +5479,35 @@ class MergeTask(object):
 		if settings.get("PORTAGE_DEBUG", "") == "1":
 			self.edebug = 1
 		self.pkgsettings = {}
+		self._blocker_db = {}
 		for root in trees:
 			self.pkgsettings[root] = portage.config(
 				clone=trees[root]["vartree"].settings)
+			self._blocker_db[root] = BlockerDB(
+				trees[root]["vartree"],
+				trees[root]["porttree"].dbapi)
 		self.curval = 0
 		self._spawned_pids = []
-		self._uninstall_queue = []
+
+	def _find_blockers(self, new_pkg):
+		for opt in ("--buildpkgonly", "--nodeps",
+			"--fetchonly", "--fetch-all-uri", "--pretend"):
+			if opt in self.myopts:
+				return None
+
+		blocker_dblinks = []
+		for blocking_pkg in self._blocker_db[
+			new_pkg.root].findInstalledBlockers(new_pkg):
+			if new_pkg.slot_atom == blocking_pkg.slot_atom:
+				continue
+			if new_pkg.cpv == blocking_pkg.cpv:
+				continue
+			blocker_dblinks.append(portage.dblink(
+				blocking_pkg.category, blocking_pkg.pf, blocking_pkg.root,
+				self.pkgsettings[blocking_pkg.root], treetype="vartree",
+				vartree=self.trees[blocking_pkg.root]["vartree"]))
+
+		return blocker_dblinks
 
 	def merge(self, mylist, favorites, mtimedb):
 		try:
@@ -5403,17 +5537,6 @@ class MergeTask(object):
 				pass
 			spawned_pids.remove(pid)
 
-	def _dequeue_uninstall_tasks(self, mtimedb):
-		if not self._uninstall_queue:
-			return
-		for uninst_task in self._uninstall_queue:
-			root_config = self.trees[uninst_task.root]["root_config"]
-			unmerge(root_config, self.myopts, "unmerge",
-				[uninst_task.cpv], mtimedb["ldpath"], clean_world=0)
-			del mtimedb["resume"]["mergelist"][0]
-			mtimedb.commit()
-		del self._uninstall_queue[:]
-
 	def _merge(self, mylist, favorites, mtimedb):
 		from portage.elog import elog_process
 		from portage.elog.filtering import filter_mergephases
@@ -5554,7 +5677,10 @@ class MergeTask(object):
 			metadata = pkg.metadata
 			if pkg.installed:
 				if not (buildpkgonly or fetchonly or pretend):
-					self._uninstall_queue.append(pkg)
+					unmerge(root_config, self.myopts, "unmerge",
+						[pkg.cpv], mtimedb["ldpath"], clean_world=0)
+					del mtimedb["resume"]["mergelist"][0]
+					mtimedb.commit()
 				continue
 
 			if x[0]=="blocks":
@@ -5655,20 +5781,22 @@ class MergeTask(object):
 							return retval
 						bintree = self.trees[myroot]["bintree"]
 						bintree.inject(pkg_key, filename=binpkg_tmpfile)
-						self._dequeue_uninstall_tasks(mtimedb)
+
 						if "--buildpkgonly" not in self.myopts:
 							msg = " === (%s of %s) Merging (%s::%s)" % \
 								(mergecount, len(mymergelist), pkg_key, y)
 							short_msg = "emerge: (%s of %s) %s Merge" % \
 								(mergecount, len(mymergelist), pkg_key)
 							emergelog(xterm_titles, msg, short_msg=short_msg)
+
 							retval = portage.merge(pkgsettings["CATEGORY"],
 								pkgsettings["PF"], pkgsettings["D"],
 								os.path.join(pkgsettings["PORTAGE_BUILDDIR"],
 								"build-info"), myroot, pkgsettings,
 								myebuild=pkgsettings["EBUILD"],
 								mytree="porttree", mydbapi=portdb,
-								vartree=vartree, prev_mtimes=ldpath_mtimes)
+								vartree=vartree, prev_mtimes=ldpath_mtimes,
+								blockers=self._find_blockers(pkg))
 							if retval != os.EX_OK:
 								return retval
 						elif "noclean" not in pkgsettings.features:
@@ -5687,14 +5815,15 @@ class MergeTask(object):
 							prev_mtimes=ldpath_mtimes)
 						if retval != os.EX_OK:
 							return retval
-						self._dequeue_uninstall_tasks(mtimedb)
+
 						retval = portage.merge(pkgsettings["CATEGORY"],
 							pkgsettings["PF"], pkgsettings["D"],
 							os.path.join(pkgsettings["PORTAGE_BUILDDIR"],
 							"build-info"), myroot, pkgsettings,
 							myebuild=pkgsettings["EBUILD"],
 							mytree="porttree", mydbapi=portdb,
-							vartree=vartree, prev_mtimes=ldpath_mtimes)
+							vartree=vartree, prev_mtimes=ldpath_mtimes,
+							blockers=self._find_blockers(pkg))
 						if retval != os.EX_OK:
 							return retval
 				finally:
@@ -5716,7 +5845,6 @@ class MergeTask(object):
 							portage.locks.unlockdir(catdir_lock)
 
 			elif x[0]=="binary":
-				self._dequeue_uninstall_tasks(mtimedb)
 				#merge the tbz2
 				mytbz2 = self.trees[myroot]["bintree"].getname(pkg_key)
 				if "--getbinpkg" in self.myopts:
@@ -5772,7 +5900,8 @@ class MergeTask(object):
 				retval = portage.pkgmerge(mytbz2, x[1], pkgsettings,
 					mydbapi=bindb,
 					vartree=self.trees[myroot]["vartree"],
-					prev_mtimes=ldpath_mtimes)
+					prev_mtimes=ldpath_mtimes,
+					blockers=self._find_blockers(pkg))
 				if retval != os.EX_OK:
 					return retval
 				#need to check for errors
@@ -7888,6 +8017,7 @@ def action_build(settings, trees, mtimedb,
 	fetchonly = "--fetchonly" in myopts or "--fetch-all-uri" in myopts
 	ask = "--ask" in myopts
 	nodeps = "--nodeps" in myopts
+	oneshot = "--oneshot" in myopts or "--onlydeps" in myopts
 	tree = "--tree" in myopts
 	if nodeps and tree:
 		tree = False
@@ -8059,7 +8189,7 @@ def action_build(settings, trees, mtimedb,
 					mergecount += 1
 
 			if mergecount==0:
-				if "--noreplace" in myopts and favorites:
+				if "--noreplace" in myopts and not oneshot and favorites:
 					print
 					for x in favorites:
 						print " %s %s" % (good("*"), x)
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index c51497eef..139fa57e0 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -5313,13 +5313,13 @@ def movefile(src,dest,newmtime=None,sstat=None,mysettings=None):
 	return newmtime
 
 def merge(mycat, mypkg, pkgloc, infloc, myroot, mysettings, myebuild=None,
-	mytree=None, mydbapi=None, vartree=None, prev_mtimes=None):
+	mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None):
 	if not os.access(myroot, os.W_OK):
 		writemsg("Permission denied: access('%s', W_OK)\n" % myroot,
 			noiselevel=-1)
 		return errno.EACCES
 	mylink = dblink(mycat, mypkg, myroot, mysettings, treetype=mytree,
-		vartree=vartree)
+		vartree=vartree, blockers=blockers)
 	return mylink.merge(pkgloc, infloc, myroot, myebuild,
 		mydbapi=mydbapi, prev_mtimes=prev_mtimes)
 
@@ -6184,7 +6184,8 @@ class FetchlistDict(UserDict.DictMixin):
 		"""Returns keys for all packages within pkgdir"""
 		return self.portdb.cp_list(self.cp, mytree=self.mytree)
 
-def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes=None):
+def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None,
+	vartree=None, prev_mtimes=None, blockers=None):
 	"""will merge a .tbz2 file, returning a list of runtime dependencies
 		that must be satisfied, or None if there was a merge error.	This
 		code assumes the package exists."""
@@ -6277,7 +6278,7 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes
 		#tbz2_lock = None
 
 		mylink = dblink(mycat, mypkg, myroot, mysettings, vartree=vartree,
-			treetype="bintree")
+			treetype="bintree", blockers=blockers)
 		retval = mylink.merge(pkgloc, infloc, myroot, myebuild, cleanup=0,
 			mydbapi=mydbapi, prev_mtimes=prev_mtimes)
 		did_merge_phase = True
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 145016edc..f433789cd 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -932,7 +932,7 @@ class dblink(object):
 	}
 
 	def __init__(self, cat, pkg, myroot, mysettings, treetype=None,
-		vartree=None):
+		vartree=None, blockers=None):
 		"""
 		Creates a DBlink object for a given CPV.
 		The given CPV may not be present in the database already.
@@ -961,6 +961,7 @@ class dblink(object):
 			from portage import db
 			vartree = db[myroot]["vartree"]
 		self.vartree = vartree
+		self._blockers = blockers
 
 		self.dbroot = normalize_path(os.path.join(myroot, VDB_PATH))
 		self.dbcatdir = self.dbroot+"/"+cat
@@ -1037,6 +1038,11 @@ class dblink(object):
 		if os.path.exists(self.dbdir+"/CONTENTS"):
 			os.unlink(self.dbdir+"/CONTENTS")
 
+	def _clear_contents_cache(self):
+		self.contentscache = None
+		self._contents_inodes = None
+		self._contents_basenames = None
+
 	def getcontents(self):
 		"""
 		Get the installed files of a given package (aka what that package installed)
@@ -1923,6 +1929,7 @@ class dblink(object):
 		"""
 
 		srcroot = normalize_path(srcroot).rstrip(os.path.sep) + os.path.sep
+		destroot = normalize_path(destroot).rstrip(os.path.sep) + os.path.sep
 
 		if not os.path.isdir(srcroot):
 			writemsg("!!! Directory Not Found: D='%s'\n" % srcroot,
@@ -2063,8 +2070,11 @@ class dblink(object):
 			self._preserve_libs(srcroot, destroot, myfilelist+mylinklist, counter, inforoot)
 
 		# check for package collisions
-		collisions = self._collision_protect(srcroot, destroot, others_in_slot,
-			myfilelist+mylinklist)
+		blockers = self._blockers
+		if blockers is None:
+			blockers = []
+		collisions = self._collision_protect(srcroot, destroot,
+			others_in_slot + blockers, myfilelist + mylinklist)
 
 		# Make sure the ebuild environment is initialized and that ${T}/elog
 		# exists for logging of collision-protect eerror messages.
@@ -2284,6 +2294,44 @@ class dblink(object):
 		self.dbdir = self.dbpkgdir
 		self.delete()
 		_movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings)
+
+		# Check for file collisions with blocking packages
+		# and remove any colliding files from their CONTENTS
+		# since they now belong to this package.
+		self._clear_contents_cache()
+		contents = self.getcontents()
+		destroot_len = len(destroot) - 1
+		for blocker in blockers:
+			blocker_contents = blocker.getcontents()
+			collisions = []
+			for filename in blocker_contents:
+				relative_filename = filename[destroot_len:]
+				if self.isowner(relative_filename, destroot):
+					collisions.append(filename)
+			if not collisions:
+				continue
+			for filename in collisions:
+				del blocker_contents[filename]
+			f = atomic_ofstream(os.path.join(blocker.dbdir, "CONTENTS"))
+			try:
+				for filename in sorted(blocker_contents):
+					entry_data = blocker_contents[filename]
+					entry_type = entry_data[0]
+					relative_filename = filename[destroot_len:]
+					if entry_type == "obj":
+						entry_type, mtime, md5sum = entry_data
+						line = "%s %s %s %s\n" % \
+							(entry_type, relative_filename, md5sum, mtime)
+					elif entry_type == "sym":
+						entry_type, mtime, link = entry_data
+						line = "%s %s -> %s %s\n" % \
+							(entry_type, relative_filename, link, mtime)
+					else: # dir, dev, fif
+						line = "%s %s\n" % (entry_type, relative_filename)
+					f.write(line)
+			finally:
+				f.close()
+
 		# Due to mtime granularity, mtime checks do not always properly
 		# invalidate vardbapi caches.
 		self.vartree.dbapi.mtdircache.pop(self.cat, None)