Merge package files in a subprocess.

This allows the Scheduler to run in the main thread while files are
moved or copied asynchronously.

2 files changed, 121 insertions, 67 deletions

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
new file mode 100644
index 000000000..b5af7142c
--- /dev/null
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -0,0 +1,41 @@
+# Copyright 2010 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import portage
+from portage import os
+from _emerge.SpawnProcess import SpawnProcess
+
+class MergeProcess(SpawnProcess):
+	"""
+	Merge package files in a subprocess, so the Scheduler can run in the
+	main thread while files are moved or copied asynchronously.
+	"""
+
+	__slots__ = ('cfgfiledict', 'conf_mem_file', \
+		'destroot', 'dblink', 'srcroot',)
+
+	def _spawn(self, args, fd_pipes=None, **kwargs):
+		"""
+		Fork a subprocess, apply local settings, and call
+		dblink._merge_process().
+		"""
+
+		pid = os.fork()
+		if pid != 0:
+			portage.process.spawned_pids.append(pid)
+			return [pid]
+
+		portage.process._setup_pipes(fd_pipes)
+
+		portage.output.havecolor = self.dblink.settings.get('NOCOLOR') \
+			not in ('yes', 'true')
+
+		# In this subprocess we want dblink._display_merge() to use
+		# stdout/stderr directly since they are pipes. This behavior
+		# is triggered when dblink._scheduler is None.
+		self.dblink._scheduler = None
+
+		rval = self.dblink._merge_process(self.srcroot, self.destroot,
+			self.cfgfiledict, self.conf_mem_file)
+
+		os._exit(rval)
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 4b0b47ba0..5e00f7fd7 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -10,6 +10,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.checksum:_perform_md5_merge@perform_md5',
 	'portage.data:portage_gid,portage_uid,secpass',
 	'portage.dbapi.dep_expand:dep_expand',
+	'portage.dbapi._MergeProcess:MergeProcess',
 	'portage.dep:dep_getkey,isjustname,match_from_list,' + \
 	 	'use_reduce,_slot_re',
 	'portage.elog:elog_process,_preload_elog_modules',
@@ -3183,19 +3184,11 @@ class dblink(object):
 			'w', encoding=_encodings['repo.content'], errors='backslashreplace'
 			).write(str(counter))
 
-		# open CONTENTS file (possibly overwriting old one) for recording
-		outfile = codecs.open(_unicode_encode(
-			os.path.join(self.dbtmpdir, 'CONTENTS'),
-			encoding=_encodings['fs'], errors='strict'),
-			mode='w', encoding=_encodings['repo.content'],
-			errors='backslashreplace')
-
 		self.updateprotect()
 
 		#if we have a file containing previously-merged config file md5sums, grab it.
 		conf_mem_file = os.path.join(self._eroot, CONFIG_MEMORY_FILE)
 		cfgfiledict = grabdict(conf_mem_file)
-		cfgfiledict_orig = cfgfiledict.copy()
 		if "NOCONFMEM" in self.settings:
 			cfgfiledict["IGNORE"]=1
 		else:
@@ -3210,61 +3203,18 @@ class dblink(object):
 				cfgfiledict["IGNORE"] = 1
 				break
 
-		# Don't bump mtimes on merge since some application require
-		# preservation of timestamps.  This means that the unmerge phase must
-		# check to see if file belongs to an installed instance in the same
-		# slot.
-		mymtime = None
-
-		# set umask to 0 for merging; back up umask, save old one in prevmask (since this is a global change)
-		prevmask = os.umask(0)
-		secondhand = []
-
-		# we do a first merge; this will recurse through all files in our srcroot but also build up a
-		# "second hand" of symlinks to merge later
-		if self.mergeme(srcroot, destroot, outfile, secondhand, "", cfgfiledict, mymtime):
-			return 1
-
-		# now, it's time for dealing our second hand; we'll loop until we can't merge anymore.	The rest are
-		# broken symlinks.  We'll merge them too.
-		lastlen = 0
-		while len(secondhand) and len(secondhand)!=lastlen:
-			# clear the thirdhand.	Anything from our second hand that
-			# couldn't get merged will be added to thirdhand.
-
-			thirdhand = []
-			if self.mergeme(srcroot, destroot, outfile, thirdhand,
-				secondhand, cfgfiledict, mymtime):
-				return 1
-
-			#swap hands
-			lastlen = len(secondhand)
-
-			# our thirdhand now becomes our secondhand.  It's ok to throw
-			# away secondhand since thirdhand contains all the stuff that
-			# couldn't be merged.
-			secondhand = thirdhand
+		merge_task = MergeProcess(
+			background=(self.settings.get('PORTAGE_BACKGROUND') == '1'),
+			cfgfiledict=cfgfiledict, conf_mem_file=conf_mem_file, dblink=self,
+			destroot=destroot,
+			logfile=self.settings.get('PORTAGE_LOG_FILE'),
+			scheduler=(scheduler or PollScheduler().sched_iface),
+			srcroot=srcroot)
 
-		if len(secondhand):
-			# force merge of remaining symlinks (broken or circular; oh well)
-			if self.mergeme(srcroot, destroot, outfile, None,
-				secondhand, cfgfiledict, mymtime):
-				return 1
-		self._md5_merge_map.clear()
-
-		#restore umask
-		os.umask(prevmask)
-
-		#if we opened it, close it
-		outfile.flush()
-		outfile.close()
-
-		# write out our collection of md5sums
-		cfgfiledict.pop("IGNORE", None)
-		if cfgfiledict != cfgfiledict_orig:
-			ensure_dirs(os.path.dirname(conf_mem_file),
-				gid=portage_gid, mode=0o2750, mask=0o2)
-			writedict(cfgfiledict, conf_mem_file)
+		merge_task.start()
+		rval = merge_task.wait()
+		if rval != os.EX_OK:
+			return rval
 
 		# These caches are populated during collision-protect and the data
 		# they contain is now invalid. It's very important to invalidate
@@ -3449,6 +3399,74 @@ class dblink(object):
 
 		return backup_p
 
+	def _merge_process(self, srcroot, destroot, cfgfiledict, conf_mem_file):
+
+		cfgfiledict_orig = cfgfiledict.copy()
+
+		# open CONTENTS file (possibly overwriting old one) for recording
+		outfile = codecs.open(_unicode_encode(
+			os.path.join(self.dbtmpdir, 'CONTENTS'),
+			encoding=_encodings['fs'], errors='strict'),
+			mode='w', encoding=_encodings['repo.content'],
+			errors='backslashreplace')
+
+		# Don't bump mtimes on merge since some application require
+		# preservation of timestamps.  This means that the unmerge phase must
+		# check to see if file belongs to an installed instance in the same
+		# slot.
+		mymtime = None
+
+		# set umask to 0 for merging; back up umask, save old one in prevmask (since this is a global change)
+		prevmask = os.umask(0)
+		secondhand = []
+
+		# we do a first merge; this will recurse through all files in our srcroot but also build up a
+		# "second hand" of symlinks to merge later
+		if self.mergeme(srcroot, destroot, outfile, secondhand, "", cfgfiledict, mymtime):
+			return 1
+
+		# now, it's time for dealing our second hand; we'll loop until we can't merge anymore.	The rest are
+		# broken symlinks.  We'll merge them too.
+		lastlen = 0
+		while len(secondhand) and len(secondhand)!=lastlen:
+			# clear the thirdhand.	Anything from our second hand that
+			# couldn't get merged will be added to thirdhand.
+
+			thirdhand = []
+			if self.mergeme(srcroot, destroot, outfile, thirdhand,
+				secondhand, cfgfiledict, mymtime):
+				return 1
+
+			#swap hands
+			lastlen = len(secondhand)
+
+			# our thirdhand now becomes our secondhand.  It's ok to throw
+			# away secondhand since thirdhand contains all the stuff that
+			# couldn't be merged.
+			secondhand = thirdhand
+
+		if len(secondhand):
+			# force merge of remaining symlinks (broken or circular; oh well)
+			if self.mergeme(srcroot, destroot, outfile, None,
+				secondhand, cfgfiledict, mymtime):
+				return 1
+
+		#restore umask
+		os.umask(prevmask)
+
+		#if we opened it, close it
+		outfile.flush()
+		outfile.close()
+
+		# write out our collection of md5sums
+		if cfgfiledict != cfgfiledict_orig:
+			cfgfiledict.pop("IGNORE", None)
+			ensure_dirs(os.path.dirname(conf_mem_file),
+				gid=portage_gid, mode=0o2750, mask=0o2)
+			writedict(cfgfiledict, conf_mem_file)
+
+		return os.EX_OK
+
 	def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime):
 		"""
 		
@@ -3479,7 +3497,6 @@ class dblink(object):
 
 		showMessage = self._display_merge
 		writemsg = self._display_merge
-		scheduler = self._scheduler
 
 		os = _os_merge
 		sep = os.sep
@@ -3499,10 +3516,6 @@ class dblink(object):
 
 		for i, x in enumerate(mergelist):
 
-			if scheduler is not None and \
-				0 == i % self._file_merge_yield_interval:
-				scheduler.scheduleYield()
-
 			mysrc = join(srcroot, offset, x)
 			mydest = join(destroot, offset, x)
 			# myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/")