* Add "fd_pipes" and "returnpid" parameters to doebuild() and pass

   these into spawn calls, enabling ebuild processes to execute
   asynchronously.

 * Add a EbuildPhase class that's derived from the pty logging code
   inside portage.spawn().

 * Integrate post-phase code from spawnebuild() into EbuildBuild.execute()
   so that it still gets called even though doebuild() calls execute
   asynchronously.

svn path=/main/trunk/; revision=10849

2 files changed, 211 insertions, 29 deletions

diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index dc266ddb0..84b5769a3 100644
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -1541,22 +1541,184 @@ class EbuildBuild(Task):
 		root_config = self.pkg.root_config
 		portdb = root_config.trees["porttree"].dbapi
 		ebuild_path = portdb.findname(self.pkg.cpv)
-		debug = self.settings.get("PORTAGE_DEBUG") == "1"
+		settings = self.settings
+		debug = settings.get("PORTAGE_DEBUG") == "1"
+		cleanup = 1
 
 		retval = portage.doebuild(ebuild_path, "clean",
-			root_config.root, self.settings, debug, cleanup=1,
+			root_config.root, settings, debug, cleanup=cleanup,
 			mydbapi=portdb, tree="porttree")
 		if retval != os.EX_OK:
 			return retval
 
+		# This initializes PORTAGE_LOG_FILE.
+		portage.prepare_build_dirs(root_config.root, settings, cleanup)
+
+		fd_pipes = {
+			0 : sys.stdin.fileno(),
+			1 : sys.stdout.fileno(),
+			2 : sys.stderr.fileno(),
+		}
+
 		for mydo in self._phases:
-			retval = portage.doebuild(ebuild_path, mydo,
-				root_config.root, self.settings, debug,
-				mydbapi=portdb, tree="porttree")
+			ebuild_phase = EbuildPhase(fd_pipes=fd_pipes,
+				pkg=self.pkg, phase=mydo, settings=settings)
+			ebuild_phase.start()
+			ebuild_phase._output_handler()
+			retval = ebuild_phase.wait()
+
+			portage._post_phase_userpriv_perms(settings)
+			if mydo == "install":
+				portage._check_build_log(settings)
+				if retval == os.EX_OK:
+					retval = portage._post_src_install_checks(settings)
+
 			if retval != os.EX_OK:
 				return retval
+
 		return os.EX_OK
 
+class EbuildPhase(SlotObject):
+
+	__slots__ = ("fd_pipes", "phase", "pkg", "settings",
+		"pid", "returncode", "files")
+
+	_file_names = ("log", "stdout", "ebuild")
+	_files_dict = slot_dict_class(_file_names)
+	_bufsize = 4096
+
+	def start(self):
+		root_config = self.pkg.root_config
+		portdb = root_config.trees["porttree"].dbapi
+		ebuild_path = portdb.findname(self.pkg.cpv)
+		settings = self.settings
+		debug = settings.get("PORTAGE_DEBUG") == "1"
+		logfile = settings.get("PORTAGE_LOG_FILE")
+		master_fd = None
+		slave_fd = None
+		fd_pipes = self.fd_pipes.copy()
+
+		# flush any pending output
+		for fd in fd_pipes.itervalues():
+			if fd == sys.stdout.fileno():
+				sys.stdout.flush()
+			if fd == sys.stderr.fileno():
+				sys.stderr.flush()
+
+		fd_pipes_orig = None
+		self.files = self._files_dict()
+		files = self.files
+		got_pty = False
+
+		portage._doebuild_exit_status_unlink(
+			settings.get("EBUILD_EXIT_STATUS_FILE"))
+
+		if logfile:
+			if portage._disable_openpty:
+				master_fd, slave_fd = os.pipe()
+			else:
+				from pty import openpty
+				try:
+					master_fd, slave_fd = openpty()
+					got_pty = True
+				except EnvironmentError, e:
+					portage._disable_openpty = True
+					portage.writemsg("openpty failed: '%s'\n" % str(e),
+						noiselevel=-1)
+					del e
+					master_fd, slave_fd = os.pipe()
+
+			if got_pty:
+				# Disable post-processing of output since otherwise weird
+				# things like \n -> \r\n transformations may occur.
+				import termios
+				mode = termios.tcgetattr(slave_fd)
+				mode[1] &= ~termios.OPOST
+				termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
+
+			import fcntl
+			fcntl.fcntl(master_fd, fcntl.F_SETFL,
+				fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
+
+			fd_pipes.setdefault(0, sys.stdin.fileno())
+			fd_pipes_orig = fd_pipes.copy()
+			if got_pty and os.isatty(fd_pipes_orig[1]):
+				from portage.output import get_term_size, set_term_size
+				rows, columns = get_term_size()
+				set_term_size(rows, columns, slave_fd)
+			fd_pipes[0] = fd_pipes_orig[0]
+			fd_pipes[1] = slave_fd
+			fd_pipes[2] = slave_fd
+
+		retval = portage.doebuild(ebuild_path, self.phase,
+			root_config.root, settings, debug,
+			mydbapi=portdb, tree="porttree",
+			fd_pipes=fd_pipes, returnpid=True)
+
+		self.pid = retval[0]
+
+		if logfile:
+			os.close(slave_fd)
+			files["log"] = open(logfile, 'a')
+			files["stdout"] = os.fdopen(os.dup(fd_pipes_orig[1]), 'w')
+			files["ebuild"] = os.fdopen(master_fd, 'r')
+
+	def _output_handler(self):
+		log_file = self.files.get("log")
+		if log_file is None:
+			return
+		ebuild_file = self.files["ebuild"]
+		stdout_file = self.files["stdout"]
+		iwtd = [ebuild_file]
+		owtd = []
+		ewtd = []
+		import array, select
+		buffsize = self._bufsize
+		eof = False
+		while not eof:
+			events = select.select(iwtd, owtd, ewtd)
+			for f in events[0]:
+				# Use non-blocking mode to prevent read
+				# calls from blocking indefinitely.
+				buf = array.array('B')
+				try:
+					buf.fromfile(f, buffsize)
+				except EOFError:
+					pass
+				if not buf:
+					eof = True
+					break
+				if f is ebuild_file:
+					buf.tofile(stdout_file)
+					stdout_file.flush()
+					buf.tofile(log_file)
+					log_file.flush()
+		log_file.close()
+		stdout_file.close()
+		ebuild_file.close()
+
+	def wait(self):
+		pid = self.pid
+		retval = os.waitpid(pid, 0)[1]
+		portage.process.spawned_pids.remove(pid)
+		if retval != os.EX_OK:
+			if retval & 0xff:
+				retval = (retval & 0xff) << 8
+			else:
+				retval = retval >> 8
+
+		msg = portage._doebuild_exit_status_check(
+			self.phase, self.settings)
+		if msg:
+			retval = 1
+			from textwrap import wrap
+			from portage.elog.messages import eerror
+			for l in wrap(msg, 72):
+				eerror(l, phase=self.phase, key=self.pkg.cpv)
+
+		self.returncode = retval
+		return self.returncode
+
 class EbuildBinpkg(Task):
 	"""
 	This assumes that src_install() has successfully completed.
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 34f05d19c..1f0003326 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -2940,6 +2940,10 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero
 		env=mysettings.environ()
 		keywords["opt_name"]="[%s]" % mysettings["PF"]
 
+	if "EMERGE_FROM" in mysettings:
+		# emerge handles logging externally
+		keywords.pop("logfile", None)
+
 	fd_pipes = keywords.get("fd_pipes")
 	if fd_pipes is None:
 		fd_pipes = {
@@ -4160,12 +4164,15 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0):
 	return 1
 
 # parse actionmap to spawn ebuild with the appropriate args
-def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None):
+def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0,
+	logfile=None, fd_pipes=None, returnpid=False):
 	if "EMERGE_FROM" not in mysettings and \
 		(alwaysdep or "noauto" not in mysettings.features):
 		# process dependency first
 		if "dep" in actionmap[mydo]:
-			retval=spawnebuild(actionmap[mydo]["dep"],actionmap,mysettings,debug,alwaysdep=alwaysdep,logfile=logfile)
+			retval = spawnebuild(actionmap[mydo]["dep"], actionmap,
+				mysettings, debug, alwaysdep=alwaysdep, logfile=logfile,
+				fd_pipes=fd_pipes, returnpid=returnpid)
 			if retval:
 				return retval
 	kwargs = actionmap[mydo]["args"]
@@ -4177,10 +4184,13 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None):
 		mysettings._filter_calling_env = True
 	try:
 		phase_retval = spawn(actionmap[mydo]["cmd"] % mydo,
-			mysettings, debug=debug, logfile=logfile, **kwargs)
+			mysettings, debug=debug, logfile=logfile,
+			fd_pipes=fd_pipes, returnpid=returnpid, **kwargs)
 	finally:
 		mysettings["EBUILD_PHASE"] = ""
 		mysettings._filter_calling_env = filter_calling_env_state
+	if returnpid:
+		return phase_retval
 	msg = _doebuild_exit_status_check(mydo, mysettings)
 	if msg:
 		phase_retval = 1
@@ -4189,27 +4199,29 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None):
 		for l in wrap(msg, 72):
 			eerror(l, phase=mydo, key=mysettings.mycpv)
 
-	if "userpriv" in mysettings.features and \
-		not kwargs["droppriv"] and secpass >= 2:
+	_post_phase_userpriv_perms(mysettings)
+	if mydo == "install":
+		_check_build_log(mysettings)
+		if phase_retval == os.EX_OK:
+			phase_retval = _post_src_install_checks(mysettings)
+	return phase_retval
+
+def _post_phase_userpriv_perms(mysettings):
+	if "userpriv" in mysettings.features and secpass >= 2:
 		""" Privileged phases may have left files that need to be made
 		writable to a less privileged user."""
 		apply_recursive_permissions(mysettings["T"],
 			uid=portage_uid, gid=portage_gid, dirmode=070, dirmask=0,
 			filemode=060, filemask=0)
 
-	if phase_retval == os.EX_OK:
-		if mydo == "install" and logfile:
-			_check_build_log(mysettings)
-
-		if mydo == "install":
-			_post_src_install_uid_fix(mysettings)
-			qa_retval = _spawn_misc_sh(mysettings, ["install_qa_check",
-				"install_symlink_html_docs"], **kwargs)
-			if qa_retval != os.EX_OK:
-				writemsg("!!! install_qa_check failed; exiting.\n",
-					noiselevel=-1)
-				return qa_retval
-	return phase_retval
+def _post_src_install_checks(mysettings):
+	_post_src_install_uid_fix(mysettings)
+	retval = _spawn_misc_sh(mysettings, ["install_qa_check",
+		"install_symlink_html_docs"])
+	if retval != os.EX_OK:
+		writemsg("!!! install_qa_check failed; exiting.\n",
+			noiselevel=-1)
+	return retval
 
 def _check_build_log(mysettings):
 	"""
@@ -4220,7 +4232,9 @@ def _check_build_log(mysettings):
 	  * command not found
 	  * Unrecognized configure options
 	"""
-	logfile = mysettings.get("PORTAGE_LOG_FILE", None)
+	logfile = mysettings.get("PORTAGE_LOG_FILE")
+	if logfile is None:
+		return
 	try:
 		f = open(logfile, 'rb')
 	except EnvironmentError:
@@ -4777,8 +4791,9 @@ _doebuild_broken_manifests = set()
 
 def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
 	fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None,
-	mydbapi=None, vartree=None, prev_mtimes=None):
-	
+	mydbapi=None, vartree=None, prev_mtimes=None,
+	fd_pipes=None, returnpid=False):
+
 	"""
 	Wrapper function that invokes specific ebuild phases through the spawning
 	of ebuild.sh
@@ -5190,7 +5205,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
 		elif mydo == "setup":
 			retval = spawn(
 				_shell_quote(ebuild_sh_binary) + " " + mydo, mysettings,
-				debug=debug, free=1, logfile=logfile)
+				debug=debug, free=1, logfile=logfile, fd_pipes=fd_pipes,
+				returnpid=returnpid)
+			if returnpid:
+				return retval
 			retval = exit_status_check(retval)
 			if secpass >= 2:
 				""" Privileged phases may have left files that need to be made
@@ -5418,7 +5436,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
 					raise portage.exception.PermissionDenied(
 						"access('%s', os.W_OK)" % parent_dir)
 			retval = spawnebuild(mydo,
-				actionmap, mysettings, debug, logfile=logfile)
+				actionmap, mysettings, debug, logfile=logfile,
+				fd_pipes=fd_pipes, returnpid=returnpid)
 		elif mydo=="qmerge":
 			# check to ensure install was run.  this *only* pops up when users
 			# forget it and are using ebuild
@@ -5438,7 +5457,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
 				mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes)
 		elif mydo=="merge":
 			retval = spawnebuild("install", actionmap, mysettings, debug,
-				alwaysdep=1, logfile=logfile)
+				alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes,
+				returnpid=returnpid)
 			retval = exit_status_check(retval)
 			if retval != os.EX_OK:
 				# The merge phase handles this already.  Callers don't know how