From 2758391fc9f3791cdd3260cb8113aad43d8063a3 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sun, 29 Jun 2008 14:51:05 +0000 Subject: * Add "fd_pipes" and "returnpid" parameters to doebuild() and pass these into spawn calls, enabling ebuild processes to execute asynchronously. * Add a EbuildPhase class that's derived from the pty logging code inside portage.spawn(). * Integrate post-phase code from spawnebuild() into EbuildBuild.execute() so that it still gets called even though doebuild() calls execute asynchronously. svn path=/main/trunk/; revision=10849 --- pym/_emerge/__init__.py | 172 ++++++++++++++++++++++++++++++++++++++++++++++-- pym/portage/__init__.py | 68 ++++++++++++------- 2 files changed, 211 insertions(+), 29 deletions(-) diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index dc266ddb0..84b5769a3 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1541,22 +1541,184 @@ class EbuildBuild(Task): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(self.pkg.cpv) - debug = self.settings.get("PORTAGE_DEBUG") == "1" + settings = self.settings + debug = settings.get("PORTAGE_DEBUG") == "1" + cleanup = 1 retval = portage.doebuild(ebuild_path, "clean", - root_config.root, self.settings, debug, cleanup=1, + root_config.root, settings, debug, cleanup=cleanup, mydbapi=portdb, tree="porttree") if retval != os.EX_OK: return retval + # This initializes PORTAGE_LOG_FILE. + portage.prepare_build_dirs(root_config.root, settings, cleanup) + + fd_pipes = { + 0 : sys.stdin.fileno(), + 1 : sys.stdout.fileno(), + 2 : sys.stderr.fileno(), + } + for mydo in self._phases: - retval = portage.doebuild(ebuild_path, mydo, - root_config.root, self.settings, debug, - mydbapi=portdb, tree="porttree") + ebuild_phase = EbuildPhase(fd_pipes=fd_pipes, + pkg=self.pkg, phase=mydo, settings=settings) + ebuild_phase.start() + ebuild_phase._output_handler() + retval = ebuild_phase.wait() + + portage._post_phase_userpriv_perms(settings) + if mydo == "install": + portage._check_build_log(settings) + if retval == os.EX_OK: + retval = portage._post_src_install_checks(settings) + if retval != os.EX_OK: return retval + return os.EX_OK +class EbuildPhase(SlotObject): + + __slots__ = ("fd_pipes", "phase", "pkg", "settings", + "pid", "returncode", "files") + + _file_names = ("log", "stdout", "ebuild") + _files_dict = slot_dict_class(_file_names) + _bufsize = 4096 + + def start(self): + root_config = self.pkg.root_config + portdb = root_config.trees["porttree"].dbapi + ebuild_path = portdb.findname(self.pkg.cpv) + settings = self.settings + debug = settings.get("PORTAGE_DEBUG") == "1" + logfile = settings.get("PORTAGE_LOG_FILE") + master_fd = None + slave_fd = None + fd_pipes = self.fd_pipes.copy() + + # flush any pending output + for fd in fd_pipes.itervalues(): + if fd == sys.stdout.fileno(): + sys.stdout.flush() + if fd == sys.stderr.fileno(): + sys.stderr.flush() + + fd_pipes_orig = None + self.files = self._files_dict() + files = self.files + got_pty = False + + portage._doebuild_exit_status_unlink( + settings.get("EBUILD_EXIT_STATUS_FILE")) + + if logfile: + if portage._disable_openpty: + master_fd, slave_fd = os.pipe() + else: + from pty import openpty + try: + master_fd, slave_fd = openpty() + got_pty = True + except EnvironmentError, e: + portage._disable_openpty = True + portage.writemsg("openpty failed: '%s'\n" % str(e), + noiselevel=-1) + del e + master_fd, slave_fd = os.pipe() + + if got_pty: + # Disable post-processing of output since otherwise weird + # things like \n -> \r\n transformations may occur. + import termios + mode = termios.tcgetattr(slave_fd) + mode[1] &= ~termios.OPOST + termios.tcsetattr(slave_fd, termios.TCSANOW, mode) + + import fcntl + fcntl.fcntl(master_fd, fcntl.F_SETFL, + fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + fd_pipes.setdefault(0, sys.stdin.fileno()) + fd_pipes_orig = fd_pipes.copy() + if got_pty and os.isatty(fd_pipes_orig[1]): + from portage.output import get_term_size, set_term_size + rows, columns = get_term_size() + set_term_size(rows, columns, slave_fd) + fd_pipes[0] = fd_pipes_orig[0] + fd_pipes[1] = slave_fd + fd_pipes[2] = slave_fd + + retval = portage.doebuild(ebuild_path, self.phase, + root_config.root, settings, debug, + mydbapi=portdb, tree="porttree", + fd_pipes=fd_pipes, returnpid=True) + + self.pid = retval[0] + + if logfile: + os.close(slave_fd) + files["log"] = open(logfile, 'a') + files["stdout"] = os.fdopen(os.dup(fd_pipes_orig[1]), 'w') + files["ebuild"] = os.fdopen(master_fd, 'r') + + def _output_handler(self): + log_file = self.files.get("log") + if log_file is None: + return + ebuild_file = self.files["ebuild"] + stdout_file = self.files["stdout"] + iwtd = [ebuild_file] + owtd = [] + ewtd = [] + import array, select + buffsize = self._bufsize + eof = False + while not eof: + events = select.select(iwtd, owtd, ewtd) + for f in events[0]: + # Use non-blocking mode to prevent read + # calls from blocking indefinitely. + buf = array.array('B') + try: + buf.fromfile(f, buffsize) + except EOFError: + pass + if not buf: + eof = True + break + if f is ebuild_file: + buf.tofile(stdout_file) + stdout_file.flush() + buf.tofile(log_file) + log_file.flush() + log_file.close() + stdout_file.close() + ebuild_file.close() + + def wait(self): + pid = self.pid + retval = os.waitpid(pid, 0)[1] + portage.process.spawned_pids.remove(pid) + if retval != os.EX_OK: + if retval & 0xff: + retval = (retval & 0xff) << 8 + else: + retval = retval >> 8 + + msg = portage._doebuild_exit_status_check( + self.phase, self.settings) + if msg: + retval = 1 + from textwrap import wrap + from portage.elog.messages import eerror + for l in wrap(msg, 72): + eerror(l, phase=self.phase, key=self.pkg.cpv) + + self.returncode = retval + return self.returncode + class EbuildBinpkg(Task): """ This assumes that src_install() has successfully completed. diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 34f05d19c..1f0003326 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2940,6 +2940,10 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero env=mysettings.environ() keywords["opt_name"]="[%s]" % mysettings["PF"] + if "EMERGE_FROM" in mysettings: + # emerge handles logging externally + keywords.pop("logfile", None) + fd_pipes = keywords.get("fd_pipes") if fd_pipes is None: fd_pipes = { @@ -4160,12 +4164,15 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0): return 1 # parse actionmap to spawn ebuild with the appropriate args -def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None): +def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, + logfile=None, fd_pipes=None, returnpid=False): if "EMERGE_FROM" not in mysettings and \ (alwaysdep or "noauto" not in mysettings.features): # process dependency first if "dep" in actionmap[mydo]: - retval=spawnebuild(actionmap[mydo]["dep"],actionmap,mysettings,debug,alwaysdep=alwaysdep,logfile=logfile) + retval = spawnebuild(actionmap[mydo]["dep"], actionmap, + mysettings, debug, alwaysdep=alwaysdep, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid) if retval: return retval kwargs = actionmap[mydo]["args"] @@ -4177,10 +4184,13 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None): mysettings._filter_calling_env = True try: phase_retval = spawn(actionmap[mydo]["cmd"] % mydo, - mysettings, debug=debug, logfile=logfile, **kwargs) + mysettings, debug=debug, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid, **kwargs) finally: mysettings["EBUILD_PHASE"] = "" mysettings._filter_calling_env = filter_calling_env_state + if returnpid: + return phase_retval msg = _doebuild_exit_status_check(mydo, mysettings) if msg: phase_retval = 1 @@ -4189,27 +4199,29 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None): for l in wrap(msg, 72): eerror(l, phase=mydo, key=mysettings.mycpv) - if "userpriv" in mysettings.features and \ - not kwargs["droppriv"] and secpass >= 2: + _post_phase_userpriv_perms(mysettings) + if mydo == "install": + _check_build_log(mysettings) + if phase_retval == os.EX_OK: + phase_retval = _post_src_install_checks(mysettings) + return phase_retval + +def _post_phase_userpriv_perms(mysettings): + if "userpriv" in mysettings.features and secpass >= 2: """ Privileged phases may have left files that need to be made writable to a less privileged user.""" apply_recursive_permissions(mysettings["T"], uid=portage_uid, gid=portage_gid, dirmode=070, dirmask=0, filemode=060, filemask=0) - if phase_retval == os.EX_OK: - if mydo == "install" and logfile: - _check_build_log(mysettings) - - if mydo == "install": - _post_src_install_uid_fix(mysettings) - qa_retval = _spawn_misc_sh(mysettings, ["install_qa_check", - "install_symlink_html_docs"], **kwargs) - if qa_retval != os.EX_OK: - writemsg("!!! install_qa_check failed; exiting.\n", - noiselevel=-1) - return qa_retval - return phase_retval +def _post_src_install_checks(mysettings): + _post_src_install_uid_fix(mysettings) + retval = _spawn_misc_sh(mysettings, ["install_qa_check", + "install_symlink_html_docs"]) + if retval != os.EX_OK: + writemsg("!!! install_qa_check failed; exiting.\n", + noiselevel=-1) + return retval def _check_build_log(mysettings): """ @@ -4220,7 +4232,9 @@ def _check_build_log(mysettings): * command not found * Unrecognized configure options """ - logfile = mysettings.get("PORTAGE_LOG_FILE", None) + logfile = mysettings.get("PORTAGE_LOG_FILE") + if logfile is None: + return try: f = open(logfile, 'rb') except EnvironmentError: @@ -4777,8 +4791,9 @@ _doebuild_broken_manifests = set() def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None, - mydbapi=None, vartree=None, prev_mtimes=None): - + mydbapi=None, vartree=None, prev_mtimes=None, + fd_pipes=None, returnpid=False): + """ Wrapper function that invokes specific ebuild phases through the spawning of ebuild.sh @@ -5190,7 +5205,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, elif mydo == "setup": retval = spawn( _shell_quote(ebuild_sh_binary) + " " + mydo, mysettings, - debug=debug, free=1, logfile=logfile) + debug=debug, free=1, logfile=logfile, fd_pipes=fd_pipes, + returnpid=returnpid) + if returnpid: + return retval retval = exit_status_check(retval) if secpass >= 2: """ Privileged phases may have left files that need to be made @@ -5418,7 +5436,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, raise portage.exception.PermissionDenied( "access('%s', os.W_OK)" % parent_dir) retval = spawnebuild(mydo, - actionmap, mysettings, debug, logfile=logfile) + actionmap, mysettings, debug, logfile=logfile, + fd_pipes=fd_pipes, returnpid=returnpid) elif mydo=="qmerge": # check to ensure install was run. this *only* pops up when users # forget it and are using ebuild @@ -5438,7 +5457,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes) elif mydo=="merge": retval = spawnebuild("install", actionmap, mysettings, debug, - alwaysdep=1, logfile=logfile) + alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes, + returnpid=returnpid) retval = exit_status_check(retval) if retval != os.EX_OK: # The merge phase handles this already. Callers don't know how -- cgit v1.2.3-1-g7c22