From 1b1dadc3a629c27f7bc599897cf844ee0206f22b Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 2 Jul 2008 01:13:15 +0000 Subject: * Rename existing EbuildBuild class to EbuildExecuter. * Split more code out of Scheduler._execute_task() and use it to make a more comprehensive EbuildBuild class. svn path=/main/trunk/; revision=10885 --- pym/_emerge/__init__.py | 337 ++++++++++++++++++++++++++++++------------------ 1 file changed, 210 insertions(+), 127 deletions(-) (limited to 'pym') diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index e2cb01a29..e875cfe8b 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -52,6 +52,7 @@ bad = create_color_func("BAD") # white looks bad on terminals with white background from portage.output import bold as white +import portage.elog import portage.dep portage.dep._dep_check_strict = True import portage.util @@ -1439,7 +1440,7 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): class EbuildFetcher(SlotObject): - __slots__ = ("fetch_all", "pkg", "pretend", "settings") + __slots__ = ("cancelled", "fetch_all", "pkg", "pretend", "settings") def execute(self): portdb = self.pkg.root_config.trees["porttree"].dbapi @@ -1454,7 +1455,7 @@ class EbuildFetcher(SlotObject): class EbuildFetcherAsync(SlotObject): - __slots__ = ("log_file", "fd_pipes", "pkg", + __slots__ = ("cancelled", "log_file", "fd_pipes", "pkg", "register", "unregister", "pid", "returncode", "files") @@ -1463,6 +1464,10 @@ class EbuildFetcherAsync(SlotObject): _bufsize = 4096 def start(self): + + if self.cancelled: + return + # flush any pending output fd_pipes = self.fd_pipes if fd_pipes is None: @@ -1554,6 +1559,18 @@ class EbuildFetcherAsync(SlotObject): self._set_returncode(retval) return self.returncode + def cancel(self): + if self.isAlive(): + os.kill(self.pid, signal.SIGTERM) + self.cancelled = True + if self.pid is not None: + self.wait() + return self.returncode + + def isAlive(self): + return self.pid is not None and \ + self.returncode is None + def wait(self): if self.returncode is not None: return self.returncode @@ -1644,20 +1661,137 @@ class EbuildBuildDir(SlotObject): class AlreadyLocked(portage.exception.PortageException): pass -class EbuildBuild(Task): - """ - TODO: Support asynchronous execution, to implement parallel builds. - """ +class EbuildBuild(SlotObject): + + __slots__ = ("args_set", "find_blockers", + "ldpath_mtimes", "logger", "opts", + "pkg", "pkg_count", "scheduler", + "settings") + + def execute(self): + + args_set = self.args_set + find_blockers = self.find_blockers + ldpath_mtimes = self.ldpath_mtimes + logger = self.logger + opts = self.opts + pkg = self.pkg + pkg_count = self.pkg_count + scheduler = self.scheduler + settings = self.settings + root_config = pkg.root_config + root = root_config.root + system_set = root_config.sets["system"] + world_set = root_config.sets["world"] + vartree = root_config.trees["vartree"] + portdb = root_config.trees["porttree"].dbapi + debug = settings.get("PORTAGE_DEBUG") == "1" + features = self.settings.features + settings["EMERGE_FROM"] = pkg.type_name + settings.backup_changes("EMERGE_FROM") + settings.reset() + ebuild_path = portdb.findname(self.pkg.cpv) + + #buildsyspkg: Check if we need to _force_ binary package creation + issyspkg = "buildsyspkg" in features and \ + system_set.findAtomForPackage(pkg) and \ + not opts.buildpkg + + if opts.fetchonly: + fetcher = EbuildFetcher(fetch_all=opts.fetch_all_uri, + pkg=pkg, pretend=opts.pretend, settings=settings) + retval = fetcher.execute() + if retval != os.EX_OK: + from portage.elog.messages import eerror + eerror("!!! Fetch for %s failed, continuing..." % pkg.cpv, + phase="unpack", key=pkg.cpv) + return retval + + build_dir = EbuildBuildDir(pkg=pkg, settings=settings) + try: + build_dir.lock() + # Cleaning is triggered before the setup + # phase, in portage.doebuild(). + msg = " === (%s of %s) Cleaning (%s::%s)" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) + short_msg = "emerge: (%s of %s) %s Clean" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv) + logger.log(msg, short_msg=short_msg) + + if opts.buildpkg or issyspkg: + if issyspkg: + portage.writemsg(">>> This is a system package, " + \ + "let's pack a rescue tarball.\n", noiselevel=-1) + msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) + short_msg = "emerge: (%s of %s) %s Compile" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv) + logger.log(msg, short_msg=short_msg) + + build = EbuildExecuter(pkg=pkg, register=scheduler.register, + schedule=scheduler.schedule, settings=settings, + unregister=scheduler.unregister) + retval = build.execute() + if retval != os.EX_OK: + return retval + + build = EbuildBinpkg(pkg=pkg, settings=settings) + retval = build.execute() + if retval != os.EX_OK: + return retval + + if opts.buildpkgonly: + msg = " === (%s of %s) Merging (%s::%s)" % \ + (pkg_count.curval, pkg_count.maxval, + pkg.cpv, ebuild_path) + short_msg = "emerge: (%s of %s) %s Merge" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv) + logger.log(msg, short_msg=short_msg) + + merge = EbuildMerge( + find_blockers=find_blockers, + ldpath_mtimes=ldpath_mtimes, + pkg=pkg, settings=settings) + retval = merge.execute() + if retval != os.EX_OK: + return retval + elif "noclean" not in settings.features: + portage.doebuild(ebuild_path, "clean", root, + settings, debug=debug, mydbapi=portdb, + tree="porttree") + else: + msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) + short_msg = "emerge: (%s of %s) %s Compile" % \ + (pkg_count.curval, pkg_count.curval, pkg.cpv) + logger.log(msg, short_msg=short_msg) + + build = EbuildExecuter(pkg=pkg, register=scheduler.register, + schedule=scheduler.schedule, settings=settings, + unregister=scheduler.unregister) + retval = build.execute() + if retval != os.EX_OK: + return retval + + merge = EbuildMerge( + find_blockers=self.find_blockers, + ldpath_mtimes=ldpath_mtimes, + pkg=pkg, settings=settings) + retval = merge.execute() + + if retval != os.EX_OK: + return retval + finally: + if build_dir.locked: + portage.elog.elog_process(pkg.cpv, settings) + build_dir.unlock() + +class EbuildExecuter(SlotObject): + __slots__ = ("pkg", "register", "schedule", "settings", "unregister") _phases = ("setup", "unpack", "compile", "test", "install") - def _get_hash_key(self): - hash_key = getattr(self, "_hash_key", None) - if hash_key is None: - self._hash_key = ("EbuildBuild", self.pkg._get_hash_key()) - return self._hash_key - def execute(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi @@ -2013,8 +2147,8 @@ class BinpkgFetcherAsync(SlotObject): def start(self): if self.cancelled: - self.pid = -1 return + writemsg(">>> starting parallel binpkg fetcher\n") fd_pipes = self.fd_pipes if fd_pipes is None: @@ -2196,6 +2330,12 @@ class BinpkgMerge(Task): return self._hash_key def execute(self): + + settings = self.settings + settings["EMERGE_FROM"] = self.pkg.type_name + settings.backup_changes("EMERGE_FROM") + settings.reset() + root_config = self.pkg.root_config retval = portage.pkgmerge(self.pkg_path, root_config.root, self.settings, @@ -6654,6 +6794,22 @@ class Scheduler(object): _fetch_log = "/var/log/emerge-fetch.log" + class _iface_class(SlotObject): + __slots__ = ("register", "schedule", "unregister") + + class _build_opts_class(SlotObject): + __slots__ = ("buildpkg", "buildpkgonly", + "fetch_all_uri", "fetchonly", "pretend") + + class _pkg_count_class(SlotObject): + __slots__ = ("curval", "maxval") + + class _emerge_log_class(SlotObject): + __slots__ = ("xterm_titles",) + + def log(self, *pargs, **kwargs): + emergelog(self.xterm_titles, *pargs, **kwargs) + def __init__(self, settings, trees, mtimedb, myopts, spinner, mergelist, favorites, digraph): self.settings = settings @@ -6664,6 +6820,10 @@ class Scheduler(object): self._mtimedb = mtimedb self._mergelist = mergelist self._favorites = favorites + self._args_set = InternalPackageSet(favorites) + self._build_opts = self._build_opts_class() + for k in self._build_opts.__slots__: + setattr(self._build_opts, k, "--" + k.replace("_", "-") in myopts) self.edebug = 0 if settings.get("PORTAGE_DEBUG", "") == "1": self.edebug = 1 @@ -6672,6 +6832,11 @@ class Scheduler(object): self.pkgsettings[root] = portage.config( clone=trees[root]["vartree"].settings) self.curval = 0 + self._logger = self._emerge_log_class( + xterm_titles=("notitles" not in settings.features)) + self._sched_iface = self._iface_class( + register=self._register, schedule=self._schedule, + unregister=self._unregister) self._poll_event_handlers = {} self._poll = select.poll() from collections import deque @@ -6905,12 +7070,12 @@ class Scheduler(object): "--onlydeps" in self.myopts pretend = "--pretend" in self.myopts ldpath_mtimes = mtimedb["ldpath"] - xterm_titles = "notitles" not in self.settings.features + logger = self._logger if "--resume" in self.myopts: # We're resuming. print colorize("GOOD", "*** Resuming merge...") - emergelog(xterm_titles, " *** Resuming merge...") + self._logger.log(" *** Resuming merge...") # Do this before verifying the ebuild Manifests since it might # be possible for the user to use --resume --skipfirst get past @@ -6923,6 +7088,7 @@ class Scheduler(object): getbinpkg = "--getbinpkg" in self.myopts if self._parallel_fetch: + portage.writemsg(">>> starting parallel fetch\n") for pkg in mylist: if not isinstance(pkg, Package): continue @@ -6981,7 +7147,8 @@ class Scheduler(object): # Filter mymergelist so that all the len(mymergelist) calls # below (for display) do not count Uninstall instances. mymergelist = [x for x in mymergelist if x[-1] == "merge"] - mergecount=0 + pkg_count = self._pkg_count_class( + curval=0, maxval=len(mymergelist)) for x in task_list: if x[0] == "blocks": continue @@ -7005,24 +7172,24 @@ class Scheduler(object): else: raise AssertionError("Package type: '%s'" % pkg_type) if not x.installed: - mergecount += 1 + pkg_count.curval += 1 try: self._execute_task(bad_resume_opts, failed_fetches, - mydbapi, mergecount, + mydbapi, pkg_count, myfeat, mymergelist, x, - prefetchers, xterm_titles) + prefetchers) except self._pkg_failure, e: return e.status - return self._post_merge(mtimedb, xterm_titles, failed_fetches) + return self._post_merge(mtimedb, + self._logger.xterm_titles, failed_fetches) def _execute_task(self, bad_resume_opts, - failed_fetches, mydbapi, mergecount, myfeat, - mymergelist, pkg, prefetchers, xterm_titles): + failed_fetches, mydbapi, pkg_count, myfeat, + mymergelist, pkg, prefetchers): favorites = self._favorites mtimedb = self._mtimedb - from portage.elog import elog_process - from portage.elog.filtering import filter_mergephases + mergecount = pkg_count.curval pkgsettings = self.pkgsettings[pkg.root] buildpkgonly = "--buildpkgonly" in self.myopts fetch_all = "--fetch-all-uri" in self.myopts @@ -7035,6 +7202,7 @@ class Scheduler(object): xterm_titles = "notitles" not in self.settings.features x = pkg + y = None root_config = pkg.root_config system_set = root_config.sets["system"] args_set = InternalPackageSet(favorites) @@ -7056,7 +7224,7 @@ class Scheduler(object): if x[0]=="blocks": pkgindex=3 - y = portdb.findname(pkg_key) + if "--pretend" not in self.myopts: print "\n>>> Emerging (" + \ colorize("MERGE_LIST_PROGRESS", str(mergecount)) + " of " + \ @@ -7066,108 +7234,18 @@ class Scheduler(object): str(mergecount)+" of "+str(len(mymergelist))+\ ") "+x[pkgindex]+" to "+x[1]) - pkgsettings["EMERGE_FROM"] = x[0] - pkgsettings.backup_changes("EMERGE_FROM") - pkgsettings.reset() - - #buildsyspkg: Check if we need to _force_ binary package creation - issyspkg = ("buildsyspkg" in myfeat) \ - and x[0] != "blocks" \ - and system_set.findAtomForPackage(pkg) \ - and "--buildpkg" not in self.myopts - if x[0] in ["ebuild","blocks"]: - if x[0] == "blocks" and "--fetchonly" not in self.myopts: - raise Exception, "Merging a blocker" - elif fetchonly: - fetcher = EbuildFetcher(fetch_all=fetch_all, - pkg=pkg, pretend=pretend, settings=pkgsettings) - retval = fetcher.execute() - if (retval is None) or retval: - print - print "!!! Fetch for",y,"failed, continuing..." - print - failed_fetches.append(pkg_key) - self.curval += 1 - return - - build_dir = EbuildBuildDir(pkg=pkg, settings=pkgsettings) - try: - build_dir.lock() - # Cleaning is triggered before the setup - # phase, in portage.doebuild(). - msg = " === (%s of %s) Cleaning (%s::%s)" % \ - (mergecount, len(mymergelist), pkg_key, y) - short_msg = "emerge: (%s of %s) %s Clean" % \ - (mergecount, len(mymergelist), pkg_key) - emergelog(xterm_titles, msg, short_msg=short_msg) - - if "--buildpkg" in self.myopts or issyspkg: - if issyspkg: - print ">>> This is a system package, " + \ - "let's pack a rescue tarball." - msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ - (mergecount, len(mymergelist), pkg_key, y) - short_msg = "emerge: (%s of %s) %s Compile" % \ - (mergecount, len(mymergelist), pkg_key) - emergelog(xterm_titles, msg, short_msg=short_msg) - - build = EbuildBuild(pkg=pkg, register=self._register, - schedule=self._schedule, settings=pkgsettings, - unregister=self._unregister) - retval = build.execute() - if retval != os.EX_OK: - raise self._pkg_failure(retval) - - build = EbuildBinpkg(pkg=pkg, settings=pkgsettings) - retval = build.execute() - if retval != os.EX_OK: - raise self._pkg_failure(retval) - - if "--buildpkgonly" not in self.myopts: - msg = " === (%s of %s) Merging (%s::%s)" % \ - (mergecount, len(mymergelist), pkg_key, y) - short_msg = "emerge: (%s of %s) %s Merge" % \ - (mergecount, len(mymergelist), pkg_key) - emergelog(xterm_titles, msg, short_msg=short_msg) - - merge = EbuildMerge( - find_blockers=self._find_blockers(pkg), - ldpath_mtimes=ldpath_mtimes, - pkg=pkg, pretend=pretend, settings=pkgsettings) - retval = merge.execute() - if retval != os.EX_OK: - raise self._pkg_failure(retval) - elif "noclean" not in pkgsettings.features: - portage.doebuild(y, "clean", myroot, - pkgsettings, self.edebug, mydbapi=portdb, - tree="porttree") - else: - msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ - (mergecount, len(mymergelist), pkg_key, y) - short_msg = "emerge: (%s of %s) %s Compile" % \ - (mergecount, len(mymergelist), pkg_key) - emergelog(xterm_titles, msg, short_msg=short_msg) + self._schedule() - build = EbuildBuild(pkg=pkg, register=self._register, - schedule=self._schedule, settings=pkgsettings, - unregister=self._unregister) - retval = build.execute() - if retval != os.EX_OK: - raise self._pkg_failure(retval) - - merge = EbuildMerge( - find_blockers=self._find_blockers(pkg), - ldpath_mtimes=ldpath_mtimes, - pkg=pkg, pretend=pretend, settings=pkgsettings) - retval = merge.execute() - - if retval != os.EX_OK: - raise self._pkg_failure(retval) - finally: - if build_dir.locked: - elog_process(pkg.cpv, pkgsettings, - phasefilter=filter_mergephases) - build_dir.unlock() + if x.type_name == "ebuild": + y = portdb.findname(pkg.cpv) + build = EbuildBuild(args_set=self._args_set, + find_blockers=self._find_blockers(pkg), + ldpath_mtimes=ldpath_mtimes, logger=self._logger, + opts=self._build_opts, pkg=pkg, pkg_count=pkg_count, + settings=pkgsettings, scheduler=self._sched_iface) + retval = build.execute() + if retval != os.EX_OK: + raise self._pkg_failure(retval) elif x.type_name == "binary": # The prefetcher have already completed or it @@ -7182,17 +7260,22 @@ class Scheduler(object): prefetcher = prefetchers.get(pkg) if prefetcher is not None: if not prefetcher.isAlive(): + writemsg(">>> prefetcher not alive, cancelling\n") prefetcher.cancel() else: + writemsg(">>> prefetcher alive, waiting\n") retval = None while retval is None: self._schedule() retval = prefetcher.poll() del prefetcher + else: + writemsg(">>> prefetcher does not exist\n") fetcher = BinpkgFetcher(pkg=pkg, pretend=pretend, use_locks=("distlocks" in pkgsettings.features)) mytbz2 = fetcher.pkg_path + y = mytbz2 if "--getbinpkg" in self.myopts: retval = fetcher.execute() if fetcher.remote: @@ -7280,7 +7363,7 @@ class Scheduler(object): (mergecount, len(mymergelist), x[pkgindex]) emergelog(xterm_titles, (" === (%s of %s) " + \ "Post-Build Cleaning (%s::%s)") % \ - (mergecount, len(mymergelist), x[pkgindex], y), + (mergecount, len(mymergelist), pkg.cpv, y), short_msg=short_msg) emergelog(xterm_titles, " ::: completed emerge ("+\ str(mergecount)+" of "+str(len(mymergelist))+") "+\ -- cgit v1.2.3-1-g7c22