diff options
| -rw-r--r-- | pym/_emerge/EbuildBuild.py | 25 | ||||
| -rw-r--r-- | pym/_emerge/EbuildFetcher.py | 120 |
2 files changed, 129 insertions, 16 deletions
diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index b6beb495b..1776d1eac 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -166,12 +166,34 @@ class EbuildBuild(CompositeTask): portage.prepare_build_dirs(self.pkg.root, self.settings, 1) fetcher = EbuildFetcher(config_pool=self.config_pool, + ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=self.background, logfile=self.settings.get('PORTAGE_LOG_FILE'), pkg=self.pkg, scheduler=self.scheduler) + try: + already_fetched = fetcher.already_fetched(self.settings) + except portage.exception.InvalidDependString as e: + msg_lines = [] + msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ + (self.pkg.cpv, e) + msg_lines.append(msg) + fetcher._eerror(msg_lines) + portage.elog.elog_process(self.pkg.cpv, self.settings) + self.returncode = 1 + self._current_task = None + self._unlock_builddir() + self.wait() + return + + if already_fetched: + # This case is optimized to skip the fetch queue. + fetcher = None + self._fetch_exit(fetcher) + return + # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetch_exit) @@ -180,7 +202,8 @@ class EbuildBuild(CompositeTask): def _fetch_exit(self, fetcher): - if self._default_exit(fetcher) != os.EX_OK: + if fetcher is not None and \ + self._default_exit(fetcher) != os.EX_OK: self._fetch_failed() return diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py index c076288cc..066678748 100644 --- a/pym/_emerge/EbuildFetcher.py +++ b/pym/_emerge/EbuildFetcher.py @@ -14,24 +14,97 @@ from portage import _encodings from portage import _unicode_encode from portage import _unicode_decode from portage.elog.messages import eerror -from portage.package.ebuild.fetch import fetch +from portage.package.ebuild.fetch import _check_distfile, fetch from portage.util._pty import _create_pty_or_pipe class EbuildFetcher(SpawnProcess): - __slots__ = ("config_pool", "fetchonly", "fetchall", "pkg", "prefetch") + \ + __slots__ = ("config_pool", "ebuild_path", "fetchonly", "fetchall", + "pkg", "prefetch") + \ ("_digests", "_settings", "_uri_map") + def already_fetched(self, settings): + """ + Returns True if all files are already exist locally and have correct + digests, otherwise return False. When returning True, appropriate + digest checking messages are produced for display and/or logging. + When returning False, no messages are produced, since we assume + that a fetcher process will later be executed in order to produce + such messages. This will raise InvalidDependString if SRC_URI is + invalid. + """ + + uri_map = self._get_uri_map() + if not uri_map: + return True + + digests = self._get_digests() + distdir = settings["DISTDIR"] + allow_missing = "allow-missing-manifests" in settings.features + + for filename in uri_map: + # Use stat rather than lstat since fetch() creates + # symlinks when PORTAGE_RO_DISTDIRS is used. + try: + st = os.stat(os.path.join(distdir, filename)) + except OSError: + return False + if st.st_size == 0: + return False + expected_size = digests.get(filename, {}).get('size') + if expected_size is None: + continue + if st.st_size != expected_size: + return False + + stdout_orig = sys.stdout + stderr_orig = sys.stderr + global_havecolor = portage.output.havecolor + out = io.StringIO() + eout = portage.output.EOutput() + eout.quiet = settings.get("PORTAGE_QUIET") == "1" + success = True + try: + sys.stdout = out + sys.stderr = out + if portage.output.havecolor: + portage.output.havecolor = not self.background + + for filename in uri_map: + mydigests = digests.get(filename) + if mydigests is None: + if not allow_missing: + success = False + break + continue + ok, st = _check_distfile(os.path.join(distdir, filename), + mydigests, eout, show_errors=False) + if not ok: + success = False + break + finally: + sys.stdout = stdout_orig + sys.stderr = stderr_orig + portage.output.havecolor = global_havecolor + + if success: + # When returning uncessessfully, no messages are produced, since + # we assume that a fetcher process will later be executed in order + # to produce such messages. + msg = out.getvalue() + if msg: + self.scheduler.output(msg, log_path=self.logfile) + + return success + def _start(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi - ebuild_path = portdb.findname(self.pkg.cpv, myrepo=self.pkg.repo) - if ebuild_path is None: - raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) + ebuild_path = self._get_ebuild_path() try: - uri_map = self._get_uri_map(portdb, ebuild_path) + uri_map = self._get_uri_map() except portage.exception.InvalidDependString as e: msg_lines = [] msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ @@ -48,9 +121,6 @@ class EbuildFetcher(SpawnProcess): self.wait() return - self._digests = portage.Manifest( - os.path.dirname(ebuild_path), None).getTypeDigests("DIST") - settings = self.config_pool.allocate() settings.setcpv(self.pkg) portage.doebuild_environment(ebuild_path, 'fetch', @@ -75,7 +145,6 @@ class EbuildFetcher(SpawnProcess): settings["NOCOLOR"] = nocolor self._settings = settings - self._uri_map = uri_map SpawnProcess._start(self) # Free settings now since it's no longer needed in @@ -110,7 +179,7 @@ class EbuildFetcher(SpawnProcess): allow_missing = 'allow-missing-manifests' in self._settings.features try: if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, - digests=copy.deepcopy(self._digests), + digests=copy.deepcopy(self._get_digests()), allow_missing_digests=allow_missing): rval = os.EX_OK except SystemExit: @@ -122,16 +191,37 @@ class EbuildFetcher(SpawnProcess): # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval) - def _get_uri_map(self, portdb, ebuild_path): + def _get_ebuild_path(self): + if self.ebuild_path is not None: + return self.ebuild_path + portdb = self.pkg.root_config.trees["porttree"].dbapi + self.ebuild_path = portdb.findname(self.pkg.cpv, myrepo=self.pkg.repo) + if self.ebuild_path is None: + raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) + return self.ebuild_path + + def _get_digests(self): + if self._digests is not None: + return self._digests + self._digests = portage.Manifest(os.path.dirname( + self._get_ebuild_path()), None).getTypeDigests("DIST") + return self._digests + + def _get_uri_map(self): """ This can raise InvalidDependString from portdbapi.getFetchMap(). """ - pkgdir = os.path.dirname(ebuild_path) + if self._uri_map is not None: + return self._uri_map + pkgdir = os.path.dirname(self._get_ebuild_path()) mytree = os.path.dirname(os.path.dirname(pkgdir)) use = None if not self.fetchall: use = self.pkg.use.enabled - return portdb.getFetchMap(self.pkg.cpv, useflags=use, mytree=mytree) + portdb = self.pkg.root_config.trees["porttree"].dbapi + self._uri_map = portdb.getFetchMap(self.pkg.cpv, + useflags=use, mytree=mytree) + return self._uri_map def _prefetch_size_ok(self, uri_map, settings, ebuild_path): distdir = settings["DISTDIR"] @@ -148,7 +238,7 @@ class EbuildFetcher(SpawnProcess): return False sizes[filename] = st.st_size - digests = self._digests + digests = self._get_digests() for filename, actual_size in sizes.items(): size = digests.get(filename, {}).get('size') if size is None: |