EbuildBuild: skip the fetch queue when possible

Since commit f07f8386e945b48358c11c121960e4833c539752, it was possible
for EbuildBuild to wait on the fetch queue even in cases in which all
required files had been previously fetched. Now this case is optimized
to skip the fetch queue, as discribed in bug #375331, comment #2.

2 files changed, 129 insertions, 16 deletions

diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py
index b6beb495b..1776d1eac 100644
--- a/pym/_emerge/EbuildBuild.py
+++ b/pym/_emerge/EbuildBuild.py
@@ -166,12 +166,34 @@ class EbuildBuild(CompositeTask):
 		portage.prepare_build_dirs(self.pkg.root, self.settings, 1)
 
 		fetcher = EbuildFetcher(config_pool=self.config_pool,
+			ebuild_path=self._ebuild_path,
 			fetchall=self.opts.fetch_all_uri,
 			fetchonly=self.opts.fetchonly,
 			background=self.background,
 			logfile=self.settings.get('PORTAGE_LOG_FILE'),
 			pkg=self.pkg, scheduler=self.scheduler)
 
+		try:
+			already_fetched = fetcher.already_fetched(self.settings)
+		except portage.exception.InvalidDependString as e:
+			msg_lines = []
+			msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \
+				(self.pkg.cpv, e)
+			msg_lines.append(msg)
+			fetcher._eerror(msg_lines)
+			portage.elog.elog_process(self.pkg.cpv, self.settings)
+			self.returncode = 1
+			self._current_task = None
+			self._unlock_builddir()
+			self.wait()
+			return
+
+		if already_fetched:
+			# This case is optimized to skip the fetch queue.
+			fetcher = None
+			self._fetch_exit(fetcher)
+			return
+
 		# Allow the Scheduler's fetch queue to control the
 		# number of concurrent fetchers.
 		fetcher.addExitListener(self._fetch_exit)
@@ -180,7 +202,8 @@ class EbuildBuild(CompositeTask):
 
 	def _fetch_exit(self, fetcher):
 
-		if self._default_exit(fetcher) != os.EX_OK:
+		if fetcher is not None and \
+			self._default_exit(fetcher) != os.EX_OK:
 			self._fetch_failed()
 			return
 
diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py
index c076288cc..066678748 100644
--- a/pym/_emerge/EbuildFetcher.py
+++ b/pym/_emerge/EbuildFetcher.py
@@ -14,24 +14,97 @@ from portage import _encodings
 from portage import _unicode_encode
 from portage import _unicode_decode
 from portage.elog.messages import eerror
-from portage.package.ebuild.fetch import fetch
+from portage.package.ebuild.fetch import _check_distfile, fetch
 from portage.util._pty import _create_pty_or_pipe
 
 class EbuildFetcher(SpawnProcess):
 
-	__slots__ = ("config_pool", "fetchonly", "fetchall", "pkg", "prefetch") + \
+	__slots__ = ("config_pool", "ebuild_path", "fetchonly", "fetchall",
+		"pkg", "prefetch") + \
 		("_digests", "_settings", "_uri_map")
 
+	def already_fetched(self, settings):
+		"""
+		Returns True if all files are already exist locally and have correct
+		digests, otherwise return False. When returning True, appropriate
+		digest checking messages are produced for display and/or logging.
+		When returning False, no messages are produced, since we assume
+		that a fetcher process will later be executed in order to produce
+		such messages. This will raise InvalidDependString if SRC_URI is
+		invalid.
+		"""
+
+		uri_map = self._get_uri_map()
+		if not uri_map:
+			return True
+
+		digests = self._get_digests()
+		distdir = settings["DISTDIR"]
+		allow_missing = "allow-missing-manifests" in settings.features
+
+		for filename in uri_map:
+			# Use stat rather than lstat since fetch() creates
+			# symlinks when PORTAGE_RO_DISTDIRS is used.
+			try:
+				st = os.stat(os.path.join(distdir, filename))
+			except OSError:
+				return False
+			if st.st_size == 0:
+				return False
+			expected_size = digests.get(filename, {}).get('size')
+			if expected_size is None:
+				continue
+			if st.st_size != expected_size:
+				return False
+
+		stdout_orig = sys.stdout
+		stderr_orig = sys.stderr
+		global_havecolor = portage.output.havecolor
+		out = io.StringIO()
+		eout = portage.output.EOutput()
+		eout.quiet = settings.get("PORTAGE_QUIET") == "1"
+		success = True
+		try:
+			sys.stdout = out
+			sys.stderr = out
+			if portage.output.havecolor:
+				portage.output.havecolor = not self.background
+
+			for filename in uri_map:
+				mydigests = digests.get(filename)
+				if mydigests is None:
+					if not allow_missing:
+						success = False
+						break
+					continue
+				ok, st = _check_distfile(os.path.join(distdir, filename),
+					mydigests, eout, show_errors=False)
+				if not ok:
+					success = False
+					break
+		finally:
+			sys.stdout = stdout_orig
+			sys.stderr = stderr_orig
+			portage.output.havecolor = global_havecolor
+
+		if success:
+			# When returning uncessessfully, no messages are produced, since
+			# we assume that a fetcher process will later be executed in order
+			# to produce such messages.
+			msg = out.getvalue()
+			if msg:
+				self.scheduler.output(msg, log_path=self.logfile)
+
+		return success
+
 	def _start(self):
 
 		root_config = self.pkg.root_config
 		portdb = root_config.trees["porttree"].dbapi
-		ebuild_path = portdb.findname(self.pkg.cpv, myrepo=self.pkg.repo)
-		if ebuild_path is None:
-			raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv)
+		ebuild_path = self._get_ebuild_path()
 
 		try:
-			uri_map = self._get_uri_map(portdb, ebuild_path)
+			uri_map = self._get_uri_map()
 		except portage.exception.InvalidDependString as e:
 			msg_lines = []
 			msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \
@@ -48,9 +121,6 @@ class EbuildFetcher(SpawnProcess):
 			self.wait()
 			return
 
-		self._digests = portage.Manifest(
-			os.path.dirname(ebuild_path), None).getTypeDigests("DIST")
-
 		settings = self.config_pool.allocate()
 		settings.setcpv(self.pkg)
 		portage.doebuild_environment(ebuild_path, 'fetch',
@@ -75,7 +145,6 @@ class EbuildFetcher(SpawnProcess):
 			settings["NOCOLOR"] = nocolor
 
 		self._settings = settings
-		self._uri_map = uri_map
 		SpawnProcess._start(self)
 
 		# Free settings now since it's no longer needed in
@@ -110,7 +179,7 @@ class EbuildFetcher(SpawnProcess):
 		allow_missing = 'allow-missing-manifests' in self._settings.features
 		try:
 			if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly,
-				digests=copy.deepcopy(self._digests),
+				digests=copy.deepcopy(self._get_digests()),
 				allow_missing_digests=allow_missing):
 				rval = os.EX_OK
 		except SystemExit:
@@ -122,16 +191,37 @@ class EbuildFetcher(SpawnProcess):
 			# finally blocks from earlier in the call stack. See bug #345289.
 			os._exit(rval)
 
-	def _get_uri_map(self, portdb, ebuild_path):
+	def _get_ebuild_path(self):
+		if self.ebuild_path is not None:
+			return self.ebuild_path
+		portdb = self.pkg.root_config.trees["porttree"].dbapi
+		self.ebuild_path = portdb.findname(self.pkg.cpv, myrepo=self.pkg.repo)
+		if self.ebuild_path is None:
+			raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv)
+		return self.ebuild_path
+
+	def _get_digests(self):
+		if self._digests is not None:
+			return self._digests
+		self._digests = portage.Manifest(os.path.dirname(
+			self._get_ebuild_path()), None).getTypeDigests("DIST")
+		return self._digests
+
+	def _get_uri_map(self):
 		"""
 		This can raise InvalidDependString from portdbapi.getFetchMap().
 		"""
-		pkgdir = os.path.dirname(ebuild_path)
+		if self._uri_map is not None:
+			return self._uri_map
+		pkgdir = os.path.dirname(self._get_ebuild_path())
 		mytree = os.path.dirname(os.path.dirname(pkgdir))
 		use = None
 		if not self.fetchall:
 			use = self.pkg.use.enabled
-		return portdb.getFetchMap(self.pkg.cpv, useflags=use, mytree=mytree)
+		portdb = self.pkg.root_config.trees["porttree"].dbapi
+		self._uri_map = portdb.getFetchMap(self.pkg.cpv,
+			useflags=use, mytree=mytree)
+		return self._uri_map
 
 	def _prefetch_size_ok(self, uri_map, settings, ebuild_path):
 		distdir = settings["DISTDIR"]
@@ -148,7 +238,7 @@ class EbuildFetcher(SpawnProcess):
 				return False
 			sizes[filename] = st.st_size
 
-		digests = self._digests
+		digests = self._get_digests()
 		for filename, actual_size in sizes.items():
 			size = digests.get(filename, {}).get('size')
 			if size is None: