summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorZac Medico <zmedico@gentoo.org>2011-07-16 16:30:14 -0700
committerZac Medico <zmedico@gentoo.org>2011-07-16 16:30:14 -0700
commit785fe0b42a66b1d137955a4b816b3c254a1b785d (patch)
tree176c9c8d8a185994671276f5d3746db9c51db43f
parenta3ebf8f370ed20bcf419395b786aae2daccb5074 (diff)
downloadportage-785fe0b42a66b1d137955a4b816b3c254a1b785d.tar.gz
portage-785fe0b42a66b1d137955a4b816b3c254a1b785d.tar.bz2
portage-785fe0b42a66b1d137955a4b816b3c254a1b785d.zip
EbuildBuild: skip the fetch queue when possible
Since commit f07f8386e945b48358c11c121960e4833c539752, it was possible for EbuildBuild to wait on the fetch queue even in cases in which all required files had been previously fetched. Now this case is optimized to skip the fetch queue, as discribed in bug #375331, comment #2.
-rw-r--r--pym/_emerge/EbuildBuild.py25
-rw-r--r--pym/_emerge/EbuildFetcher.py120
2 files changed, 129 insertions, 16 deletions
diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py
index b6beb495b..1776d1eac 100644
--- a/pym/_emerge/EbuildBuild.py
+++ b/pym/_emerge/EbuildBuild.py
@@ -166,12 +166,34 @@ class EbuildBuild(CompositeTask):
portage.prepare_build_dirs(self.pkg.root, self.settings, 1)
fetcher = EbuildFetcher(config_pool=self.config_pool,
+ ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=self.background,
logfile=self.settings.get('PORTAGE_LOG_FILE'),
pkg=self.pkg, scheduler=self.scheduler)
+ try:
+ already_fetched = fetcher.already_fetched(self.settings)
+ except portage.exception.InvalidDependString as e:
+ msg_lines = []
+ msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \
+ (self.pkg.cpv, e)
+ msg_lines.append(msg)
+ fetcher._eerror(msg_lines)
+ portage.elog.elog_process(self.pkg.cpv, self.settings)
+ self.returncode = 1
+ self._current_task = None
+ self._unlock_builddir()
+ self.wait()
+ return
+
+ if already_fetched:
+ # This case is optimized to skip the fetch queue.
+ fetcher = None
+ self._fetch_exit(fetcher)
+ return
+
# Allow the Scheduler's fetch queue to control the
# number of concurrent fetchers.
fetcher.addExitListener(self._fetch_exit)
@@ -180,7 +202,8 @@ class EbuildBuild(CompositeTask):
def _fetch_exit(self, fetcher):
- if self._default_exit(fetcher) != os.EX_OK:
+ if fetcher is not None and \
+ self._default_exit(fetcher) != os.EX_OK:
self._fetch_failed()
return
diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py
index c076288cc..066678748 100644
--- a/pym/_emerge/EbuildFetcher.py
+++ b/pym/_emerge/EbuildFetcher.py
@@ -14,24 +14,97 @@ from portage import _encodings
from portage import _unicode_encode
from portage import _unicode_decode
from portage.elog.messages import eerror
-from portage.package.ebuild.fetch import fetch
+from portage.package.ebuild.fetch import _check_distfile, fetch
from portage.util._pty import _create_pty_or_pipe
class EbuildFetcher(SpawnProcess):
- __slots__ = ("config_pool", "fetchonly", "fetchall", "pkg", "prefetch") + \
+ __slots__ = ("config_pool", "ebuild_path", "fetchonly", "fetchall",
+ "pkg", "prefetch") + \
("_digests", "_settings", "_uri_map")
+ def already_fetched(self, settings):
+ """
+ Returns True if all files are already exist locally and have correct
+ digests, otherwise return False. When returning True, appropriate
+ digest checking messages are produced for display and/or logging.
+ When returning False, no messages are produced, since we assume
+ that a fetcher process will later be executed in order to produce
+ such messages. This will raise InvalidDependString if SRC_URI is
+ invalid.
+ """
+
+ uri_map = self._get_uri_map()
+ if not uri_map:
+ return True
+
+ digests = self._get_digests()
+ distdir = settings["DISTDIR"]
+ allow_missing = "allow-missing-manifests" in settings.features
+
+ for filename in uri_map:
+ # Use stat rather than lstat since fetch() creates
+ # symlinks when PORTAGE_RO_DISTDIRS is used.
+ try:
+ st = os.stat(os.path.join(distdir, filename))
+ except OSError:
+ return False
+ if st.st_size == 0:
+ return False
+ expected_size = digests.get(filename, {}).get('size')
+ if expected_size is None:
+ continue
+ if st.st_size != expected_size:
+ return False
+
+ stdout_orig = sys.stdout
+ stderr_orig = sys.stderr
+ global_havecolor = portage.output.havecolor
+ out = io.StringIO()
+ eout = portage.output.EOutput()
+ eout.quiet = settings.get("PORTAGE_QUIET") == "1"
+ success = True
+ try:
+ sys.stdout = out
+ sys.stderr = out
+ if portage.output.havecolor:
+ portage.output.havecolor = not self.background
+
+ for filename in uri_map:
+ mydigests = digests.get(filename)
+ if mydigests is None:
+ if not allow_missing:
+ success = False
+ break
+ continue
+ ok, st = _check_distfile(os.path.join(distdir, filename),
+ mydigests, eout, show_errors=False)
+ if not ok:
+ success = False
+ break
+ finally:
+ sys.stdout = stdout_orig
+ sys.stderr = stderr_orig
+ portage.output.havecolor = global_havecolor
+
+ if success:
+ # When returning uncessessfully, no messages are produced, since
+ # we assume that a fetcher process will later be executed in order
+ # to produce such messages.
+ msg = out.getvalue()
+ if msg:
+ self.scheduler.output(msg, log_path=self.logfile)
+
+ return success
+
def _start(self):
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
- ebuild_path = portdb.findname(self.pkg.cpv, myrepo=self.pkg.repo)
- if ebuild_path is None:
- raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv)
+ ebuild_path = self._get_ebuild_path()
try:
- uri_map = self._get_uri_map(portdb, ebuild_path)
+ uri_map = self._get_uri_map()
except portage.exception.InvalidDependString as e:
msg_lines = []
msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \
@@ -48,9 +121,6 @@ class EbuildFetcher(SpawnProcess):
self.wait()
return
- self._digests = portage.Manifest(
- os.path.dirname(ebuild_path), None).getTypeDigests("DIST")
-
settings = self.config_pool.allocate()
settings.setcpv(self.pkg)
portage.doebuild_environment(ebuild_path, 'fetch',
@@ -75,7 +145,6 @@ class EbuildFetcher(SpawnProcess):
settings["NOCOLOR"] = nocolor
self._settings = settings
- self._uri_map = uri_map
SpawnProcess._start(self)
# Free settings now since it's no longer needed in
@@ -110,7 +179,7 @@ class EbuildFetcher(SpawnProcess):
allow_missing = 'allow-missing-manifests' in self._settings.features
try:
if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly,
- digests=copy.deepcopy(self._digests),
+ digests=copy.deepcopy(self._get_digests()),
allow_missing_digests=allow_missing):
rval = os.EX_OK
except SystemExit:
@@ -122,16 +191,37 @@ class EbuildFetcher(SpawnProcess):
# finally blocks from earlier in the call stack. See bug #345289.
os._exit(rval)
- def _get_uri_map(self, portdb, ebuild_path):
+ def _get_ebuild_path(self):
+ if self.ebuild_path is not None:
+ return self.ebuild_path
+ portdb = self.pkg.root_config.trees["porttree"].dbapi
+ self.ebuild_path = portdb.findname(self.pkg.cpv, myrepo=self.pkg.repo)
+ if self.ebuild_path is None:
+ raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv)
+ return self.ebuild_path
+
+ def _get_digests(self):
+ if self._digests is not None:
+ return self._digests
+ self._digests = portage.Manifest(os.path.dirname(
+ self._get_ebuild_path()), None).getTypeDigests("DIST")
+ return self._digests
+
+ def _get_uri_map(self):
"""
This can raise InvalidDependString from portdbapi.getFetchMap().
"""
- pkgdir = os.path.dirname(ebuild_path)
+ if self._uri_map is not None:
+ return self._uri_map
+ pkgdir = os.path.dirname(self._get_ebuild_path())
mytree = os.path.dirname(os.path.dirname(pkgdir))
use = None
if not self.fetchall:
use = self.pkg.use.enabled
- return portdb.getFetchMap(self.pkg.cpv, useflags=use, mytree=mytree)
+ portdb = self.pkg.root_config.trees["porttree"].dbapi
+ self._uri_map = portdb.getFetchMap(self.pkg.cpv,
+ useflags=use, mytree=mytree)
+ return self._uri_map
def _prefetch_size_ok(self, uri_map, settings, ebuild_path):
distdir = settings["DISTDIR"]
@@ -148,7 +238,7 @@ class EbuildFetcher(SpawnProcess):
return False
sizes[filename] = st.st_size
- digests = self._digests
+ digests = self._get_digests()
for filename, actual_size in sizes.items():
size = digests.get(filename, {}).get('size')
if size is None: