summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorZac Medico <zmedico@gentoo.org>2008-03-14 02:41:31 +0000
committerZac Medico <zmedico@gentoo.org>2008-03-14 02:41:31 +0000
commit57591010d41c37479dbc43acacfaa21d9a906bd8 (patch)
treea72d3c26380353078212c4274ee51a1ed3a58481
parentba85ab2fc704f1617c7b60d4f3e454968b705376 (diff)
downloadportage-57591010d41c37479dbc43acacfaa21d9a906bd8.tar.gz
portage-57591010d41c37479dbc43acacfaa21d9a906bd8.tar.bz2
portage-57591010d41c37479dbc43acacfaa21d9a906bd8.zip
* Optimize parallel-fetch to avoid redundant checksum verification.
* Add parallel-fetch to the default FEATURES since it is more efficient now. svn path=/main/trunk/; revision=9462
-rw-r--r--cnf/make.globals2
-rw-r--r--pym/_emerge/__init__.py3
-rw-r--r--pym/portage/__init__.py24
3 files changed, 26 insertions, 3 deletions
diff --git a/cnf/make.globals b/cnf/make.globals
index d36ede9a3..9023dab6b 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -33,7 +33,7 @@ FETCHCOMMAND="/usr/bin/wget -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\"
RESUMECOMMAND="/usr/bin/wget -c -t 5 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\""
# Default user options
-FEATURES="distlocks metadata-transfer sandbox sfperms strict unmerge-orphans userfetch"
+FEATURES="distlocks parallel-fetch metadata-transfer sandbox sfperms strict unmerge-orphans userfetch"
# Enable for global testing
FEATURES="${FEATURES} preserve-libs"
diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index 68bf0d973..7a619271d 100644
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -4489,6 +4489,7 @@ class MergeTask(object):
fetch_env = os.environ.copy()
fetch_env["FEATURES"] = fetch_env.get("FEATURES", "") + " -cvs"
fetch_env["PORTAGE_NICENESS"] = "0"
+ fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1"
fetch_args = [sys.argv[0], "--resume", "--fetchonly"]
resume_opts = self.myopts.copy()
# For automatic resume, we need to prevent
@@ -6920,7 +6921,7 @@ def action_build(settings, trees, mtimedb,
if ("--resume" in myopts):
favorites=mtimedb["resume"]["favorites"]
mergetask = MergeTask(settings, trees, myopts)
- if "--fetchonly" in myopts:
+ if "PORTAGE_PARALLEL_FETCHONLY" in settings:
""" parallel-fetch uses --resume --fetchonly and we don't want
it to write the mtimedb"""
mtimedb.filename = None
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 04f7acec0..8e146c45b 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -3032,6 +3032,14 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
checksum_failure_primaryuri = 2
thirdpartymirrors = mysettings.thirdpartymirrors()
+ # In the background parallel-fetch process, it's safe to skip checksum
+ # verification of pre-existing files in $DISTDIR that have the correct
+ # file size. The parent process will verify their checksums prior to
+ # the unpack phase.
+
+ parallel_fetchonly = fetchonly and \
+ "PORTAGE_PARALLEL_FETCHONLY" in mysettings
+
check_config_instance(mysettings)
custommirrors = grabdict(os.path.join(mysettings["PORTAGE_CONFIGROOT"],
@@ -3218,7 +3226,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
if use_locks and can_fetch:
waiting_msg = None
- if "parallel-fetch" in features:
+ if not parallel_fetchonly and "parallel-fetch" in features:
waiting_msg = ("Downloading '%s'... " + \
"see /var/log/emerge-fetch.log for details.") % myfile
if locks_in_subdir:
@@ -3277,6 +3285,15 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
if mystat.st_size < mydigests[myfile]["size"] and \
not restrict_fetch:
fetched = 1 # Try to resume this download.
+ elif parallel_fetchonly and \
+ mystat.st_size == mydigests[myfile]["size"]:
+ eout = portage.output.EOutput()
+ eout.quiet = \
+ mysettings.get("PORTAGE_QUIET") == "1"
+ eout.ebegin(
+ "%s size ;-)" % (myfile, ))
+ eout.eend(0)
+ continue
else:
verified_ok, reason = portage.checksum.verify_all(
myfile_path, mydigests[myfile])
@@ -4737,6 +4754,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
fetchme = newuris[:]
checkme = alist[:]
+ if mydo == "fetch":
+ # Files are already checked inside fetch(),
+ # so do not check them again.
+ checkme = []
+
# Only try and fetch the files if we are going to need them ...
# otherwise, if user has FEATURES=noauto and they run `ebuild clean
# unpack compile install`, we will try and fetch 4 times :/