diff options
| -rw-r--r-- | cnf/make.globals | 3 | ||||
| -rw-r--r-- | man/make.conf.5 | 8 | ||||
| -rw-r--r-- | pym/portage/__init__.py | 60 |
3 files changed, 68 insertions, 3 deletions
diff --git a/cnf/make.globals b/cnf/make.globals index 36d8591ae..5e8150131 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -59,6 +59,9 @@ AUTOCLEAN="yes" # Number of mirrors to try when a downloaded file has an incorrect checksum. PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5" +# Minimum size of existing file for RESUMECOMMAND to be called. +PORTAGE_FETCH_RESUME_MIN_SIZE="350K" + # Number of times 'emerge --sync' will run before giving up. PORTAGE_RSYNC_RETRIES="3" diff --git a/man/make.conf.5 b/man/make.conf.5 index 18d750a21..d35acf16b 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -412,6 +412,14 @@ Please see /etc/make.conf.example for elog documentation. \fBPORTAGE_FETCH_CHECKSUM_TRY_MIRRORS\fR = \fI5\fR Number of mirrors to try when a downloaded file has an incorrect checksum. .TP +\PORTAGE_FETCH_RESUME_MIN_SIZE\fR = \fI350K\fR +Minimum size of existing file for RESUMECOMMAND to be called. Files smaller +than this size will be removed and FETCHCOMMAND will be called to download +the file from the beginning. This is useful for helping to ensure that small +garbage files such as html 404 pages are properly discarded. The variable +should contain an integer number of bytes and may have a suffix such as +K, M, or G. +.TP \fBPORTAGE_NICENESS\fR = \fI[number]\fR The value of this variable will be added to the current nice level that emerge is running at. In other words, this will not set the nice level, diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index a64e90c5c..4233c2eea 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -938,7 +938,8 @@ class config(object): "PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", - "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_GPG_DIR", + "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE", + "PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", "PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR", @@ -3010,6 +3011,20 @@ def _checksum_failure_temp_file(distdir, basename): os.rename(filename, temp_filename) return temp_filename +_fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)') + +_size_suffix_map = { + '' : 0, + 'K' : 10, + 'M' : 20, + 'G' : 30, + 'T' : 40, + 'P' : 50, + 'E' : 60, + 'Z' : 70, + 'Y' : 80, +} + def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1): "fetch files. Will use digest file if available." @@ -3050,6 +3065,26 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", checksum_failure_max_tries = v del v + fetch_resume_size_default = "350K" + fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE") + if fetch_resume_size is not None: + fetch_resume_size = "".join(fetch_resume_size.split()) + match = _fetch_resume_size_re.match(fetch_resume_size) + if match is None or \ + (match.group(2).upper() not in _size_suffix_map): + writemsg("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE" + \ + " contains an unrecognized format: '%s'\n" % \ + mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1) + writemsg("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE " + \ + "default value: %s\n" % fetch_resume_size_default, + noiselevel=-1) + fetch_resume_size = None + if fetch_resume_size is None: + fetch_resume_size = fetch_resume_size_default + match = _fetch_resume_size_re.match(fetch_resume_size) + fetch_resume_size = int(match.group(1)) * \ + 2 ** _size_suffix_map[match.group(2)] + # Behave like the package has RESTRICT="primaryuri" after a # couple of checksum failures, to increase the probablility # of success before checksum_failure_max_tries is reached. @@ -3405,8 +3440,27 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if fetched != 2 and has_space: #we either need to resume or start the download - #you can't use "continue" when you're inside a "try" block - if fetched==1: + if fetched == 1: + try: + mystat = os.stat(myfile_path) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + fetched = 0 + else: + if mystat.st_size < fetch_resume_size: + writemsg((">>> Deleting distfile with size " + \ + "%d (smaller than " "PORTAGE_FETCH_RESU" + \ + "ME_MIN_SIZE)\n") % mystat.st_size) + try: + os.unlink(myfile_path) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + fetched = 0 + if fetched == 1: #resume mode: writemsg(">>> Resuming download...\n") locfetch=resumecommand |