From e06cb6d66db37ac7ab77acf65038b1f770c13c96 Mon Sep 17 00:00:00 2001 From: W-Mark Kubacki Date: Wed, 1 Aug 2012 19:49:34 +0200 Subject: Use If-Modified-Since HTTP-header and avoid downloading a remote index if the local copy is recent enough. --- pym/portage/util/_urlopen.py | 45 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) (limited to 'pym/portage/util/_urlopen.py') diff --git a/pym/portage/util/_urlopen.py b/pym/portage/util/_urlopen.py index 307624bc4..42961883d 100644 --- a/pym/portage/util/_urlopen.py +++ b/pym/portage/util/_urlopen.py @@ -2,6 +2,9 @@ # Distributed under the terms of the GNU General Public License v2 import sys +from datetime import datetime +from time import mktime +from email.utils import formatdate, parsedate try: from urllib.request import urlopen as _urlopen @@ -14,15 +17,39 @@ except ImportError: import urllib2 as urllib_request from urllib import splituser as urllib_parse_splituser -def urlopen(url): +if sys.hexversion >= 0x3000000: + long = int + +# to account for the difference between TIMESTAMP of the index' contents +# and the file-'mtime' +TIMESTAMP_TOLERANCE=5 + +def urlopen(url, if_modified_since=None): + parse_result = urllib_parse.urlparse(url) try: - return _urlopen(url) + if parse_result.scheme not in ("http", "https"): + return _urlopen(url) + request = urllib_request.Request(url) + request.add_header('User-Agent', 'Gentoo Portage') + if if_modified_since: + request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since)) + opener = urllib_request.build_opener() + hdl = opener.open(request) + if hdl.headers.get('last-modified', ''): + try: + add_header = hdl.headers.add_header + except AttributeError: + # Python 2 + add_header = hdl.headers.addheader + add_header('timestamp', _http_to_timestamp(hdl.headers.get('last-modified'))) + return hdl except SystemExit: raise - except Exception: + except Exception as e: + if hasattr(e, 'code') and e.code == 304: # HTTPError 304: not modified + raise if sys.hexversion < 0x3000000: raise - parse_result = urllib_parse.urlparse(url) if parse_result.scheme not in ("http", "https") or \ not parse_result.username: raise @@ -40,3 +67,13 @@ def _new_urlopen(url): auth_handler = urllib_request.HTTPBasicAuthHandler(password_manager) opener = urllib_request.build_opener(auth_handler) return opener.open(url) + +def _timestamp_to_http(timestamp): + dt = datetime.fromtimestamp(float(long(timestamp)+TIMESTAMP_TOLERANCE)) + stamp = mktime(dt.timetuple()) + return formatdate(timeval=stamp, localtime=False, usegmt=True) + +def _http_to_timestamp(http_datetime_string): + tuple = parsedate(http_datetime_string) + timestamp = mktime(tuple) + return str(long(timestamp)) -- cgit v1.2.3-1-g7c22