Use If-Modified-Since HTTP-header and avoid downloading a remote index if the local copy is recent enough.

1 files changed, 41 insertions, 4 deletions

diff --git a/pym/portage/util/_urlopen.py b/pym/portage/util/_urlopen.py
index 307624bc4..42961883d 100644
--- a/pym/portage/util/_urlopen.py
+++ b/pym/portage/util/_urlopen.py
@@ -2,6 +2,9 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import sys
+from datetime import datetime
+from time import mktime
+from email.utils import formatdate, parsedate
 
 try:
 	from urllib.request import urlopen as _urlopen
@@ -14,15 +17,39 @@ except ImportError:
 	import urllib2 as urllib_request
 	from urllib import splituser as urllib_parse_splituser
 
-def urlopen(url):
+if sys.hexversion >= 0x3000000:
+	long = int
+
+# to account for the difference between TIMESTAMP of the index' contents
+#  and the file-'mtime'
+TIMESTAMP_TOLERANCE=5
+
+def urlopen(url, if_modified_since=None):
+	parse_result = urllib_parse.urlparse(url)
 	try:
-		return _urlopen(url)
+		if parse_result.scheme not in ("http", "https"):
+			return _urlopen(url)
+		request = urllib_request.Request(url)
+		request.add_header('User-Agent', 'Gentoo Portage')
+		if if_modified_since:
+			request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since))
+		opener = urllib_request.build_opener()
+		hdl = opener.open(request)
+		if hdl.headers.get('last-modified', ''):
+			try:
+				add_header = hdl.headers.add_header
+			except AttributeError:
+				# Python 2
+				add_header = hdl.headers.addheader
+			add_header('timestamp', _http_to_timestamp(hdl.headers.get('last-modified')))
+		return hdl
 	except SystemExit:
 		raise
-	except Exception:
+	except Exception as e:
+		if hasattr(e, 'code') and e.code == 304: # HTTPError 304: not modified
+			raise
 		if sys.hexversion < 0x3000000:
 			raise
-		parse_result = urllib_parse.urlparse(url)
 		if parse_result.scheme not in ("http", "https") or \
 			not parse_result.username:
 			raise
@@ -40,3 +67,13 @@ def _new_urlopen(url):
 	auth_handler = urllib_request.HTTPBasicAuthHandler(password_manager)
 	opener = urllib_request.build_opener(auth_handler)
 	return opener.open(url)
+
+def _timestamp_to_http(timestamp):
+	dt = datetime.fromtimestamp(float(long(timestamp)+TIMESTAMP_TOLERANCE))
+	stamp = mktime(dt.timetuple())
+	return formatdate(timeval=stamp, localtime=False, usegmt=True)
+
+def _http_to_timestamp(http_datetime_string):
+	tuple = parsedate(http_datetime_string)
+	timestamp = mktime(tuple)
+	return str(long(timestamp))