From 11c0619c63b54346ee5c67cd67ab1ccb24f5f947 Mon Sep 17 00:00:00 2001 From: W-Mark Kubacki Date: Wed, 8 Aug 2012 18:49:36 +0200 Subject: Portage writes a compressed copy of 'Packages' index file. This behaviour is enabled by FEATURES="compress-index". The resulting file is 'Packages.gz' and its modification time will match that of 'Packages'. Web-servers use that copy to avoid repeated on-the-fly compression. In order to re-use 'atomic_ofstream' usage of 'codecs.zlib_codec' has been considered and discarded, because 'GzipFile' yields smaller files. (According to Mark's tests 62% smaller.) Example usage, Nginx: location =/Packages { gzip_static on; default_type text/plain; } Apache httpd (use with caution): RewriteRule ^(.*)/Packages$ $1/Packages.gz [T=text/plain,E=GZIP:gzip,L] Header set Content-Encoding gzip --- man/make.conf.5 | 7 +++++++ pym/portage/const.py | 2 +- pym/portage/dbapi/bintree.py | 21 +++++++++++++++------ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/man/make.conf.5 b/man/make.conf.5 index 876a8a330..02cc8098d 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -268,6 +268,13 @@ space. Make sure you have built both binutils and gdb with USE=zlib support for this to work. See \fBsplitdebug\fR for general split debug information (upon which this feature depends). .TP +.B compress\-index +If set then a compressed copy of 'Packages' index file will be written. +This feature is intended for Gentoo binhosts using certain webservers +(such as, but not limited to, Nginx with gzip_static module) to avoid +redundant on\-the\-fly compression. The resulting file will be called +'Packages.gz' and its modification time will match that of 'Packages'. +.TP .B config\-protect\-if\-modified This causes the \fBCONFIG_PROTECT\fR behavior to be skipped for files that have not been modified since they were installed. This feature is diff --git a/pym/portage/const.py b/pym/portage/const.py index ceef5c56b..c2049f871 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -89,7 +89,7 @@ SUPPORTED_FEATURES = frozenset([ "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", "ccache", "chflags", "clean-logs", "collision-protect", "compress-build-logs", "compressdebug", - "config-protect-if-modified", + "compress-index", "config-protect-if-modified", "digest", "distcc", "distcc-pump", "distlocks", "downgrade-backup", "ebuild-locks", "fakeroot", "fail-clean", "force-mirror", "force-prefix", "getbinpkg", diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 03675031e..77fc0c4ef 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -41,6 +41,7 @@ import sys import tempfile import textwrap import warnings +from gzip import GzipFile from itertools import chain try: from urllib.parse import urlparse @@ -1186,13 +1187,21 @@ class binarytree(object): pkgindex.packages.append(d) self._update_pkgindex_header(pkgindex.header) - pkgindex_filename = os.path.join(self.pkgdir, "Packages") - f = atomic_ofstream(pkgindex_filename) - pkgindex.write(f) - f.close() - # some seconds might have elapsed since TIMESTAMP + contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO()) + pkgindex.write(contents) + contents = contents.getvalue() atime = mtime = long(pkgindex.header["TIMESTAMP"]) - os.utime(pkgindex_filename, (atime, mtime)) + + pkgindex_filename = os.path.join(self.pkgdir, "Packages") + output_files = [(atomic_ofstream(pkgindex_filename, mode="wb"), pkgindex_filename)] + if "compress-index" in self.settings.features: + gz_fname = pkgindex_filename + ".gz" + output_files.append((GzipFile(gz_fname, mode="wb"), gz_fname)) + for f, fname in output_files: + f.write(contents) + f.close() + # some seconds might have elapsed since TIMESTAMP + os.utime(fname, (atime, mtime)) finally: if pkgindex_lock: unlockfile(pkgindex_lock) -- cgit v1.2.3-1-g7c22