summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorZac Medico <zmedico@gentoo.org>2007-05-31 12:47:05 +0000
committerZac Medico <zmedico@gentoo.org>2007-05-31 12:47:05 +0000
commit00152eff9e23a6a6f0efb226c3f234bcd9c20865 (patch)
treeb7887bae37d7a8cb8cba4c079eceb9fa5f84943a
parenteea590a6ef30d1bc2b9be766af6fa6cb5eb4413e (diff)
downloadportage-00152eff9e23a6a6f0efb226c3f234bcd9c20865.tar.gz
portage-00152eff9e23a6a6f0efb226c3f234bcd9c20865.tar.bz2
portage-00152eff9e23a6a6f0efb226c3f234bcd9c20865.zip
Add PORTAGE_BINHOST support for fetching a remote Packages index. If the index is not found then it falls back to the old directory listing method.
svn path=/main/trunk/; revision=6694
-rw-r--r--pym/portage/dbapi/bintree.py99
-rw-r--r--pym/portage/getbinpkg.py14
2 files changed, 102 insertions, 11 deletions
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index fd6292894..4d598ad7a 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -122,7 +122,9 @@ class binarytree(object):
self.move_slot_ent = self.dbapi.move_slot_ent
self.populated = 0
self.tree = {}
- self.remotepkgs = {}
+ self._remote_has_index = False
+ self._remotepkgs = None # remote metadata indexed by cpv
+ self.remotepkgs = {} # indexed by tbz2 name (deprecated)
self.invalids = []
self.settings = settings
self._pkg_paths = {}
@@ -478,15 +480,77 @@ class binarytree(object):
noiselevel=-1)
if getbinpkgs and \
- self.settings["PORTAGE_BINHOST"] and not self.remotepkgs:
+ "PORTAGE_BINHOST" in self.settings and \
+ not self._remotepkgs:
+
+ base_url = self.settings["PORTAGE_BINHOST"]
+ from portage.const import CACHE_PATH
+ from urlparse import urlparse
+ urldata = urlparse(base_url)
+ pkgindex_file = os.path.join(CACHE_PATH, "binhost",
+ urldata[1] + urldata[2], "Packages")
+ pkgindex = portage.getbinpkg.PackageIndex()
+ try:
+ f = open(pkgindex_file)
+ try:
+ pkgindex.read(f)
+ finally:
+ f.close()
+ except EnvironmentError, e:
+ if e.errno != errno.ENOENT:
+ raise
+ local_timestamp = pkgindex.header.get("TIMESTAMP", None)
+ import urllib, urlparse
+ rmt_idx = portage.getbinpkg.PackageIndex()
+ try:
+ f = urllib.urlopen(urlparse.urljoin(base_url, "Packages"))
+ try:
+ rmt_idx.readHeader(f)
+ remote_timestamp = rmt_idx.header.get("TIMESTAMP", None)
+ if not remote_timestamp:
+ # no timestamp in the header, something's wrong
+ pkgindex = None
+ else:
+ if local_timestamp != remote_timestamp:
+ rmt_idx.readBody(f)
+ pkgindex = rmt_idx
+ finally:
+ f.close()
+ except EnvironmentError, e:
+ writemsg("\n\n!!! Error fetching binhost package" + \
+ " info from '%s'\n" % base_url)
+ writemsg("!!! %s\n\n" % str(e))
+ del e
+ pkgindex = None
+ if pkgindex is rmt_idx:
+ pkgindex.modified = False # don't update the header
+ from portage.util import atomic_ofstream, ensure_dirs
+ ensure_dirs(os.path.dirname(pkgindex_file))
+ f = atomic_ofstream(pkgindex_file)
+ try:
+ pkgindex.write(f)
+ finally:
+ f.close()
+ if pkgindex:
+ self._remotepkgs = pkgindex.packages
+ self._remote_has_index = True
+ self.remotepkgs = {}
+ for cpv, metadata in self._remotepkgs.iteritems():
+ self.dbapi.cpv_inject(cpv)
+ cat, pf = catsplit(cpv)
+ # backward compat
+ self.remotepkgs[pf+".tbz2"] = metadata
+ metadata["CATEGORY"] = cat
+ self.populated = 1
+ return
+ self._remotepkgs = {}
try:
chunk_size = long(self.settings["PORTAGE_BINHOST_CHUNKSIZE"])
if chunk_size < 8:
chunk_size = 8
except (ValueError, KeyError):
chunk_size = 3000
-
- writemsg(green("Fetching binary packages info...\n"))
+ writemsg_stdout(green("Fetching binary packages info...\n"))
self.remotepkgs = portage.getbinpkg.dir_get_metadata(
self.settings["PORTAGE_BINHOST"], chunk_size=chunk_size)
writemsg(green(" -- DONE!\n\n"))
@@ -513,6 +577,10 @@ class binarytree(object):
# invalid tbz2's can hurt things.
#print "cpv_inject("+str(fullpkg)+")"
self.dbapi.cpv_inject(fullpkg)
+ metadata = self.remotepkgs[mypkg]
+ for k, v in metadata.items():
+ metadata[k] = v.strip()
+ self._remotepkgs[fullpkg] = metadata
#print " -- Injected"
except SystemExit, e:
raise
@@ -698,10 +766,25 @@ class binarytree(object):
os.makedirs(mydest, 0775)
except (OSError, IOError):
pass
- success = portage.getbinpkg.file_get(
- self.settings["PORTAGE_BINHOST"] + "/" + tbz2name,
- mydest, fcmd=self.settings["RESUMECOMMAND"])
- self.inject(pkgname)
+ from urlparse import urljoin
+ base_url = self.settings["PORTAGE_BINHOST"]
+ fcmd = self.settings["RESUMECOMMAND"]
+ if self._remote_has_index:
+ url = urljoin(base_url, pkgname+".tbz2")
+ success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd)
+ if not success:
+ try:
+ os.unlink(self.getname(pkgname))
+ except OSError:
+ pass
+ # Fall back to the "All" directory
+ uri = urljoin(base_url, "All/"+tbz2name)
+ success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd)
+ else:
+ uri = urljoin(base_url, tbz2name)
+ success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd)
+ if success:
+ self.inject(pkgname)
return success
def getslot(self, mycatpkg):
diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py
index 76016e3c0..f44f690a6 100644
--- a/pym/portage/getbinpkg.py
+++ b/pym/portage/getbinpkg.py
@@ -596,9 +596,16 @@ class PackageIndex(object):
def __init__(self):
self.header = {}
self.packages = {}
+ self.modified = True
def read(self, pkgfile):
+ self.readHeader(pkgfile)
+ self.readBody(pkgfile)
+
+ def readHeader(self, pkgfile):
self.header.update(readpkgindex(pkgfile))
+
+ def readBody(self, pkgfile):
while True:
d = readpkgindex(pkgfile)
if not d:
@@ -612,9 +619,10 @@ class PackageIndex(object):
def write(self, pkgfile):
cpv_all = self.packages.keys()
cpv_all.sort()
- import time
- self.header["TIMESTAMP"] = str(long(time.time()))
- self.header["PACKAGES"] = str(len(cpv_all))
+ if self.modified:
+ import time
+ self.header["TIMESTAMP"] = str(long(time.time()))
+ self.header["PACKAGES"] = str(len(cpv_all))
keys = self.header.keys()
keys.sort()
writepkgindex(pkgfile, [(k, self.header[k]) for k in keys])