summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--pym/portage/dbapi/bintree.py104
-rw-r--r--pym/portage/getbinpkg.py20
2 files changed, 122 insertions, 2 deletions
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 2a9651df5..d15f00b8b 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -115,6 +115,7 @@ class binarytree(object):
self._pkg_paths = {}
self._all_directory = os.path.isdir(
os.path.join(self.pkgdir, "All"))
+ self._pkgindex_keys = set(["CPV", "SLOT", "MTIME", "SIZE"])
def move_ent(self, mylist):
if not self.populated:
@@ -371,20 +372,66 @@ class binarytree(object):
dirs.remove("All")
dirs.sort()
dirs.insert(0, "All")
+ pkgfile = os.path.join(self.pkgdir, "Packages")
+ metadata = {}
+ header = {}
+ try:
+ f = open(pkgfile)
+ except EnvironmentError:
+ pass
+ else:
+ try:
+ header = portage.getbinpkg.readpkgindex(f)
+ while True:
+ d = portage.getbinpkg.readpkgindex(f)
+ if not d:
+ break
+ mycpv = d.get("CPV")
+ if not mycpv:
+ continue
+ d.setdefault("SLOT", "0")
+ metadata[mycpv] = d
+ finally:
+ f.close()
+ del f
+ update_pkgindex = False
for mydir in dirs:
for myfile in listdir(os.path.join(self.pkgdir, mydir)):
if not myfile.endswith(".tbz2"):
continue
mypath = os.path.join(mydir, myfile)
full_path = os.path.join(self.pkgdir, mypath)
- if os.path.islink(full_path):
+ s = os.lstat(full_path)
+ if stat.S_ISLNK(s.st_mode):
continue
+ if mydir != "All":
+ # Validate data from the package index and try to avoid
+ # reading the xpak if possible.
+ mycpv = mydir + "/" + myfile[:-5]
+ d = metadata.get(mycpv)
+ skip = False
+ if d:
+ try:
+ if long(d.get("MTIME")) == long(s.st_mtime):
+ skip = True
+ except ValueError:
+ pass
+ if skip and not self._pkgindex_keys.difference(d):
+ pkg_paths[mycpv] = mypath
+ self.dbapi.cpv_inject(mycpv)
+ if not self.dbapi._aux_cache_keys.difference(d):
+ aux_cache = {}
+ for k in self.dbapi._aux_cache_keys:
+ aux_cache[k] = d[k]
+ self.dbapi._aux_cache[mycpv] = aux_cache
+ continue
mytbz2 = portage.xpak.tbz2(full_path)
# For invalid packages, mycat could be None.
mycat = mytbz2.getfile("CATEGORY")
mypf = mytbz2.getfile("PF")
+ slot = mytbz2.getfile("SLOT")
mypkg = myfile[:-5]
- if not mycat or not mypf:
+ if not mycat or not mypf or not slot:
#old-style or corrupt package
writemsg("!!! Invalid binary package: '%s'\n" % full_path,
noiselevel=-1)
@@ -394,6 +441,7 @@ class binarytree(object):
self.invalids.append(mypkg)
continue
mycat = mycat.strip()
+ slot = slot.strip()
if mycat != mydir and mydir != "All":
continue
if mypkg != mypf.strip():
@@ -412,7 +460,59 @@ class binarytree(object):
continue
pkg_paths[mycpv] = mypath
self.dbapi.cpv_inject(mycpv)
+ update_pkgindex = True
+ d = metadata.get(mycpv, {})
+ if d:
+ # Reuse metadata such as MD5, since we won't calculate
+ # MD5 here due to the performance hit.
+ mtime = d.get("MTIME")
+ if mtime:
+ # genpgkindex really should include the mtime and
+ # then this mtime check should be forced.
+ try:
+ if long(mtime) != long(s.st_mtime):
+ d.clear()
+ except ValueError:
+ d.clear()
+ if d:
+ try:
+ if long(d.get("SIZE")) != long(s.st_size):
+ d.clear()
+ except ValueError:
+ d.clear()
+
+ d["CPV"] = mycpv
+ d["SLOT"] = slot
+ d["MTIME"] = str(long(s.st_mtime))
+ d["SIZE"] = str(s.st_size)
+ metadata[mycpv] = d
+ if not self.dbapi._aux_cache_keys.difference(d):
+ aux_cache = {}
+ for k in self.dbapi._aux_cache_keys:
+ aux_cache[k] = d[k]
+ self.dbapi._aux_cache[mycpv] = aux_cache
+
self._pkg_paths = pkg_paths
+ if update_pkgindex and os.access(self.pkgdir, os.W_OK):
+ cpv_all = self._pkg_paths.keys()
+ stale = set(metadata).difference(cpv_all)
+ for cpv in stale:
+ del metadata[cpv]
+ cpv_all.sort()
+ import time
+ from portage.util import atomic_ofstream
+ header["TIMESTAMP"] = str(long(time.time()))
+ header["PACKAGES"] = str(len(cpv_all))
+ f = atomic_ofstream(pkgfile)
+ try:
+ portage.getbinpkg.writepkgindex(f, header.iteritems())
+ for cpv in cpv_all:
+ d = metadata[cpv]
+ if d["SLOT"] == "0":
+ del d["SLOT"]
+ portage.getbinpkg.writepkgindex(f, d.iteritems())
+ finally:
+ f.close()
if getbinpkgs and not self.settings["PORTAGE_BINHOST"]:
writemsg(red("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py
index b0d5ca9eb..070b036b5 100644
--- a/pym/portage/getbinpkg.py
+++ b/pym/portage/getbinpkg.py
@@ -570,3 +570,23 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
conn.close()
return metadata[baseurl]["data"]
+
+def readpkgindex(pkgfile):
+ d = {}
+ for line in pkgfile:
+ line = line.rstrip("\n")
+ if not line:
+ break
+ line = line.split(":", 1)
+ if not len(line) == 2:
+ continue
+ k, v = line
+ if v:
+ v = v[1:]
+ d[k] = v
+ return d
+
+def writepkgindex(pkgfile, items):
+ for k, v in items:
+ pkgfile.write("%s: %s\n" % (k, v))
+ pkgfile.write("\n")