Optimize `emaint` --fix binhost so that it the Packages file isn't re-read

and re-written for each package that's updated. Instead, hold a lock for
the whole time the command is running and just update it once.

svn path=/main/trunk/; revision=11307

3 files changed, 82 insertions, 31 deletions

diff --git a/bin/emaint b/bin/emaint
index b9b7a6676..561d26719 100755
--- a/bin/emaint
+++ b/bin/emaint
@@ -112,8 +112,7 @@ class BinhostHandler(object):
 		self._bintree = portage.db[myroot]["bintree"]
 		self._bintree.populate()
 		self._pkgindex_file = os.path.join(self._bintree.pkgdir, "Packages")
-		from portage import getbinpkg
-		self._pkgindex = getbinpkg.PackageIndex()
+		self._pkgindex = self._bintree._new_pkgindex()
 		f = open(self._pkgindex_file, 'r')
 		try:
 			self._pkgindex.read(f)
@@ -149,7 +148,7 @@ class BinhostHandler(object):
 		cpv_all = self._bintree.dbapi.cpv_all()
 		cpv_all.sort()
 		missing = []
-		maxval = len(cpv_all)
+		maxval = 0
 		if onProgress:
 			onProgress(maxval, 0)
 		pkgindex = self._pkgindex
@@ -157,35 +156,61 @@ class BinhostHandler(object):
 		metadata = {}
 		for d in pkgindex.packages:
 			metadata[d["CPV"]] = d
+
 		for i, cpv in enumerate(cpv_all):
 			d = metadata.get(cpv)
 			if not d or "MD5" not in d:
-				bintree.inject(cpv)
-			if onProgress:
-				onProgress(maxval, i+1)
+				missing.append(cpv)
+
 		stale = set(metadata).difference(cpv_all)
-		if stale:
+		if missing or stale:
 			from portage import locks
 			pkgindex_lock = locks.lockfile(
 				self._pkgindex_file, wantnewlockfile=1)
 			try:
-				from portage import getbinpkg
-				pkgindex = getbinpkg.PackageIndex()
+				# Repopulate with lock held.
+				bintree._populate()
+				cpv_all = self._bintree.dbapi.cpv_all()
+				cpv_all.sort()
+
+				pkgindex = bintree._new_pkgindex()
 				self._pkgindex = pkgindex
 				f = open(self._pkgindex_file, 'r')
 				try:
 					self._pkgindex.read(f)
 				finally:
 					f.close()
-				from portage.dbapi.bintree import binarytree
-				self._bintree = binarytree(bintree.root, bintree.pkgdir,
-					settings=bintree.settings)
-				del bintree
-				portage.db[self._bintree.root]["bintree"] = self._bintree
-				self._bintree._populate()
+
+				metadata = {}
+				for d in pkgindex.packages:
+					metadata[d["CPV"]] = d
+
+				# Recount missing packages, with lock held.
+				del missing[:]
+				for i, cpv in enumerate(cpv_all):
+					d = metadata.get(cpv)
+					if not d or "MD5" not in d:
+						missing.append(cpv)
+
+				maxval = len(missing)
+				for i, cpv in enumerate(missing):
+					try:
+						metadata[cpv] = bintree._pkgindex_entry(cpv)
+					except portage.exception.InvalidDependString:
+						writemsg("!!! Invalid binary package: '%s'\n" % \
+							bintree.getname(cpv), noiselevel=-1)
+
+					if onProgress:
+						onProgress(maxval, i+1)
+
 				for cpv in set(metadata).difference(
 					self._bintree.dbapi.cpv_all()):
 					del metadata[cpv]
+
+				# We've updated the pkgindex, so set it to
+				# repopulate when necessary.
+				bintree.populated = False
+
 				del pkgindex.packages[:]
 				pkgindex.packages.extend(metadata.itervalues())
 				from portage.util import atomic_ofstream
@@ -196,6 +221,11 @@ class BinhostHandler(object):
 					f.close()
 			finally:
 				locks.unlockfile(pkgindex_lock)
+
+		if onProgress:
+			if maxval == 0:
+				maxval = 1
+			onProgress(maxval, maxval)
 		return None
 
 class MoveHandler(object):
diff --git a/pym/portage/cache/mappings.py b/pym/portage/cache/mappings.py
index 96e9e553b..112301476 100644
--- a/pym/portage/cache/mappings.py
+++ b/pym/portage/cache/mappings.py
@@ -262,6 +262,9 @@ def slot_dict_class(keys, prefix="_val_"):
 					except AttributeError:
 						pass
 
+			def __str__(self):
+				return str(dict(self.iteritems()))
+
 		v = SlotDict
 		_slot_dict_classes[v.allowed_keys] = v
 	return v
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 86b8aba81..59835c073 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -764,9 +764,6 @@ class binarytree(object):
 				noiselevel=-1)
 			return
 		slot = slot.strip()
-		from portage.checksum import perform_multiple_checksums
-		digests = perform_multiple_checksums(
-			full_path, hashes=self._pkgindex_hashes)
 		self.dbapi.cpv_inject(cpv)
 		self.dbapi._aux_cache.pop(cpv, None)
 
@@ -797,26 +794,16 @@ class binarytree(object):
 					del f
 			if not self._pkgindex_version_supported(pkgindex):
 				pkgindex = self._new_pkgindex()
-			d = digests
-			d["CPV"] = cpv
-			d["SLOT"] = slot
-			d["MTIME"] = str(long(s.st_mtime))
-			d["SIZE"] = str(s.st_size)
-			rel_path = self._pkg_paths[cpv]
-			# record location if it's non-default
-			if rel_path != cpv + ".tbz2":
-				d["PATH"] = rel_path
-			from itertools import izip
-			d.update(izip(self._pkgindex_aux_keys,
-				self.dbapi.aux_get(cpv, self._pkgindex_aux_keys)))
+
 			try:
-				self._eval_use_flags(cpv, d)
+				d = self._pkgindex_entry(cpv)
 			except portage.exception.InvalidDependString:
 				writemsg("!!! Invalid binary package: '%s'\n" % \
 					self.getname(cpv), noiselevel=-1)
 				self.dbapi.cpv_remove(cpv)
 				del self._pkg_paths[cpv]
 				return
+
 			# If found, remove package(s) with duplicate path.
 			for i in xrange(len(pkgindex.packages) - 1, -1, -1):
 				d2 = pkgindex.packages[i]
@@ -825,6 +812,7 @@ class binarytree(object):
 				if d2.get("PATH", "") == d.get("PATH", ""):
 					del pkgindex.packages[i]
 			pkgindex.packages.append(d)
+
 			self._update_pkgindex_header(pkgindex.header)
 			from portage.util import atomic_ofstream
 			f = atomic_ofstream(os.path.join(self.pkgdir, "Packages"))
@@ -836,6 +824,36 @@ class binarytree(object):
 			if pkgindex_lock:
 				unlockfile(pkgindex_lock)
 
+	def _pkgindex_entry(self, cpv):
+		"""
+		Performs checksums and evaluates USE flag conditionals.
+		Raises InvalidDependString if necessary.
+		@rtype: dict
+		@returns: a dict containing entry for the give cpv.
+		"""
+
+		pkg_path = self.getname(cpv)
+		from portage.checksum import perform_multiple_checksums
+
+		d = dict(izip(self._pkgindex_aux_keys,
+			self.dbapi.aux_get(cpv, self._pkgindex_aux_keys)))
+
+		d.update(perform_multiple_checksums(
+			pkg_path, hashes=self._pkgindex_hashes))
+
+		d["CPV"] = cpv
+		st = os.stat(pkg_path)
+		d["MTIME"] = str(long(st.st_mtime))
+		d["SIZE"] = str(st.st_size)
+
+		rel_path = self._pkg_paths[cpv]
+		# record location if it's non-default
+		if rel_path != cpv + ".tbz2":
+			d["PATH"] = rel_path
+
+		self._eval_use_flags(cpv, d)
+		return d
+
 	def _new_pkgindex(self):
 		return portage.getbinpkg.PackageIndex(
 			allowed_pkg_keys=self._pkgindex_allowed_pkg_keys,