Add a generic portage.cache.mappings.slot_dict_class() function which

generates mapping classes that behave similar to a dict but store
values as object attributes that are allocated via __slots__. Instances
of these objects have a smaller memory footprint than a normal dict object.
These classes are used to reduce the memory footprint of the dbapi.aux_get()
caches and the Package.metadata attribute.

svn path=/main/trunk/; revision=10790

4 files changed, 151 insertions, 91 deletions

diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index 2f54b6b65..ca76400a3 100644
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -1371,108 +1371,33 @@ class Package(Task):
 			return True
 		return False
 
-class _PackageMetadataWrapper(object):
+_all_metadata_keys = set(x for x in portage.auxdbkeys \
+	if not x.startswith("UNUSED_"))
+_all_metadata_keys.discard("CDEPEND")
+_all_metadata_keys.update(Package.metadata_keys)
+
+from portage.cache.mappings import slot_dict_class
+_PackageMetadataWrapperBase = slot_dict_class(_all_metadata_keys)
+
+class _PackageMetadataWrapper(_PackageMetadataWrapperBase):
 	"""
 	Detect metadata updates and synchronize Package attributes.
 	"""
-	_keys = set(x for x in portage.auxdbkeys \
-		if not x.startswith("UNUSED_"))
-	_keys.discard("CDEPEND")
-	_keys.update(Package.metadata_keys)
-	_keys = tuple(sorted(_keys))
-	__slots__ = ("__weakref__", "_pkg") + tuple("_val_" + k for k in _keys)
+
+	__slots__ = ("_pkg",)
 	_wrapped_keys = frozenset(
 		["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"])
 
 	def __init__(self, pkg, metadata):
+		_PackageMetadataWrapperBase.__init__(self)
 		self._pkg = pkg
 		self.update(metadata)
 
-	def __iter__(self):
-		for k, v in self.iteritems():
-			yield k
-
-	def __len__(self):
-		l = 0
-		for i in self.iteritems():
-			l += 1
-		return l
-
-	def keys(self):
-		return list(self)
-
-	def iteritems(self):
-		for k in self._keys:
-			try:
-				yield (k, getattr(self, "_val_" + k))
-			except AttributeError:
-				pass
-
-	def items(self):
-		return list(self.iteritems())
-
-	def itervalues(self):
-		for k, v in self.itervalues():
-			yield v
-
-	def values(self):
-		return list(self.itervalues())
-
-	def __delitem__(self, k):
-		try:
-			delattr(self, "_val_" + k)
-		except AttributeError:
-			raise KeyError(k)
-
 	def __setitem__(self, k, v):
-		setattr(self, "_val_" + k, v)
+		_PackageMetadataWrapperBase.__setitem__(self, k, v)
 		if k in self._wrapped_keys:
 			getattr(self, "_set_" + k.lower())(k, v)
 
-	def update(self, d):
-		i = getattr(d, "iteritems", None)
-		if i is None:
-			i = d
-		else:
-			i = i()
-		for k, v in i:
-			self[k] = v
-
-	def __getitem__(self, k):
-		try:
-			return getattr(self, "_val_" + k)
-		except AttributeError:
-			raise KeyError(k)
-
-	def get(self, key, default=None):
-		try:
-			return self[key]
-		except KeyError:
-			return default
-
-	def __contains__(self, k):
-		return hasattr(self, "_val_" + k)
-
-	def pop(self, key, *args):
-		if len(args) > 1:
-			raise TypeError("pop expected at most 2 arguments, got " + \
-				repr(1 + len(args)))
-		try:
-			value = self[key]
-		except KeyError:
-			if args:
-				return args[0]
-			raise
-		del self[key]
-		return value
-
-	def clear(self):
-		for k in self._keys:
-			try:
-				delattr(self, "_val_" + k)
-			except AttributError:
-				pass
-
 	def _set_inherited(self, k, v):
 		if isinstance(v, basestring):
 			v = frozenset(v.split())
diff --git a/pym/portage/cache/mappings.py b/pym/portage/cache/mappings.py
index 9aa5a21e2..7c347a4f9 100644
--- a/pym/portage/cache/mappings.py
+++ b/pym/portage/cache/mappings.py
@@ -4,6 +4,7 @@
 # $Id$
 
 import UserDict
+import weakref
 
 class ProtectedDict(UserDict.DictMixin):
 	"""
@@ -101,3 +102,128 @@ class LazyLoad(UserDict.DictMixin):
 			self.pull = None
 		return key in self.d
 
+_slot_dict_classes = weakref.WeakValueDictionary()
+
+def slot_dict_class(keys):
+	if isinstance(keys, frozenset):
+		keys_set = keys
+	else:
+		keys_set = frozenset(keys)
+	v = _slot_dict_classes.get(keys_set)
+	if v is None:
+
+		class SlotDict(object):
+
+			_keys = keys_set
+			__slots__ = ("__weakref__",) + tuple("_val_" + k for k in _keys)
+
+			def __iter__(self):
+				for k, v in self.iteritems():
+					yield k
+
+			def __len__(self):
+				l = 0
+				for i in self.iteritems():
+					l += 1
+				return l
+
+			def keys(self):
+				return list(self)
+
+			def iteritems(self):
+				for k in self._keys:
+					try:
+						yield (k, getattr(self, "_val_" + k))
+					except AttributeError:
+						pass
+
+			def items(self):
+				return list(self.iteritems())
+
+			def itervalues(self):
+				for k, v in self.itervalues():
+					yield v
+
+			def values(self):
+				return list(self.itervalues())
+
+			def __delitem__(self, k):
+				try:
+					delattr(self, "_val_" + k)
+				except AttributeError:
+					raise KeyError(k)
+
+			def __setitem__(self, k, v):
+				setattr(self, "_val_" + k, v)
+
+			def setdefault(self, key, default=None):
+				try:
+					return self[key]
+				except KeyError:
+					self[key] = default
+				return default
+
+			def update(self, d):
+				i = getattr(d, "iteritems", None)
+				if i is None:
+					i = d
+				else:
+					i = i()
+				for k, v in i:
+					self[k] = v
+
+			def __getitem__(self, k):
+				try:
+					return getattr(self, "_val_" + k)
+				except AttributeError:
+					raise KeyError(k)
+
+			def get(self, key, default=None):
+				try:
+					return self[key]
+				except KeyError:
+					return default
+
+			def __contains__(self, k):
+				return hasattr(self, "_val_" + k)
+
+			def has_key(self, k):
+				return k in self
+
+			def pop(self, key, *args):
+				if len(args) > 1:
+					raise TypeError(
+						"pop expected at most 2 arguments, got " + \
+						repr(1 + len(args)))
+				try:
+					value = self[key]
+				except KeyError:
+					if args:
+						return args[0]
+					raise
+				del self[key]
+				return value
+
+			def popitem(self):
+				try:
+					k, v = self.iteritems().next()
+				except StopIteration:
+					raise KeyError, 'container is empty'
+				del self[k]
+				return (k, v)
+
+			def copy(self):
+				c = self.__class__()
+				c.update(self)
+				return c
+
+			def clear(self):
+				for k in self._keys:
+					try:
+						delattr(self, "_val_" + k)
+					except AttributError:
+						pass
+
+		v = SlotDict
+		_slot_dict_classes[keys_set] = v
+		return v
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index fe8b372c2..b3c36a23b 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+from portage.cache.mappings import slot_dict_class
 from portage.dep import isvalidatom, isjustname, dep_getkey, match_from_list
 from portage.dbapi.virtual import fakedbapi
 from portage.exception import InvalidPackageName, InvalidAtom, \
@@ -33,6 +34,7 @@ class bindbapi(fakedbapi):
 			["CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS",
 			"LICENSE", "PDEPEND", "PROVIDE",
 			"RDEPEND", "repository", "RESTRICT", "SLOT", "USE"])
+		self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys)
 		self._aux_cache = {}
 
 	def match(self, *pargs, **kwargs):
@@ -75,7 +77,7 @@ class bindbapi(fakedbapi):
 			if not mydata.setdefault("EAPI", "0"):
 				mydata["EAPI"] = "0"
 		if cache_me:
-			aux_cache = {}
+			aux_cache = self._aux_cache_slot_dict()
 			for x in self._aux_cache_keys:
 				aux_cache[x] = mydata.get(x, "")
 			self._aux_cache[mycpv] = aux_cache
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 3c27c2d95..b6e39f63b 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -3,6 +3,7 @@
 # $Id$
 
 from portage.cache.cache_errors import CacheError
+from portage.cache.mappings import slot_dict_class
 from portage.const import REPO_NAME_LOC
 from portage.data import portage_gid, secpass
 from portage.dbapi import dbapi
@@ -138,6 +139,10 @@ class portdbapi(dbapi):
 			["DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE",
 			"PDEPEND", "PROVIDE", "RDEPEND", "repository",
 			"RESTRICT", "SLOT"])
+
+		# Repoman modifies _aux_cache_keys, so delay _aux_cache_slot_dict
+		# initialization until the first aux_get call.
+		self._aux_cache_slot_dict = None
 		self._aux_cache = {}
 		self._broken_ebuilds = set()
 
@@ -386,7 +391,10 @@ class portdbapi(dbapi):
 				returnme.append(mydata.get(x,""))
 
 		if cache_me:
-			aux_cache = {}
+			if self._aux_cache_slot_dict is None:
+				self._aux_cache_slot_dict = \
+					slot_dict_class(self._aux_cache_keys)
+			aux_cache = self._aux_cache_slot_dict()
 			for x in self._aux_cache_keys:
 				aux_cache[x] = mydata.get(x, "")
 			self._aux_cache[mycpv] = aux_cache
@@ -866,4 +874,3 @@ class portagetree(object):
 		except Exception, e:
 			pass
 		return myslot
-