summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorZac Medico <zmedico@gentoo.org>2008-06-25 22:36:19 +0000
committerZac Medico <zmedico@gentoo.org>2008-06-25 22:36:19 +0000
commit04066d484cf68bda8de37c037aad003a44088e8a (patch)
treec1b7bc121c23ac651303c8d91e896f6622fa5bc7
parent4ac209e6cf6c49d88b06a4ddaa3bacd11695791d (diff)
downloadportage-04066d484cf68bda8de37c037aad003a44088e8a.tar.gz
portage-04066d484cf68bda8de37c037aad003a44088e8a.tar.bz2
portage-04066d484cf68bda8de37c037aad003a44088e8a.zip
Add a generic portage.cache.mappings.slot_dict_class() function which
generates mapping classes that behave similar to a dict but store values as object attributes that are allocated via __slots__. Instances of these objects have a smaller memory footprint than a normal dict object. These classes are used to reduce the memory footprint of the dbapi.aux_get() caches and the Package.metadata attribute. svn path=/main/trunk/; revision=10790
-rw-r--r--pym/_emerge/__init__.py101
-rw-r--r--pym/portage/cache/mappings.py126
-rw-r--r--pym/portage/dbapi/bintree.py4
-rw-r--r--pym/portage/dbapi/porttree.py11
4 files changed, 151 insertions, 91 deletions
diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index 2f54b6b65..ca76400a3 100644
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -1371,108 +1371,33 @@ class Package(Task):
return True
return False
-class _PackageMetadataWrapper(object):
+_all_metadata_keys = set(x for x in portage.auxdbkeys \
+ if not x.startswith("UNUSED_"))
+_all_metadata_keys.discard("CDEPEND")
+_all_metadata_keys.update(Package.metadata_keys)
+
+from portage.cache.mappings import slot_dict_class
+_PackageMetadataWrapperBase = slot_dict_class(_all_metadata_keys)
+
+class _PackageMetadataWrapper(_PackageMetadataWrapperBase):
"""
Detect metadata updates and synchronize Package attributes.
"""
- _keys = set(x for x in portage.auxdbkeys \
- if not x.startswith("UNUSED_"))
- _keys.discard("CDEPEND")
- _keys.update(Package.metadata_keys)
- _keys = tuple(sorted(_keys))
- __slots__ = ("__weakref__", "_pkg") + tuple("_val_" + k for k in _keys)
+
+ __slots__ = ("_pkg",)
_wrapped_keys = frozenset(
["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"])
def __init__(self, pkg, metadata):
+ _PackageMetadataWrapperBase.__init__(self)
self._pkg = pkg
self.update(metadata)
- def __iter__(self):
- for k, v in self.iteritems():
- yield k
-
- def __len__(self):
- l = 0
- for i in self.iteritems():
- l += 1
- return l
-
- def keys(self):
- return list(self)
-
- def iteritems(self):
- for k in self._keys:
- try:
- yield (k, getattr(self, "_val_" + k))
- except AttributeError:
- pass
-
- def items(self):
- return list(self.iteritems())
-
- def itervalues(self):
- for k, v in self.itervalues():
- yield v
-
- def values(self):
- return list(self.itervalues())
-
- def __delitem__(self, k):
- try:
- delattr(self, "_val_" + k)
- except AttributeError:
- raise KeyError(k)
-
def __setitem__(self, k, v):
- setattr(self, "_val_" + k, v)
+ _PackageMetadataWrapperBase.__setitem__(self, k, v)
if k in self._wrapped_keys:
getattr(self, "_set_" + k.lower())(k, v)
- def update(self, d):
- i = getattr(d, "iteritems", None)
- if i is None:
- i = d
- else:
- i = i()
- for k, v in i:
- self[k] = v
-
- def __getitem__(self, k):
- try:
- return getattr(self, "_val_" + k)
- except AttributeError:
- raise KeyError(k)
-
- def get(self, key, default=None):
- try:
- return self[key]
- except KeyError:
- return default
-
- def __contains__(self, k):
- return hasattr(self, "_val_" + k)
-
- def pop(self, key, *args):
- if len(args) > 1:
- raise TypeError("pop expected at most 2 arguments, got " + \
- repr(1 + len(args)))
- try:
- value = self[key]
- except KeyError:
- if args:
- return args[0]
- raise
- del self[key]
- return value
-
- def clear(self):
- for k in self._keys:
- try:
- delattr(self, "_val_" + k)
- except AttributError:
- pass
-
def _set_inherited(self, k, v):
if isinstance(v, basestring):
v = frozenset(v.split())
diff --git a/pym/portage/cache/mappings.py b/pym/portage/cache/mappings.py
index 9aa5a21e2..7c347a4f9 100644
--- a/pym/portage/cache/mappings.py
+++ b/pym/portage/cache/mappings.py
@@ -4,6 +4,7 @@
# $Id$
import UserDict
+import weakref
class ProtectedDict(UserDict.DictMixin):
"""
@@ -101,3 +102,128 @@ class LazyLoad(UserDict.DictMixin):
self.pull = None
return key in self.d
+_slot_dict_classes = weakref.WeakValueDictionary()
+
+def slot_dict_class(keys):
+ if isinstance(keys, frozenset):
+ keys_set = keys
+ else:
+ keys_set = frozenset(keys)
+ v = _slot_dict_classes.get(keys_set)
+ if v is None:
+
+ class SlotDict(object):
+
+ _keys = keys_set
+ __slots__ = ("__weakref__",) + tuple("_val_" + k for k in _keys)
+
+ def __iter__(self):
+ for k, v in self.iteritems():
+ yield k
+
+ def __len__(self):
+ l = 0
+ for i in self.iteritems():
+ l += 1
+ return l
+
+ def keys(self):
+ return list(self)
+
+ def iteritems(self):
+ for k in self._keys:
+ try:
+ yield (k, getattr(self, "_val_" + k))
+ except AttributeError:
+ pass
+
+ def items(self):
+ return list(self.iteritems())
+
+ def itervalues(self):
+ for k, v in self.itervalues():
+ yield v
+
+ def values(self):
+ return list(self.itervalues())
+
+ def __delitem__(self, k):
+ try:
+ delattr(self, "_val_" + k)
+ except AttributeError:
+ raise KeyError(k)
+
+ def __setitem__(self, k, v):
+ setattr(self, "_val_" + k, v)
+
+ def setdefault(self, key, default=None):
+ try:
+ return self[key]
+ except KeyError:
+ self[key] = default
+ return default
+
+ def update(self, d):
+ i = getattr(d, "iteritems", None)
+ if i is None:
+ i = d
+ else:
+ i = i()
+ for k, v in i:
+ self[k] = v
+
+ def __getitem__(self, k):
+ try:
+ return getattr(self, "_val_" + k)
+ except AttributeError:
+ raise KeyError(k)
+
+ def get(self, key, default=None):
+ try:
+ return self[key]
+ except KeyError:
+ return default
+
+ def __contains__(self, k):
+ return hasattr(self, "_val_" + k)
+
+ def has_key(self, k):
+ return k in self
+
+ def pop(self, key, *args):
+ if len(args) > 1:
+ raise TypeError(
+ "pop expected at most 2 arguments, got " + \
+ repr(1 + len(args)))
+ try:
+ value = self[key]
+ except KeyError:
+ if args:
+ return args[0]
+ raise
+ del self[key]
+ return value
+
+ def popitem(self):
+ try:
+ k, v = self.iteritems().next()
+ except StopIteration:
+ raise KeyError, 'container is empty'
+ del self[k]
+ return (k, v)
+
+ def copy(self):
+ c = self.__class__()
+ c.update(self)
+ return c
+
+ def clear(self):
+ for k in self._keys:
+ try:
+ delattr(self, "_val_" + k)
+ except AttributError:
+ pass
+
+ v = SlotDict
+ _slot_dict_classes[keys_set] = v
+ return v
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index fe8b372c2..b3c36a23b 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -2,6 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
# $Id$
+from portage.cache.mappings import slot_dict_class
from portage.dep import isvalidatom, isjustname, dep_getkey, match_from_list
from portage.dbapi.virtual import fakedbapi
from portage.exception import InvalidPackageName, InvalidAtom, \
@@ -33,6 +34,7 @@ class bindbapi(fakedbapi):
["CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS",
"LICENSE", "PDEPEND", "PROVIDE",
"RDEPEND", "repository", "RESTRICT", "SLOT", "USE"])
+ self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys)
self._aux_cache = {}
def match(self, *pargs, **kwargs):
@@ -75,7 +77,7 @@ class bindbapi(fakedbapi):
if not mydata.setdefault("EAPI", "0"):
mydata["EAPI"] = "0"
if cache_me:
- aux_cache = {}
+ aux_cache = self._aux_cache_slot_dict()
for x in self._aux_cache_keys:
aux_cache[x] = mydata.get(x, "")
self._aux_cache[mycpv] = aux_cache
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 3c27c2d95..b6e39f63b 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -3,6 +3,7 @@
# $Id$
from portage.cache.cache_errors import CacheError
+from portage.cache.mappings import slot_dict_class
from portage.const import REPO_NAME_LOC
from portage.data import portage_gid, secpass
from portage.dbapi import dbapi
@@ -138,6 +139,10 @@ class portdbapi(dbapi):
["DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE",
"PDEPEND", "PROVIDE", "RDEPEND", "repository",
"RESTRICT", "SLOT"])
+
+ # Repoman modifies _aux_cache_keys, so delay _aux_cache_slot_dict
+ # initialization until the first aux_get call.
+ self._aux_cache_slot_dict = None
self._aux_cache = {}
self._broken_ebuilds = set()
@@ -386,7 +391,10 @@ class portdbapi(dbapi):
returnme.append(mydata.get(x,""))
if cache_me:
- aux_cache = {}
+ if self._aux_cache_slot_dict is None:
+ self._aux_cache_slot_dict = \
+ slot_dict_class(self._aux_cache_keys)
+ aux_cache = self._aux_cache_slot_dict()
for x in self._aux_cache_keys:
aux_cache[x] = mydata.get(x, "")
self._aux_cache[mycpv] = aux_cache
@@ -866,4 +874,3 @@ class portagetree(object):
except Exception, e:
pass
return myslot
-