From 04066d484cf68bda8de37c037aad003a44088e8a Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 25 Jun 2008 22:36:19 +0000 Subject: Add a generic portage.cache.mappings.slot_dict_class() function which generates mapping classes that behave similar to a dict but store values as object attributes that are allocated via __slots__. Instances of these objects have a smaller memory footprint than a normal dict object. These classes are used to reduce the memory footprint of the dbapi.aux_get() caches and the Package.metadata attribute. svn path=/main/trunk/; revision=10790 --- pym/_emerge/__init__.py | 101 +++++---------------------------- pym/portage/cache/mappings.py | 126 ++++++++++++++++++++++++++++++++++++++++++ pym/portage/dbapi/bintree.py | 4 +- pym/portage/dbapi/porttree.py | 11 +++- 4 files changed, 151 insertions(+), 91 deletions(-) diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 2f54b6b65..ca76400a3 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1371,108 +1371,33 @@ class Package(Task): return True return False -class _PackageMetadataWrapper(object): +_all_metadata_keys = set(x for x in portage.auxdbkeys \ + if not x.startswith("UNUSED_")) +_all_metadata_keys.discard("CDEPEND") +_all_metadata_keys.update(Package.metadata_keys) + +from portage.cache.mappings import slot_dict_class +_PackageMetadataWrapperBase = slot_dict_class(_all_metadata_keys) + +class _PackageMetadataWrapper(_PackageMetadataWrapperBase): """ Detect metadata updates and synchronize Package attributes. """ - _keys = set(x for x in portage.auxdbkeys \ - if not x.startswith("UNUSED_")) - _keys.discard("CDEPEND") - _keys.update(Package.metadata_keys) - _keys = tuple(sorted(_keys)) - __slots__ = ("__weakref__", "_pkg") + tuple("_val_" + k for k in _keys) + + __slots__ = ("_pkg",) _wrapped_keys = frozenset( ["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"]) def __init__(self, pkg, metadata): + _PackageMetadataWrapperBase.__init__(self) self._pkg = pkg self.update(metadata) - def __iter__(self): - for k, v in self.iteritems(): - yield k - - def __len__(self): - l = 0 - for i in self.iteritems(): - l += 1 - return l - - def keys(self): - return list(self) - - def iteritems(self): - for k in self._keys: - try: - yield (k, getattr(self, "_val_" + k)) - except AttributeError: - pass - - def items(self): - return list(self.iteritems()) - - def itervalues(self): - for k, v in self.itervalues(): - yield v - - def values(self): - return list(self.itervalues()) - - def __delitem__(self, k): - try: - delattr(self, "_val_" + k) - except AttributeError: - raise KeyError(k) - def __setitem__(self, k, v): - setattr(self, "_val_" + k, v) + _PackageMetadataWrapperBase.__setitem__(self, k, v) if k in self._wrapped_keys: getattr(self, "_set_" + k.lower())(k, v) - def update(self, d): - i = getattr(d, "iteritems", None) - if i is None: - i = d - else: - i = i() - for k, v in i: - self[k] = v - - def __getitem__(self, k): - try: - return getattr(self, "_val_" + k) - except AttributeError: - raise KeyError(k) - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def __contains__(self, k): - return hasattr(self, "_val_" + k) - - def pop(self, key, *args): - if len(args) > 1: - raise TypeError("pop expected at most 2 arguments, got " + \ - repr(1 + len(args))) - try: - value = self[key] - except KeyError: - if args: - return args[0] - raise - del self[key] - return value - - def clear(self): - for k in self._keys: - try: - delattr(self, "_val_" + k) - except AttributError: - pass - def _set_inherited(self, k, v): if isinstance(v, basestring): v = frozenset(v.split()) diff --git a/pym/portage/cache/mappings.py b/pym/portage/cache/mappings.py index 9aa5a21e2..7c347a4f9 100644 --- a/pym/portage/cache/mappings.py +++ b/pym/portage/cache/mappings.py @@ -4,6 +4,7 @@ # $Id$ import UserDict +import weakref class ProtectedDict(UserDict.DictMixin): """ @@ -101,3 +102,128 @@ class LazyLoad(UserDict.DictMixin): self.pull = None return key in self.d +_slot_dict_classes = weakref.WeakValueDictionary() + +def slot_dict_class(keys): + if isinstance(keys, frozenset): + keys_set = keys + else: + keys_set = frozenset(keys) + v = _slot_dict_classes.get(keys_set) + if v is None: + + class SlotDict(object): + + _keys = keys_set + __slots__ = ("__weakref__",) + tuple("_val_" + k for k in _keys) + + def __iter__(self): + for k, v in self.iteritems(): + yield k + + def __len__(self): + l = 0 + for i in self.iteritems(): + l += 1 + return l + + def keys(self): + return list(self) + + def iteritems(self): + for k in self._keys: + try: + yield (k, getattr(self, "_val_" + k)) + except AttributeError: + pass + + def items(self): + return list(self.iteritems()) + + def itervalues(self): + for k, v in self.itervalues(): + yield v + + def values(self): + return list(self.itervalues()) + + def __delitem__(self, k): + try: + delattr(self, "_val_" + k) + except AttributeError: + raise KeyError(k) + + def __setitem__(self, k, v): + setattr(self, "_val_" + k, v) + + def setdefault(self, key, default=None): + try: + return self[key] + except KeyError: + self[key] = default + return default + + def update(self, d): + i = getattr(d, "iteritems", None) + if i is None: + i = d + else: + i = i() + for k, v in i: + self[k] = v + + def __getitem__(self, k): + try: + return getattr(self, "_val_" + k) + except AttributeError: + raise KeyError(k) + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def __contains__(self, k): + return hasattr(self, "_val_" + k) + + def has_key(self, k): + return k in self + + def pop(self, key, *args): + if len(args) > 1: + raise TypeError( + "pop expected at most 2 arguments, got " + \ + repr(1 + len(args))) + try: + value = self[key] + except KeyError: + if args: + return args[0] + raise + del self[key] + return value + + def popitem(self): + try: + k, v = self.iteritems().next() + except StopIteration: + raise KeyError, 'container is empty' + del self[k] + return (k, v) + + def copy(self): + c = self.__class__() + c.update(self) + return c + + def clear(self): + for k in self._keys: + try: + delattr(self, "_val_" + k) + except AttributError: + pass + + v = SlotDict + _slot_dict_classes[keys_set] = v + return v diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index fe8b372c2..b3c36a23b 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +from portage.cache.mappings import slot_dict_class from portage.dep import isvalidatom, isjustname, dep_getkey, match_from_list from portage.dbapi.virtual import fakedbapi from portage.exception import InvalidPackageName, InvalidAtom, \ @@ -33,6 +34,7 @@ class bindbapi(fakedbapi): ["CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", "repository", "RESTRICT", "SLOT", "USE"]) + self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys) self._aux_cache = {} def match(self, *pargs, **kwargs): @@ -75,7 +77,7 @@ class bindbapi(fakedbapi): if not mydata.setdefault("EAPI", "0"): mydata["EAPI"] = "0" if cache_me: - aux_cache = {} + aux_cache = self._aux_cache_slot_dict() for x in self._aux_cache_keys: aux_cache[x] = mydata.get(x, "") self._aux_cache[mycpv] = aux_cache diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 3c27c2d95..b6e39f63b 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -3,6 +3,7 @@ # $Id$ from portage.cache.cache_errors import CacheError +from portage.cache.mappings import slot_dict_class from portage.const import REPO_NAME_LOC from portage.data import portage_gid, secpass from portage.dbapi import dbapi @@ -138,6 +139,10 @@ class portdbapi(dbapi): ["DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", "repository", "RESTRICT", "SLOT"]) + + # Repoman modifies _aux_cache_keys, so delay _aux_cache_slot_dict + # initialization until the first aux_get call. + self._aux_cache_slot_dict = None self._aux_cache = {} self._broken_ebuilds = set() @@ -386,7 +391,10 @@ class portdbapi(dbapi): returnme.append(mydata.get(x,"")) if cache_me: - aux_cache = {} + if self._aux_cache_slot_dict is None: + self._aux_cache_slot_dict = \ + slot_dict_class(self._aux_cache_keys) + aux_cache = self._aux_cache_slot_dict() for x in self._aux_cache_keys: aux_cache[x] = mydata.get(x, "") self._aux_cache[mycpv] = aux_cache @@ -866,4 +874,3 @@ class portagetree(object): except Exception, e: pass return myslot - -- cgit v1.2.3-1-g7c22