diff options
| -rwxr-xr-x | bin/portageq | 35 | ||||
| -rw-r--r-- | pym/_emerge/__init__.py | 15 | ||||
| -rw-r--r-- | pym/portage/dbapi/vartree.py | 293 |
3 files changed, 264 insertions, 79 deletions
diff --git a/bin/portageq b/bin/portageq index c3fe8b37d..fe5260bc0 100755 --- a/bin/portageq +++ b/bin/portageq @@ -188,26 +188,21 @@ def owners(argv): return 2 files.append(f[len(root):]) - found_owner = False - for cpv in vardb.cpv_all(): - cat, pkg = catsplit(cpv) - mylink = dblink(cat, pkg, root, settings, vartree=vardb.vartree) - myfiles = [] - for f in files: - if mylink.isowner(f, root): - myfiles.append(f) - if myfiles: - found_owner = True - sys.stdout.write("%s\n" % cpv) - for f in myfiles: - sys.stdout.write("\t%s\n" % \ - os.path.join(root, f.lstrip(os.path.sep))) - sys.stdout.flush() - if not found_owner: - sys.stderr.write("None of the installed packages claim the file(s).\n") - sys.stderr.flush() - return 1 - return 0 + owners = vardb._owners.get_owners(files) + + for pkg, owned_files in owners.iteritems(): + cpv = pkg.mycpv + sys.stdout.write("%s\n" % cpv) + for f in sorted(owned_files): + sys.stdout.write("\t%s\n" % \ + os.path.join(root, f.lstrip(os.path.sep))) + if owners: + sys.stdout.flush() + return 0 + + sys.stderr.write("None of the installed packages claim the file(s).\n") + sys.stderr.flush() + return 1 owners.uses_root = True diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 520a86104..68f976cbe 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -2560,6 +2560,7 @@ class depgraph(object): myroot = self.target_root dbs = self._filtered_trees[myroot]["dbs"] vardb = self.trees[myroot]["vartree"].dbapi + real_vardb = self._trees_orig[myroot]["vartree"].dbapi portdb = self.trees[myroot]["porttree"].dbapi bindb = self.trees[myroot]["bintree"].dbapi pkgsettings = self.pkgsettings[myroot] @@ -2638,16 +2639,12 @@ class depgraph(object): " $ROOT.\n") % x, noiselevel=-1) return 0, [] relative_path = x[len(myroot):] - vartree = self._trees_orig[myroot]["vartree"] owner_cpv = None - for cpv in vardb.cpv_all(): - self.spinner.update() - cat, pf = portage.catsplit(cpv) - if portage.dblink(cat, pf, myroot, - pkgsettings, vartree=vartree).isowner( - relative_path, myroot): - owner_cpv = cpv - break + for pkg, relative_path in \ + real_vardb._owners.iter_owners([relative_path]): + owner_cpv = pkg.mycpv + break + if owner_cpv is None: portage.writemsg(("\n\n!!! '%s' is not claimed " + \ "by any package.\n") % x, noiselevel=-1) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index f2593be62..fd28a6705 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -238,6 +238,9 @@ class vardbapi(dbapi): _excluded_dirs = re.compile(r'^(\..*|-MERGING-.*|' + \ "|".join(_excluded_dirs) + r')$') + _aux_cache_version = "1" + _owners_cache_version = "1" + # Number of uncached packages to trigger cache update, since # it's wasteful to update it for every vdb change. _aux_cache_threshold = 5 @@ -275,8 +278,7 @@ class vardbapi(dbapi): "EAPI", "HOMEPAGE", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND", "repository", "RESTRICT" , "SLOT", "USE"]) - self._aux_cache = None - self._aux_cache_version = "1" + self._aux_cache_obj = None self._aux_cache_filename = os.path.join(self.root, CACHE_PATH.lstrip(os.path.sep), "vdb_metadata.pickle") self._counter_path = os.path.join(root, @@ -290,6 +292,7 @@ class vardbapi(dbapi): self.plib_registry = None self.linkmap = LinkageMap(self) + self._owners = self._owners_db(self) def getpath(self, mykey, filename=None): rValue = os.path.join(self.root, VDB_PATH, mykey) @@ -562,6 +565,7 @@ class vardbapi(dbapi): if self._aux_cache is not None and \ len(self._aux_cache["modified"]) >= self._aux_cache_threshold and \ secpass >= 2: + self._owners.populate() # index any unindexed contents valid_nodes = set(self.cpv_all()) for cpv in self._aux_cache["packages"].keys(): if cpv not in valid_nodes: @@ -577,6 +581,54 @@ class vardbapi(dbapi): pass self._aux_cache["modified"] = set() + @property + def _aux_cache(self): + if self._aux_cache_obj is None: + self._aux_cache_init() + return self._aux_cache_obj + + def _aux_cache_init(self): + try: + f = open(self._aux_cache_filename) + mypickle = cPickle.Unpickler(f) + mypickle.find_global = None + aux_cache = mypickle.load() + f.close() + del f + except (IOError, OSError, EOFError, cPickle.UnpicklingError), e: + if isinstance(e, cPickle.UnpicklingError): + writemsg("!!! Error loading '%s': %s\n" % \ + (self._aux_cache_filename, str(e)), noiselevel=-1) + del e + + if not aux_cache or \ + not isinstance(aux_cache, dict) or \ + aux_cache.get("version") != self._aux_cache_version or \ + not aux_cache.get("packages"): + aux_cache = {"version": self._aux_cache_version} + aux_cache["packages"] = {} + + owners = aux_cache.get("owners") + if owners is not None: + if not isinstance(owners, dict): + owners = None + elif "version" not in owners: + owners = None + elif owners["version"] != self._owners_cache_version: + owners = None + elif "base_names" not in owners: + owners = None + + if owners is None: + owners = { + "base_names" : {}, + "version" : self._owners_cache_version + } + aux_cache["owners"] = owners + + aux_cache["modified"] = set() + self._aux_cache_obj = aux_cache + def aux_get(self, mycpv, wants): """This automatically caches selected keys that are frequently needed by emerge for dependency calculations. The cached metadata is @@ -601,26 +653,6 @@ class vardbapi(dbapi): cache_these = set(self._aux_cache_keys) cache_these.update(cache_these_wants) - if self._aux_cache is None: - try: - f = open(self._aux_cache_filename) - mypickle = cPickle.Unpickler(f) - mypickle.find_global = None - self._aux_cache = mypickle.load() - f.close() - del f - except (IOError, OSError, EOFError, cPickle.UnpicklingError), e: - if isinstance(e, cPickle.UnpicklingError): - writemsg("!!! Error loading '%s': %s\n" % \ - (self._aux_cache_filename, str(e)), noiselevel=-1) - del e - if not self._aux_cache or \ - not isinstance(self._aux_cache, dict) or \ - self._aux_cache.get("version") != self._aux_cache_version or \ - not self._aux_cache.get("packages"): - self._aux_cache = {"version": self._aux_cache_version} - self._aux_cache["packages"] = {} - self._aux_cache["modified"] = set() mydir = self.getpath(mycpv) mydir_stat = None try: @@ -801,6 +833,173 @@ class vardbapi(dbapi): write_atomic(self._counter_path, str(counter)) return counter + def _dblink(self, cpv): + category, pf = catsplit(cpv) + return dblink(category, pf, self.root, + self.settings, vartree=self.vartree) + + class _owners_cache(object): + """ + This class maintains an hash table that serves to index package + contents by mapping the basename of file to a list of possible + packages that own it. This is used to optimize owner lookups + by narrowing the search down to a smaller number of packages. + """ + try: + from hashlib import md5 as _new_hash + except ImportError: + from md5 import new as _new_hash + + _hash_bits = 16 + _hex_chars = _hash_bits / 4 + + def __init__(self, vardb): + self._vardb = vardb + + def add(self, cpv): + root_len = len(self._vardb.root) + contents = self._vardb._dblink(cpv).getcontents() + pkg_hash = self._hash_pkg(cpv) + if not contents: + # Empty path is a code used to represent empty contents. + self._add_path("", pkg_hash) + for x in contents: + relative_path = x[root_len:] + self._add_path(x, pkg_hash) + self._vardb._aux_cache["modified"].add(cpv) + + def _add_path(self, path, pkg_hash): + """ + Empty path is a code that represents empty contents. + """ + if path: + name = os.path.basename(path.rstrip(os.path.sep)) + if not name: + return + else: + name = path + name_hash = self._hash_str(name) + base_names = self._vardb._aux_cache["owners"]["base_names"] + pkgs = base_names.get(name_hash) + if pkgs is None: + pkgs = {} + base_names[name_hash] = pkgs + pkgs[pkg_hash] = None + + def _hash_str(self, s): + h = self._new_hash() + h.update(s) + h = h.hexdigest() + h = h[-self._hex_chars:] + h = int(h, 16) + return h + + def _hash_pkg(self, cpv): + counter, mtime = self._vardb.aux_get( + cpv, ["COUNTER", "_mtime_"]) + try: + counter = int(counter) + except ValueError: + counter = 0 + return (cpv, counter, mtime) + + class _owners_db(object): + + def __init__(self, vardb): + self._vardb = vardb + + def populate(self): + self._populate() + + def _populate(self): + owners_cache = vardbapi._owners_cache(self._vardb) + cached_hashes = set() + base_names = self._vardb._aux_cache["owners"]["base_names"] + + # Take inventory of all cached package hashes. + for hash_values in base_names.itervalues(): + cached_hashes.update(hash_values) + + # Create sets of valid package hashes and uncached packages. + uncached_pkgs = set() + hash_pkg = owners_cache._hash_pkg + valid_pkg_hashes = set() + for cpv in self._vardb.cpv_all(): + hash_value = hash_pkg(cpv) + valid_pkg_hashes.add(hash_value) + if hash_value not in cached_hashes: + uncached_pkgs.add(cpv) + + # Cache any missing packages. + for cpv in uncached_pkgs: + owners_cache.add(cpv) + + # Delete any stale cache. + stale_hashes = cached_hashes.difference(valid_pkg_hashes) + if stale_hashes: + for base_name_hash, bucket in base_names.items(): + for hash_value in stale_hashes.intersection(bucket): + del bucket[hash_value] + if not bucket: + del base_names[base_name_hash] + + return owners_cache + + def get_owners(self, path_iter): + """ + @return the owners as a dblink -> set(files) mapping. + """ + owners = {} + for owner, f in self.iter_owners(path_iter): + owned_files = owners.get(owner) + if owned_files is None: + owned_files = set() + owners[owner] = owned_files + owned_files.add(f) + return owners + + def iter_owners(self, path_iter): + """ + Iterate over tuples of (dblink, path). In order to avoid + consuming too many resources for too much time, resources + are only allocated for the duration of a given iter_owners() + call. Therefore, to maximize reuse of resources when searching + for multiple files, it's best to search for them all in a single + call. + """ + + owners_cache = self._populate() + + vardb = self._vardb + root = vardb.root + hash_pkg = owners_cache._hash_pkg + hash_str = owners_cache._hash_str + base_names = self._vardb._aux_cache["owners"]["base_names"] + + dblink_cache = {} + + def dblink(cpv): + x = dblink_cache.get(cpv) + if x is None: + x = self._vardb._dblink(cpv) + dblink_cache[cpv] = x + return x + + for path in path_iter: + name = os.path.basename(path.rstrip(os.path.sep)) + if not name: + continue + + name_hash = hash_str(name) + pkgs = base_names.get(name_hash) + if pkgs is not None: + for hash_value in pkgs: + cpv, counter, mtime = hash_value + if hash_pkg(cpv) != hash_value: + continue + if dblink(cpv).isowner(path, root): + yield dblink(cpv), path + class vartree(object): "this tree will scan a var/db/pkg database located at root (passed to init)" def __init__(self, root="/", virtual=None, clone=None, categories=None, @@ -2202,36 +2401,30 @@ class dblink(object): eerror(msg) - if collision_protect: + msg = [] + msg.append("") + msg.append("Searching all installed" + \ + " packages for file collisions...") + msg.append("") + msg.append("Press Ctrl-C to Stop") + msg.append("") + eerror(msg) + + owners = self.vartree.dbapi._owners.get_owners(files) + self.vartree.dbapi.flush_cache() + + for pkg, owned_files in owners.iteritems(): + cpv = pkg.mycpv msg = [] - msg.append("") - msg.append("Searching all installed" + \ - " packages for file collisions...") - msg.append("") - msg.append("Press Ctrl-C to Stop") - msg.append("") + msg.append("%s" % cpv) + for f in sorted(owned_files): + msg.append("\t%s" % os.path.join(destroot, + f.lstrip(os.path.sep))) eerror(msg) - - found_owner = False - for cpv in self.vartree.dbapi.cpv_all(): - cat, pkg = catsplit(cpv) - mylink = dblink(cat, pkg, destroot, self.settings, - vartree=self.vartree) - mycollisions = [] - for f in collisions: - if mylink.isowner(f, destroot): - mycollisions.append(f) - if mycollisions: - found_owner = True - msg = [] - msg.append("%s" % cpv) - for f in mycollisions: - msg.append("\t%s" % os.path.join(destroot, - f.lstrip(os.path.sep))) - eerror(msg) - if not found_owner: - eerror(["None of the installed" + \ - " packages claim the file(s)."]) + if not owners: + eerror(["None of the installed" + \ + " packages claim the file(s)."]) + if collision_protect: return 1 writemsg_stdout(">>> Merging %s to %s\n" % (self.mycpv, destroot)) |