From d9785f96de5f1697b5f87def919dc358cc2cc30d Mon Sep 17 00:00:00 2001 From: Brian Harring Date: Sat, 5 Nov 2005 06:13:33 +0000 Subject: integration chunk of the cache subsystem patch. Will deal with UNUSED_0* after this svn path=/main/branches/2.0/; revision=2258 --- bin/emerge | 127 ++++++++++++---------------- bin/repoman | 2 +- pym/eclass_cache.py | 75 +++++++++++++++++ pym/portage.py | 236 +++++++++++----------------------------------------- 4 files changed, 176 insertions(+), 264 deletions(-) create mode 100644 pym/eclass_cache.py diff --git a/bin/emerge b/bin/emerge index 6ea0aceb0..3a38f66a0 100755 --- a/bin/emerge +++ b/bin/emerge @@ -2671,14 +2671,12 @@ if myaction in ["sync","rsync","metadata"] and (not "--help" in myopts): portage.portdb.flush_cache() - try: - os.umask(002) - os.chown(cachedir, os.getuid(), portage.portage_gid) - os.chmod(cachedir, 02775) - except SystemExit, e: - raise # Needed else can't exit - except: - pass + ec = portage.eclass_cache.cache(portage.portdb.porttree_root) + # kinda ugly. + # XXX: nuke the filter when mr UNUSED_0? keys are dead + cm = portage.settings.load_best_module("portdbapi.metadbmodule")(myportdir, "metadata/cache", + filter(lambda x: not x.startswith("UNUSED_0"), portage.auxdbkeys)) + # we don't make overlay trees cache here, plus we don't trust portage.settings.categories porttree_root = portage.portdb.porttree_root conf = portage.config(config_profile_path=portage.settings.profile_path[:], \ @@ -2694,75 +2692,54 @@ if myaction in ["sync","rsync","metadata"] and (not "--help" in myopts): pass pdb = portage.portdbapi(porttree_root, conf) - cp_list = pdb.cp_all() - if len(cp_list) == 0: - print "no metadata to transfer, exiting" - sys.exit(0) - cp_list.sort() - pcnt=0 - pcntstr="" - pcntcount=len(cp_list)/100.0 - nextupdate=pcntcount - current=1 - - def cleanse_cache(pdb, cat, saves, porttree_root=porttree_root): - try: - if len(saves): - d={} - for v in saves: - d[portage.catsplit(v)[1]] = True - for pv in pdb.auxdb[porttree_root][cat].keys(): - if pv not in d: - pdb.auxdb[porttree_root][cat].del_key(pv) - else: - try: - pdb.auxdb[porttree_root][cat].clear() - del pdb.auxdb[porttree_root][cat] - except KeyError: - pass - except KeyError: - # stop breaking things, cleansing is minor. - pass - savelist = [] - catlist = [] - oldcat = portage.catsplit(cp_list[0])[0] - for cp in cp_list: - current += 1 - if current >= nextupdate: - pcnt += 1 - nextupdate += pcntcount - if "--quiet" not in myopts: - pcntstr = str(pcnt) - sys.stdout.write("\b"*(len(pcntstr)+1)+pcntstr+"%") - sys.stdout.flush() - cat = portage.catsplit(cp)[0] - if cat != oldcat: - catlist.append(oldcat) - cleanse_cache(pdb, oldcat, savelist) - savelist = [] - oldcat = cat - mymatches = pdb.xmatch("match-all", cp) - savelist.extend(mymatches) - for cpv in mymatches: - try: pdb.aux_get(cpv, ["IUSE"],metacachedir=myportdir+"/metadata/cache",debug=("cachedebug" in portage.features)) - except SystemExit: raise - except Exception, e: print "\nFailed cache update:",cpv,e - catlist.append(oldcat) - catlist.append("local") - cleanse_cache(pdb, oldcat, savelist) - filelist = portage.listdir(cachedir+"/"+myportdir) - for x in filelist: - found = False - for y in catlist: - if x.startswith(y): - found = True - break - if not found: - portage.spawn("cd /; rm -Rf "+cachedir+"/"+myportdir+"/"+x,portage.settings,free=1,droppriv=1) + cp_all_list = pdb.cp_all() + import cache.util + + class percentage_noise_maker(cache.util.quiet_mirroring): + def __init__(self, dbapi): + self.dbapi = dbapi + self.cp_all = dbapi.cp_all() + self.cp_all.sort() + l = len(self.cp_all) + self.call_update_min = 100000000 + self.min_cp_all = l/100.0 + self.count = 1 + self.pstr = '' + + def __iter__(self): + for x in self.cp_all: + self.count += 1 + if self.count > self.min_cp_all: + self.call_update_min = 0 + self.count = 0 + for y in self.dbapi.cp_list(x): + yield y + self.call_update_mine = 0 + + def update(self, *arg): + try: self.pstr = int(self.pstr) + 1 + except ValueError: self.pstr = 1 + sys.stdout.write("%s%i%%" % ("\b" * (len(str(self.pstr))+1), self.pstr)) + sys.stdout.flush() + self.call_update_min = 10000000 + + def finish(self, *arg): + sys.stdout.write("\b\b\b\b100%\n") + sys.stdout.flush() - sys.stdout.write("\n\n") + if "--quiet" in myopts: + def quicky_cpv_generator(cp_all_list): + for x in cp_all_list: + for y in pdb.cp_list(x): + yield y + source = quicky_cpv_generator(pdb.cp_all()) + noise_maker = cache.util.quiet_mirroring() + else: + noise_maker = source = percentage_noise_maker(pdb) + cache.util.mirror_cache(source, cm, pdb.auxdb[porttree_root], eclass_cache=ec, verbose_instance=noise_maker) + sys.stdout.flush() portage.portageexit() @@ -2796,7 +2773,7 @@ elif myaction=="regen": print "processing",x for y in mymatches: try: - foo=portage.portdb.aux_get(y,["DEPEND"],debug=1) + foo=portage.portdb.aux_get(y,["DEPEND"]) except SystemExit, e: # sys.exit is an exception... And consequently, we can't catch it. raise diff --git a/bin/repoman b/bin/repoman index 9e4afb231..49578ad59 100755 --- a/bin/repoman +++ b/bin/repoman @@ -898,7 +898,7 @@ for x in scanlist: fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild") continue try: - myaux=portage.db["/"]["porttree"].dbapi.aux_get(catdir+"/"+y,allvars,strict=1) + myaux=portage.db["/"]["porttree"].dbapi.aux_get(catdir+"/"+y,allvars) except KeyError: stats["ebuild.syntax"]=stats["ebuild.syntax"]+1 fails["ebuild.syntax"].append(x+"/"+y+".ebuild") diff --git a/pym/eclass_cache.py b/pym/eclass_cache.py new file mode 100644 index 000000000..066162611 --- /dev/null +++ b/pym/eclass_cache.py @@ -0,0 +1,75 @@ +# Copyright: 2005 Gentoo Foundation +# Author(s): Nicholas Carpaski (carpaski@gentoo.org), Brian Harring (ferringb@gentoo.org) +# License: GPL2 +# $Id:$ + +from portage_util import writemsg +import portage_file +import os, sys +from portage_data import portage_gid + +class cache: + """ + Maintains the cache information about eclasses used in ebuild. + """ + def __init__(self, porttree_root, overlays=[]): + self.porttree_root = porttree_root + + self.eclasses = {} # {"Name": ("location","_mtime_")} + + # screw with the porttree ordering, w/out having bash inherit match it, and I'll hurt you. + # ~harring + self.porttrees = [self.porttree_root]+overlays + self.porttrees = tuple(map(portage_file.normpath, self.porttrees)) + self._master_eclass_root = os.path.join(self.porttrees[0],"eclass") + self.update_eclasses() + + def close_caches(self): + import traceback + traceback.print_stack() + print "%s close_cache is deprecated" % self.__class__ + self.eclasses.clear() + + def flush_cache(self): + import traceback + traceback.print_stack() + print "%s flush_cache is deprecated" % self.__class__ + + self.update_eclasses() + + def update_eclasses(self): + self.eclasses = {} + eclass_len = len(".eclass") + for x in [portage_file.normpath(os.path.join(y,"eclass")) for y in self.porttrees]: + if not os.path.isdir(x): + continue + for y in [y for y in os.listdir(x) if y.endswith(".eclass")]: + try: + mtime=os.stat(x+"/"+y).st_mtime + except OSError: + continue + ys=y[:-eclass_len] + self.eclasses[ys] = (x, long(mtime)) + + def is_eclass_data_valid(self, ec_dict): + if not isinstance(ec_dict, dict): + return False + for eclass, tup in ec_dict.iteritems(): + if eclass not in self.eclasses or tuple(tup) != self.eclasses[eclass]: + return False + + return True + + def get_eclass_data(self, inherits, from_master_only=False): + ec_dict = {} + for x in inherits: + try: + ec_dict[x] = self.eclasses[x] + except: + print "ec=",ec_dict + print "inherits=",inherits + raise + if from_master_only and self.eclasses[x][0] != self._master_eclass_root: + return None + + return ec_dict diff --git a/pym/portage.py b/pym/portage.py index 473b0c4ac..7e305ea46 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -28,6 +28,7 @@ try: import commands from time import sleep from random import shuffle + from cache.cache_errors import CacheError except SystemExit, e: raise except Exception, e: @@ -97,6 +98,7 @@ try: from portage_locks import unlockfile,unlockdir,lockfile,lockdir import portage_checksum from portage_checksum import perform_md5,perform_checksum,prelink_capable + import eclass_cache from portage_localization import _ except SystemExit, e: raise @@ -921,9 +923,8 @@ class config: if self.modules["user"] == None: self.modules["user"] = {} self.modules["default"] = { - "portdbapi.metadbmodule": "portage_db_metadata.database", - "portdbapi.auxdbmodule": "portage_db_flat.database", - "eclass_cache.dbmodule": "portage_db_cpickle.database", + "portdbapi.metadbmodule": "cache.metadata.database", + "portdbapi.auxdbmodule": "cache.flat_hash.database", } self.usemask=[] @@ -5017,116 +5018,6 @@ class vartree(packagetree): def populate(self): self.populated=1 -# ---------------------------------------------------------------------------- -class eclass_cache: - """Maintains the cache information about eclasses used in ebuild.""" - def __init__(self,porttree_root,settings): - self.porttree_root = porttree_root - self.settings = settings - self.depcachedir = self.settings.depcachedir[:] - - self.dbmodule = self.settings.load_best_module("eclass_cache.dbmodule") - - self.packages = {} # {"PV": {"eclass1": ["location", "_mtime_"]}} - self.eclasses = {} # {"Name": ["location","_mtime_"]} - - # don't fool with porttree ordering unless you *ensure* that ebuild.sh's inherit - # ordering is *exactly* the same - self.porttrees=[self.porttree_root] - self.porttrees.extend(self.settings["PORTDIR_OVERLAY"].split()) - #normalize the path now, so it's not required later. - self.porttrees = [os.path.normpath(x) for x in self.porttrees] - self.update_eclasses() - - def close_caches(self): - for x in self.packages.keys(): - for y in self.packages[x].keys(): - try: - self.packages[x][y].sync() - self.packages[x][y].close() - except SystemExit, e: - raise - except Exception,e: - writemsg("Exception when closing DB: %s: %s\n" % (Exception,e)) - del self.packages[x][y] - del self.packages[x] - - def flush_cache(self): - self.packages = {} - self.eclasses = {} - self.update_eclasses() - - def update_eclasses(self): - self.eclasses = {} - for x in suffix_array(self.porttrees, "/eclass"): - if x and os.path.exists(x): - dirlist = listdir(x) - for y in dirlist: - if y[-len(".eclass"):]==".eclass": - try: - ys=y[:-len(".eclass")] - ymtime=os.stat(x+"/"+y)[stat.ST_MTIME] - except SystemExit, e: - raise - except: - continue - self.eclasses[ys] = [x, ymtime] - - def setup_package(self, location, cat, pkg): - if not self.packages.has_key(location): - self.packages[location] = {} - - if not self.packages[location].has_key(cat): - try: - self.packages[location][cat] = self.dbmodule(self.depcachedir+"/"+location, cat+"-eclass", [], uid, portage_gid) - except SystemExit, e: - raise - except Exception, e: - writemsg("\n!!! Failed to open the dbmodule for eclass caching.\n") - writemsg("!!! Generally these are permission problems. Caught exception follows:\n") - writemsg("!!! "+str(e)+"\n") - writemsg("!!! Dirname: "+str(self.depcachedir+"/"+location)+"\n") - writemsg("!!! Basename: "+str(cat+"-eclass")+"\n\n") - sys.exit(123) - - def sync(self, location, cat, pkg): - if self.packages[location].has_key(cat): - self.packages[location][cat].sync() - - def update_package(self, location, cat, pkg, eclass_list): - self.setup_package(location, cat, pkg) - if not eclass_list: - return 1 - - data = {} - for x in eclass_list: - if x not in self.eclasses: - writemsg("Eclass '%s' does not exist for '%s'\n" % (x, cat+"/"+pkg)) - return 0 - data[x] = [self.eclasses[x][0],self.eclasses[x][1]] - - self.packages[location][cat][pkg] = data - self.sync(location,cat,pkg) - return 1 - - def is_current(self, location, cat, pkg, eclass_list): - self.setup_package(location, cat, pkg) - - if not eclass_list: - return 1 - - if not (self.packages[location][cat].has_key(pkg) and self.packages[location][cat][pkg] and eclass_list): - return 0 - - myp = self.packages[location][cat][pkg] - for x in eclass_list: - if not (x in self.eclasses and x in myp and myp[x] == self.eclasses[x]): - return 0 - - return 1 - -# ---------------------------------------------------------------------------- - auxdbkeys=[ 'DEPEND', 'RDEPEND', 'SLOT', 'SRC_URI', 'RESTRICT', 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', @@ -5140,6 +5031,8 @@ auxdbkeylen=len(auxdbkeys) def close_portdbapi_caches(): for i in portdbapi.portdbapi_instances: i.close_caches() + + class portdbapi(dbapi): """this tree will scan a portage directory located at root (passed to init)""" portdbapi_instances = [] @@ -5182,33 +5075,34 @@ class portdbapi(dbapi): if self.tmpfs and not os.access(self.tmpfs, os.R_OK): self.tmpfs = None - self.eclassdb = eclass_cache(self.porttree_root, self.mysettings) + self.eclassdb = eclass_cache.cache(self.porttree_root, overlays=settings["PORTDIR_OVERLAY"].split()) self.metadb = {} self.metadbmodule = self.mysettings.load_best_module("portdbapi.metadbmodule") - self.auxdb = {} - self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule") - #if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening) self.xcache={} self.frozen=0 self.porttrees=[self.porttree_root]+self.mysettings["PORTDIR_OVERLAY"].split() + self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule") + self.auxdb = {} + # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys + # ~harring + filtered_auxdbkeys = filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys) + for x in self.porttrees: + # location, label, auxdbkeys + self.auxdb[x] = self.auxdbmodule(portage_const.DEPCACHE_PATH, x, filtered_auxdbkeys, gid=portage_gid) + def close_caches(self): for x in self.auxdb.keys(): - for y in self.auxdb[x].keys(): - self.auxdb[x][y].sync() - self.auxdb[x][y].close() - del self.auxdb[x][y] - del self.auxdb[x] - self.eclassdb.close_caches() + self.auxdb[x].sync() + self.auxdb.clear() def flush_cache(self): self.metadb = {} self.auxdb = {} - self.eclassdb.flush_cache() def finddigest(self,mycpv): try: @@ -5261,7 +5155,7 @@ class portdbapi(dbapi): # when not found return None, 0 - def aux_get(self,mycpv,mylist,strict=0,metacachedir=None,debug=0): + def aux_get(self, mycpv, mylist): "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc." 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error' @@ -5269,10 +5163,6 @@ class portdbapi(dbapi): cat,pkg = string.split(mycpv, "/", 1) - if metacachedir: - if cat not in self.metadb: - self.metadb[cat] = self.metadbmodule(metacachedir,cat,auxdbkeys,uid,portage_gid) - myebuild, mylocation=self.findname2(mycpv) if not myebuild: @@ -5314,11 +5204,6 @@ class portdbapi(dbapi): raise portage_exception.SecurityViolation, "Error in verification of signatures: %(errormsg)s" % {"errormsg":str(e)} writemsg("!!! Manifest is missing or inaccessable: %(manifest)s\n" % {"manifest":myManifestPath}) - if mylocation not in self.auxdb: - self.auxdb[mylocation] = {} - - if not self.auxdb[mylocation].has_key(cat): - self.auxdb[mylocation][cat] = self.auxdbmodule(self.depcachedir+"/"+mylocation,cat,auxdbkeys,uid,portage_gid) if os.access(myebuild, os.R_OK): emtime=os.stat(myebuild)[stat.ST_MTIME] @@ -5328,47 +5213,24 @@ class portdbapi(dbapi): raise KeyError try: - auxdb_is_valid = self.auxdb[mylocation][cat].has_key(pkg) and \ - self.auxdb[mylocation][cat][pkg].has_key("_mtime_") and \ - self.auxdb[mylocation][cat][pkg]["_mtime_"] == emtime - except SystemExit, e: - raise - except Exception, e: - auxdb_is_valid = 0 - if not metacachedir: - writemsg("auxdb exception: [%(loc)s]: %(exception)s\n" % {"loc":mylocation+"::"+cat+"/"+pkg, "exception":str(e)}) - if self.auxdb[mylocation][cat].has_key(pkg): - self.auxdb[mylocation][cat].del_key(pkg) - self.auxdb[mylocation][cat].sync() - - writemsg("auxdb is valid: "+str(auxdb_is_valid)+" "+str(pkg)+"\n", 2) - doregen = not (auxdb_is_valid and self.eclassdb.is_current(mylocation,cat,pkg,self.auxdb[mylocation][cat][pkg]["INHERITED"].split())) - - # when mylocation is not overlay directorys and metacachedir is set, - # we use cache files, which is usually on /usr/portage/metadata/cache/. - if doregen and mylocation==self.mysettings["PORTDIR"] and metacachedir and self.metadb[cat].has_key(pkg): - metadata=self.metadb[cat][pkg] - - if "EAPI" not in metadata or not metadata["EAPI"].strip(): - metadata["EAPI"] = "0" - - if not eapi_is_supported(metadata["EAPI"]): - # intentionally wipe keys. - eapi = metadata["EAPI"] - mtime = metadata.get("_mtime_", 0) - metadata = {} - map(lambda x: metadata.setdefault(x, ''), auxdbkeys) - metadata["_mtime_"] = long(mtime) - metadata["EAPI"] == "-"+eapi - + mydata = self.auxdb[mylocation][mycpv] + if emtime != long(mydata.get("_mtime_", 0)): + doregen = True + elif len(mydata.get("_eclasses_", [])) > 0: + doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"]) else: - # eclass updates only if we haven't nuked the entry. - self.eclassdb.update_package(mylocation,cat,pkg,metadata["INHERITED"].split()) + doregen = False + + except KeyError: + doregen = True + except CacheError: + doregen = True + try: del self.auxdb[mylocation][mycpv] + except KeyError: pass - self.auxdb[mylocation][cat][pkg] = metadata - self.auxdb[mylocation][cat].sync() + writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2) - elif doregen: + if doregen: writemsg("doregen: %s %s\n" % (doregen,mycpv), 2) writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n",1) @@ -5389,9 +5251,7 @@ class portdbapi(dbapi): if os.path.exists(mydbkey): try: os.unlink(mydbkey) - except SystemExit, e: - raise - except Exception, e: + except (IOError, OSError), e: portage_locks.unlockfile(mylock) self.lock_held = 0 writemsg("Uncaught handled exception: %(exception)s\n" % {"exception":str(e)}) @@ -5411,19 +5271,13 @@ class portdbapi(dbapi): os.unlink(mydbkey) mylines=mycent.readlines() mycent.close() - except SystemExit, e: - raise + except (IOError, OSError): portage_locks.unlockfile(mylock) self.lock_held = 0 writemsg(str(red("\naux_get():")+" (1) Error in "+mycpv+" ebuild.\n" " Check for syntax error or corruption in the ebuild. (--debug)\n\n")) raise KeyError - except Exception, e: - portage_locks.unlockfile(mylock) - self.lock_held = 0 - writemsg("Uncaught handled exception: %(exception)s\n" % {"exception":str(e)}) - raise portage_locks.unlockfile(mylock) self.lock_held = 0 @@ -5444,18 +5298,24 @@ class portdbapi(dbapi): map(lambda x:mydata.setdefault(x, ""), auxdbkeys) mydata["EAPI"] = "-"+eapi + if mydata.get("INHERITED", False): + mydata["_eclasses_"] = self.eclassdb.get_eclass_data(mydata["INHERITED"].split()) + else: + mydata["_eclasses_"] = {} + + del mydata["INHERITED"] + mydata["_mtime_"] = emtime - self.auxdb[mylocation][cat][pkg] = mydata - self.auxdb[mylocation][cat].sync() - if not self.eclassdb.update_package(mylocation, cat, pkg, mylines[auxdbkeys.index("INHERITED")].split()): - sys.exit(1) + self.auxdb[mylocation][mycpv] = mydata #finally, we look at our internal cache entry and return the requested data. - mydata = self.auxdb[mylocation][cat][pkg] returnme = [] for x in mylist: - returnme.append(mydata.get(x,"")) + if x == "INHERITED": + returnme.append(' '.join(mydata.get("_eclasses_", {}).keys())) + else: + returnme.append(mydata.get(x,"")) if "EAPI" in mylist: idx = mylist.index("EAPI") -- cgit v1.2.3-1-g7c22