EAPI awareness, and flat_hash/flat_list autodetection for rysnc cache.

svn path=/main/branches/2.0/; revision=2022

7 files changed, 220 insertions, 40 deletions

diff --git a/ChangeLog b/ChangeLog
index 7bc4c0ba2..953c8f252 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -7,6 +7,21 @@
     2. /var/cache/edb/world is now /var/lib/portage/world.
     3. /etc/portage/profile/virtuals is _USER_ configs only.
 
+  24 Sep 2005; Brian Harring <ferringb@gentoo.org> pym/portage.py,
+  +pym/portage_db_metadata.py, +pym/portage_db_flat_hash.py, 
+  pym/portage_db_flat.py, pym/portage_const.py:
+  EAPI awareness, and support for flat_hash metadata/cache.
+  Later autodetects if the rsync cache is flat_hash, or flat_list, 
+  and interprets accordingly.
+  
+  Flat_hash is intended to make it such that rsync cache breakages 
+  for older portage versions no longer occur; the container format 
+  can hold pretty much anything across EAPI versions.
+  Additionally... EAPI is intended such that additions to the ebuild 
+  env/helpers don't puke if you're running to old of a portage version; 
+  it allows us to 'version' the ebuild env/funcs, and do masking dependant
+  on that versioning.
+
   23 Sep 2005; Brian Harring <ferringb@gentoo.org> bin/emerge:
   cleanse_cache cleanups, should kill off all remaining issues 
   in bug #100444 .
diff --git a/bin/ebuild.sh b/bin/ebuild.sh
index 8cba9030c..d73117967 100755
--- a/bin/ebuild.sh
+++ b/bin/ebuild.sh
@@ -1898,6 +1898,7 @@ for myarg in $*; do
 		echo `echo "$CDEPEND"`     >> $dbkey
 		echo `echo "$PDEPEND"`     >> $dbkey
 		echo `echo "$PROVIDE"`     >> $dbkey
+		echo `echo "${EAPI:-0}"`   >> $dbkey
 		echo `echo "$UNUSED_01"`   >> $dbkey
 		echo `echo "$UNUSED_02"`   >> $dbkey
 		echo `echo "$UNUSED_03"`   >> $dbkey
@@ -1905,7 +1906,6 @@ for myarg in $*; do
 		echo `echo "$UNUSED_05"`   >> $dbkey
 		echo `echo "$UNUSED_06"`   >> $dbkey
 		echo `echo "$UNUSED_07"`   >> $dbkey
-		echo `echo "$UNUSED_08"`   >> $dbkey
 		set +f
 		#make sure it is writable by our group:
 		exit 0
diff --git a/pym/portage.py b/pym/portage.py
index 3cfd2bad0..7ffad8dbb 100644
--- a/pym/portage.py
+++ b/pym/portage.py
@@ -81,7 +81,7 @@ try:
 	  MOVE_BINARY, PRELINK_BINARY, WORLD_FILE, MAKE_CONF_FILE, MAKE_DEFAULTS_FILE, \
 	  DEPRECATED_PROFILE_FILE, USER_VIRTUALS_FILE, EBUILD_SH_ENV_FILE, \
 	  INVALID_ENV_FILE, CUSTOM_MIRRORS_FILE, SANDBOX_PIDS_FILE, CONFIG_MEMORY_FILE,\
-	  INCREMENTALS, STICKIES
+	  INCREMENTALS, STICKIES, EAPI
 
 	from portage_data import ostype, lchown, userland, secpass, uid, wheelgid, \
 	                         portage_uid, portage_gid
@@ -915,7 +915,7 @@ class config:
 			if self.modules["user"] == None:
 				self.modules["user"] = {}
 			self.modules["default"] = {
-				"portdbapi.metadbmodule": "portage_db_flat.database",
+				"portdbapi.metadbmodule": "portage_db_metadata.database",
 				"portdbapi.auxdbmodule":  "portage_db_flat.database",
 				"eclass_cache.dbmodule":  "portage_db_cpickle.database",
 			}
@@ -2402,6 +2402,15 @@ def doebuild(myebuild,mydo,myroot,mysettings,debug=0,listonly=0,fetchonly=0,clea
 			raise
 		except:
 			pass
+		try:
+			eapi = int(db[root][tree].dbapi.aux_get(mycpv, ["EAPI"])[0])
+			if portage_const.EAPI != eapi:
+				# can't do anything with this.
+				raise Exception("Unable to do any operations on '%s', due to the fact it's EAPI is higher then this portage versions.  Please upgrade to a portage version that supports EAPI %i" % (mycpv, eapi))
+		except SystemExit, e:
+			raise
+		except Exception, e:
+			raise Exception("Unable to pull EAPI from cpv %s, tree %s; can't confirm that it's supported by this portage, thus unable to merge it: Exception was '%s'" % (mycpv, tree, e))
 
 	if mysplit[2] == "r0":
 		mysettings["PVR"]=mysplit[1]
@@ -3914,7 +3923,14 @@ def getmaskingstatus(mycpv):
 					rValue.append("package.mask")
 
 	# keywords checking
-	mygroups = portdb.aux_get(mycpv, ["KEYWORDS"])[0].split()
+	mygroups, eapi = portdb.aux_get(mycpv, ["KEYWORDS", "EAPI"])
+	try:
+		eapi = abs(eapi)
+	except TypeError:
+		eapi = 1
+	if eapi != portage_const.EAPI:
+		return ["required EAPI %s, supported EAPI %s" % (eapi, portage_const.EAPI)]
+	mygroups = mygroups.split()
 	pgroups=groups[:]
 	myarch = settings["ARCH"]
 	pkgdict = settings.pkeywordsdict
@@ -4534,7 +4550,14 @@ class bindbapi(fakedbapi):
 				else:
 					myval = string.join(myval.split(),' ')
 				mylist.append(myval)
-
+		if "EAPI" in wants:
+			idx = wants.index("EAPI")
+			if mylist[idx] in ("", "0"):
+				mylist[idx] = 0
+			elif mylist[idx] == 0:
+				pass
+			else:
+				mylist[idx] = 1
 		return mylist
 
 
@@ -4814,6 +4837,14 @@ class vardbapi(dbapi):
 			else:
 				myd = ""
 			results.append(myd)
+		if "EAPI" in wants:
+			idx = wants.index("EAPI")
+			if mylist[idx] in ("", "0"):
+				mylist[idx] = 0
+			elif mylist[idx] == 0:
+				pass
+			else:
+				mylist[idx] = 1
 		return results
 
 
@@ -5080,9 +5111,9 @@ auxdbkeys=[
   'DEPEND',    'RDEPEND',   'SLOT',      'SRC_URI',
 	'RESTRICT',  'HOMEPAGE',  'LICENSE',   'DESCRIPTION',
 	'KEYWORDS',  'INHERITED', 'IUSE',      'CDEPEND',
-	'PDEPEND',   'PROVIDE',
+	'PDEPEND',   'PROVIDE', 'EAPI',
 	'UNUSED_01', 'UNUSED_02', 'UNUSED_03', 'UNUSED_04',
-	'UNUSED_05', 'UNUSED_06', 'UNUSED_07', 'UNUSED_08',
+	'UNUSED_05', 'UNUSED_06', 'UNUSED_07',
 	]
 auxdbkeylen=len(auxdbkeys)
 
@@ -5296,11 +5327,26 @@ class portdbapi(dbapi):
 		# we use cache files, which is usually on /usr/portage/metadata/cache/.
 		if doregen and mylocation==self.mysettings["PORTDIR"] and metacachedir and self.metadb[cat].has_key(pkg):
 			metadata=self.metadb[cat][pkg]
-			self.eclassdb.update_package(mylocation,cat,pkg,metadata["INHERITED"].split())
+			
+			try:
+				if metadata["EAPI"] == '0':
+					metadata["EAPI"] = 0
+			except KeyError:
+				metadata["EAPI"] = 0
+
+			if metadata["EAPI"] != portage_const.EAPI:
+				# intentionally wipe keys.
+				map(lambda x: metadata.setdefault("x", ''), auxdbkeys)
+				metadata["EAPI"] == -1
+
+			else:
+				# eclass updates only if we haven't nuked the entry.
+				self.eclassdb.update_package(mylocation,cat,pkg,metadata["INHERITED"].split())
+
 			self.auxdb[mylocation][cat][pkg] = metadata
 			self.auxdb[mylocation][cat].sync()
-		elif doregen:
 
+		elif doregen:
 			writemsg("doregen: %s %s\n" % (doregen,mycpv), 2)
 			writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n",1)
 
@@ -5365,6 +5411,17 @@ class portdbapi(dbapi):
 				if mylines[x][-1] == '\n':
 					mylines[x] = mylines[x][:-1]
 				mydata[auxdbkeys[x]] = mylines[x]
+
+			try:
+				eapi = int(mydata["EAPI"])
+			except ValueError:
+				eapi = 1
+				if eapi > portage_const.EAPI:
+					# if newer version, wipe everything and negate eapi
+					mydata = {}
+					map(lambda x:mydata.setdefault(x, ""), auxdbkeys)
+					mydata["EAPI"] = -eapi
+
 			mydata["_mtime_"] = emtime
 
 			self.auxdb[mylocation][cat][pkg] = mydata
@@ -5376,10 +5433,17 @@ class portdbapi(dbapi):
 		mydata   = self.auxdb[mylocation][cat][pkg]
 		returnme = []
 		for x in mylist:
-			if mydata.has_key(x):
-				returnme.append(mydata[x])
+			returnme.append(mydata.get(x,""))
+
+		if "EAPI" in mylist:
+			idx = mylist.index("EAPI")
+			if returnme[idx] in ("", "0"):
+				returnme[idx] = 0
+			elif returnme[idx] == 0:
+				pass
 			else:
-				returnme.append("")
+				try:				returnme[idx] = int(returnme[idx])
+				except ValueError:	returnme[idx] = 1
 
 		return returnme
 
@@ -5628,14 +5692,14 @@ class portdbapi(dbapi):
 			#we need to update this next line when we have fully integrated the new db api
 			auxerr=0
 			try:
-				myaux=db["/"]["porttree"].dbapi.aux_get(mycpv, ["KEYWORDS"])
+				keys, eapi = db["/"]["porttree"].dbapi.aux_get(mycpv, ["KEYWORDS", "EAPI"])
 			except (KeyError,IOError,TypeError):
 				continue
-			if not myaux[0]:
+			if not keys:
 				# KEYWORDS=""
 				#print "!!! No KEYWORDS for "+str(mycpv)+" -- Untested Status"
 				continue
-			mygroups=myaux[0].split()
+			mygroups=keys.split()
 			pgroups=groups[:]
 			match=0
 			cp = dep_getkey(mycpv)
@@ -5663,7 +5727,8 @@ class portdbapi(dbapi):
 			if not match and ((hastesting and "~*" in pgroups) or (hasstable and "*" in pgroups)):
 				match=1
 			if match:
-				newlist.append(mycpv)
+				if eapi == portage_const.EAPI:
+					newlist.append(mycpv)
 		return newlist
 
 class binarytree(packagetree):
diff --git a/pym/portage_const.py b/pym/portage_const.py
index d01f77bbb..404ca3c45 100644
--- a/pym/portage_const.py
+++ b/pym/portage_const.py
@@ -43,6 +43,8 @@ CONFIG_MEMORY_FILE      = PRIVATE_PATH + "/config"
 INCREMENTALS=["USE","FEATURES","ACCEPT_KEYWORDS","ACCEPT_LICENSE","CONFIG_PROTECT_MASK","CONFIG_PROTECT","PRELINK_PATH","PRELINK_PATH_MASK"]
 STICKIES=["KEYWORDS_ACCEPT","USE","CFLAGS","CXXFLAGS","MAKEOPTS","EXTRA_ECONF","EXTRA_EINSTALL","EXTRA_EMAKE"]
 
+EAPI = 0
+
 # ===========================================================================
 # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT
 # ===========================================================================
diff --git a/pym/portage_db_flat.py b/pym/portage_db_flat.py
index ddec4f591..4fc220bb9 100644
--- a/pym/portage_db_flat.py
+++ b/pym/portage_db_flat.py
@@ -9,6 +9,10 @@ import stat
 
 import portage_db_template
 
+# since this format is massively deprecated, 
+# we're hardcoding the previously weird line count
+magic_line_count = 22
+
 class database(portage_db_template.database):
 	def module_init(self):
 		self.lastkey  = None # Cache
@@ -42,39 +46,39 @@ class database(portage_db_template.database):
 				mykeys += [x]
 		return mykeys
 
-	def get_values(self,key):
+	def get_values(self,key, data=None):
+		""" do not use data unless you know what it does."""
+
 		if not key:
 			raise KeyError, "key is not set to a valid value"
 
-		try:
-			# give buffering a hint of the pretty much maximal cache size we deal with
-			myf = open(self.fullpath+key, "r", 8192)
-		except OSError, oe:
-			# either the file didn't exist, or it was removed under our feet.
-			return None 
-		
+		mydict = {}
+		if data == None:
+			try:
+				# give buffering a hint of the pretty much maximal cache size we deal with
+				myf = open(self.fullpath+key, "r", 8192)
+			except OSError:
+				# either the file didn't exist, or it was removed under our feet.
+				raise KeyError("failed reading key")
+
+			# nuke the newlines right off the batt.
+			data = myf.read().splitlines()
+			mydict["_mtime_"] = os.fstat(myf.fileno()).st_mtime
+			myf.close()
+		else:
+			mydict["_mtime_"] = data.pop(-1)
 
-		# nuke the newlines right off the batt.
-		data = myf.read().splitlines()
-		mdict = {}
-		
 		# rely on exceptions to note differing line counts.
 		try:
-			for x in range(0, len(self.dbkeys)):
-				mdict[self.dbkeys[x]] = data[x]
-
-			# do this now, rather then earlier- possible that earlier it might have been wasted
-			# if key count mismatched
-			mdict["_mtime_"] = os.fstat(myf.fileno()).st_mtime
+			for x in range(magic_line_count):
+				mydict[self.dbkeys[x]] = data[x]
 
 		except IndexError:
-			myf.close()
 			raise ValueError, "Key count mistmatch"
 
-		myf.close()
-		return mdict
+		return mydict
 	
-	def set_values(self,key,val):
+	def set_values(self,key, val, raw=False):
 		if not key:
 			raise KeyError, "No key provided. key:%s val:%s" % (key,val)
 		if not val:
@@ -86,12 +90,19 @@ class database(portage_db_template.database):
 
 		update_fp = self.fullpath + ".update." + str(os.getpid()) + "." + key
 		myf = open(update_fp,"w")
-		myf.writelines( [ val[x] +"\n" for x in self.dbkeys] )
+		if not raw:
+			myf.writelines( [ str(val[x]) +"\n" for x in self.dbkeys] )
+			if len(self.dbkeys) != magic_line_count:
+				myf.writelines(["\n"] * len(self.dbkeys) - magic_line_count)
+			mtime = val["_mtime_"]
+		else:
+			mtime = val.pop(-1)
+			myf.writelines(val)
 		myf.close()
 		
 		os.chown(update_fp, self.uid, self.gid)
 		os.chmod(update_fp, 0664)
-		os.utime(update_fp, (-1,long(val["_mtime_"])))
+		os.utime(update_fp, (-1,long(mtime)))
 		os.rename(update_fp, self.fullpath+key)
 
 	def del_key(self,key):
diff --git a/pym/portage_db_flat_hash.py b/pym/portage_db_flat_hash.py
new file mode 100644
index 000000000..8ee7661ff
--- /dev/null
+++ b/pym/portage_db_flat_hash.py
@@ -0,0 +1,38 @@
+# Copyright 2004 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Header: /var/cvsroot/gentoo-src/portage/pym/Attic/portage_db_flat.py,v 1.13.2.6 2005/04/19 07:14:17 ferringb Exp $
+cvs_id_string="$Id: portage_db_flat.py,v 1.13.2.6 2005/04/19 07:14:17 ferringb Exp $"[5:-2]
+
+import portage_db_flat, os
+
+class database(portage_db_flat.database):
+	
+	def get_values(self, key, data=None):
+		""" do not specify data unless you know what it does"""
+		if not key:
+			raise KeyError("key is not valid")
+		
+		if data == None:
+			try:
+				myf = open(self.fullpath + key, "r")
+			except OSError:
+				raise KeyError("failed pulling key")
+
+			data = dict(map(lambda x: x.split("=",1), myf.read().splitlines()))
+			data["_mtime_"] = os.fstat(myf.fileno()).st_mtime
+			myf.close()
+
+		mydict = {}
+		for x in self.dbkeys:
+			mydict[x] = data.get(x, "")
+		mydict["_mtime_"] = long(data["_mtime_"])
+		return mydict
+		
+	def set_values(self, key, values):
+		l = []
+		for x in values.keys():
+			if values[x] and x != "_mtime_":
+				l.append("%s=%s\n" % (x, values[x]))
+		l.append(values["_mtime_"])
+		portage_db_flat.database.set_values(self, key, l, raw=True)
+		
diff --git a/pym/portage_db_metadata.py b/pym/portage_db_metadata.py
new file mode 100644
index 000000000..60d87b2d0
--- /dev/null
+++ b/pym/portage_db_metadata.py
@@ -0,0 +1,49 @@
+# Copyright 2004 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Header: /var/cvsroot/gentoo-src/portage/pym/Attic/portage_db_flat.py,v 1.13.2.6 2005/04/19 07:14:17 ferringb Exp $
+cvs_id_string="$Id: portage_db_flat.py,v 1.13.2.6 2005/04/19 07:14:17 ferringb Exp $"[5:-2]
+
+import os, portage_db_flat_hash, portage_db_flat
+
+class database(portage_db_flat_hash.database):
+	
+	def get_values(self, key):
+		if not key:
+			raise KeyError("key is not valid")
+		
+		try:
+			myf = open(self.fullpath + key, "r")
+		except OSError:
+			raise KeyError("key is not valid")
+		mtime = os.fstat(myf.fileno()).st_mtime
+		data = myf.read().splitlines()
+		
+		# easy attempt first.
+		if len(data) != portage_db_flat.magic_line_count:
+			d = dict(map(lambda x: x.split("=",1), data))
+			d["_mtime_"] = mtime
+			return portage_db_flat_hash.database.get_values(self, key, d)
+		# this one's interesting.
+		d = {}
+
+		for line in data:
+			# yes, meant to iterate over a string.
+			hashed = False
+			for idx, c in enumerate(line):
+				if not c.isalpha():
+					if c == "=" and idx > 0:
+						hashed = True
+						d[line[:idx]] = line[idx + 1:]
+					elif c == "_" or c.isdigit():
+						continue
+					break
+				elif not c.isupper():
+					break
+
+			if not hashed:
+				# non hashed.
+				data.append(mtime)
+				return portage_db_flat.database.get_values(self, key, data=data)
+
+		d["_mtime_"] = mtime
+		return portage_db_flat_hash.database.get_values(self, key, data=d)