integration chunk of the cache subsystem patch.

Will deal with UNUSED_0* after this

svn path=/main/branches/2.0/; revision=2258

4 files changed, 176 insertions, 264 deletions

diff --git a/bin/emerge b/bin/emerge
index 6ea0aceb0..3a38f66a0 100755
--- a/bin/emerge
+++ b/bin/emerge
@@ -2671,14 +2671,12 @@ if myaction in ["sync","rsync","metadata"] and (not "--help" in myopts):
 
 		portage.portdb.flush_cache()
 
-		try:
-			os.umask(002)
-			os.chown(cachedir, os.getuid(), portage.portage_gid)
-			os.chmod(cachedir, 02775)
-		except SystemExit, e:
-			raise # Needed else can't exit
-		except:
-			pass
+		ec = portage.eclass_cache.cache(portage.portdb.porttree_root)
+		# kinda ugly.
+		# XXX: nuke the filter when mr UNUSED_0? keys are dead
+		cm = portage.settings.load_best_module("portdbapi.metadbmodule")(myportdir, "metadata/cache", 
+			filter(lambda x: not x.startswith("UNUSED_0"), portage.auxdbkeys))
+
 		# we don't make overlay trees cache here, plus we don't trust portage.settings.categories
 		porttree_root = portage.portdb.porttree_root
 		conf = portage.config(config_profile_path=portage.settings.profile_path[:], \
@@ -2694,75 +2692,54 @@ if myaction in ["sync","rsync","metadata"] and (not "--help" in myopts):
 			pass
 
 		pdb = portage.portdbapi(porttree_root, conf)
-		cp_list = pdb.cp_all()
-		if len(cp_list) == 0:
-			print "no metadata to transfer, exiting"
-			sys.exit(0)
-		cp_list.sort()
-		pcnt=0
-		pcntstr=""
-		pcntcount=len(cp_list)/100.0
-		nextupdate=pcntcount
-		current=1
-
-		def cleanse_cache(pdb, cat, saves, porttree_root=porttree_root):
-			try:
-				if len(saves):
-					d={}
-					for v in saves:
-						d[portage.catsplit(v)[1]] = True
-					for pv in pdb.auxdb[porttree_root][cat].keys():
-						if pv not in d:
-							pdb.auxdb[porttree_root][cat].del_key(pv)
-				else:
-					try:
-						pdb.auxdb[porttree_root][cat].clear()
-						del pdb.auxdb[porttree_root][cat]
-					except KeyError:
-						pass
-			except KeyError:
-				# stop breaking things, cleansing is minor.
-				pass
 
-		savelist = []
-		catlist = []
-		oldcat = portage.catsplit(cp_list[0])[0]
-		for cp in cp_list:
-			current += 1
-			if current >= nextupdate:
-				pcnt += 1
-				nextupdate += pcntcount
-				if "--quiet" not in myopts:
-					pcntstr = str(pcnt)
-					sys.stdout.write("\b"*(len(pcntstr)+1)+pcntstr+"%")
-					sys.stdout.flush()
-			cat = portage.catsplit(cp)[0]
-			if cat != oldcat:
-				catlist.append(oldcat)
-				cleanse_cache(pdb, oldcat, savelist)
-				savelist = []
-				oldcat = cat
-			mymatches = pdb.xmatch("match-all", cp)
-			savelist.extend(mymatches)
-			for cpv in mymatches:
-				try: pdb.aux_get(cpv, ["IUSE"],metacachedir=myportdir+"/metadata/cache",debug=("cachedebug" in portage.features))
-				except SystemExit: raise
-				except Exception, e: print "\nFailed cache update:",cpv,e
-		catlist.append(oldcat)
-		catlist.append("local")
-		cleanse_cache(pdb, oldcat, savelist)
-		filelist = portage.listdir(cachedir+"/"+myportdir)
-		for x in filelist:
-			found = False
-			for y in catlist:
-				if x.startswith(y):
-					found = True
-					break
-			if not found:
-				portage.spawn("cd /; rm -Rf "+cachedir+"/"+myportdir+"/"+x,portage.settings,free=1,droppriv=1)
+		cp_all_list = pdb.cp_all()
+		import cache.util
+
+		class percentage_noise_maker(cache.util.quiet_mirroring):
+			def __init__(self, dbapi):
+				self.dbapi = dbapi
+				self.cp_all = dbapi.cp_all()
+				self.cp_all.sort()
+				l = len(self.cp_all)
+				self.call_update_min = 100000000
+				self.min_cp_all = l/100.0
+				self.count = 1
+				self.pstr = ''
+			
+			def __iter__(self):
+				for x in self.cp_all:
+					self.count += 1
+					if self.count > self.min_cp_all:
+						self.call_update_min = 0
+						self.count = 0
+					for y in self.dbapi.cp_list(x):
+						yield y
+				self.call_update_mine = 0
+			
+			def update(self, *arg):
+				try:				self.pstr = int(self.pstr) + 1
+				except ValueError:	self.pstr = 1
+				sys.stdout.write("%s%i%%" % ("\b" * (len(str(self.pstr))+1), self.pstr))
+				sys.stdout.flush()
+				self.call_update_min = 10000000
+			
+			def finish(self, *arg):
+				sys.stdout.write("\b\b\b\b100%\n")
+				sys.stdout.flush()
 
 
-		sys.stdout.write("\n\n")
+		if "--quiet" in myopts:
+			def quicky_cpv_generator(cp_all_list):
+				for x in cp_all_list:
+					for y in pdb.cp_list(x):
+						yield y
+			source = quicky_cpv_generator(pdb.cp_all())
+			noise_maker = cache.util.quiet_mirroring()
+		else:
+			noise_maker = source = percentage_noise_maker(pdb)
+		cache.util.mirror_cache(source, cm, pdb.auxdb[porttree_root], eclass_cache=ec, verbose_instance=noise_maker)
+		
 		sys.stdout.flush()
 
 	portage.portageexit()
@@ -2796,7 +2773,7 @@ elif myaction=="regen":
 			print "processing",x
 		for y in mymatches:
 			try:
-				foo=portage.portdb.aux_get(y,["DEPEND"],debug=1)
+				foo=portage.portdb.aux_get(y,["DEPEND"])
 			except SystemExit, e:
 				# sys.exit is an exception... And consequently, we can't catch it.
 				raise
diff --git a/bin/repoman b/bin/repoman
index 9e4afb231..49578ad59 100755
--- a/bin/repoman
+++ b/bin/repoman
@@ -898,7 +898,7 @@ for x in scanlist:
 			fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
 			continue
 		try:
-			myaux=portage.db["/"]["porttree"].dbapi.aux_get(catdir+"/"+y,allvars,strict=1)
+			myaux=portage.db["/"]["porttree"].dbapi.aux_get(catdir+"/"+y,allvars)
 		except KeyError:
 			stats["ebuild.syntax"]=stats["ebuild.syntax"]+1
 			fails["ebuild.syntax"].append(x+"/"+y+".ebuild")
diff --git a/pym/eclass_cache.py b/pym/eclass_cache.py
new file mode 100644
index 000000000..066162611
--- /dev/null
+++ b/pym/eclass_cache.py
@@ -0,0 +1,75 @@
+# Copyright: 2005 Gentoo Foundation
+# Author(s): Nicholas Carpaski (carpaski@gentoo.org), Brian Harring (ferringb@gentoo.org)
+# License: GPL2
+# $Id:$
+
+from portage_util import writemsg
+import portage_file
+import os, sys
+from portage_data import portage_gid
+
+class cache:
+	"""
+	Maintains the cache information about eclasses used in ebuild.
+	"""
+	def __init__(self, porttree_root, overlays=[]):
+		self.porttree_root = porttree_root
+
+		self.eclasses = {} # {"Name": ("location","_mtime_")}
+
+		# screw with the porttree ordering, w/out having bash inherit match it, and I'll hurt you.
+		# ~harring
+		self.porttrees = [self.porttree_root]+overlays
+		self.porttrees = tuple(map(portage_file.normpath, self.porttrees))
+		self._master_eclass_root = os.path.join(self.porttrees[0],"eclass")
+		self.update_eclasses()
+
+	def close_caches(self):
+		import traceback
+		traceback.print_stack()
+		print "%s close_cache is deprecated" % self.__class__
+		self.eclasses.clear()
+
+	def flush_cache(self):
+		import traceback
+		traceback.print_stack()
+		print "%s flush_cache is deprecated" % self.__class__
+
+		self.update_eclasses()
+
+	def update_eclasses(self):
+		self.eclasses = {}
+		eclass_len = len(".eclass")
+		for x in [portage_file.normpath(os.path.join(y,"eclass")) for y in self.porttrees]:
+			if not os.path.isdir(x):
+				continue
+			for y in [y for y in os.listdir(x) if y.endswith(".eclass")]:
+				try:
+					mtime=os.stat(x+"/"+y).st_mtime
+				except OSError:
+					continue
+				ys=y[:-eclass_len]
+				self.eclasses[ys] = (x, long(mtime))
+	
+	def is_eclass_data_valid(self, ec_dict):
+		if not isinstance(ec_dict, dict):
+			return False
+		for eclass, tup in ec_dict.iteritems():
+			if eclass not in self.eclasses or tuple(tup) != self.eclasses[eclass]:
+				return False
+
+		return True
+
+	def get_eclass_data(self, inherits, from_master_only=False):
+		ec_dict = {}
+		for x in inherits:
+			try:
+				ec_dict[x] = self.eclasses[x]
+			except:
+				print "ec=",ec_dict
+				print "inherits=",inherits
+				raise
+			if from_master_only and self.eclasses[x][0] != self._master_eclass_root:
+				return None
+
+		return ec_dict
diff --git a/pym/portage.py b/pym/portage.py
index 473b0c4ac..7e305ea46 100644
--- a/pym/portage.py
+++ b/pym/portage.py
@@ -28,6 +28,7 @@ try:
 	import commands
 	from time import sleep
 	from random import shuffle
+	from cache.cache_errors import CacheError
 except SystemExit, e:
 	raise
 except Exception, e:
@@ -97,6 +98,7 @@ try:
 	from portage_locks import unlockfile,unlockdir,lockfile,lockdir
 	import portage_checksum
 	from portage_checksum import perform_md5,perform_checksum,prelink_capable
+	import eclass_cache
 	from portage_localization import _
 except SystemExit, e:
 	raise
@@ -921,9 +923,8 @@ class config:
 			if self.modules["user"] == None:
 				self.modules["user"] = {}
 			self.modules["default"] = {
-				"portdbapi.metadbmodule": "portage_db_metadata.database",
-				"portdbapi.auxdbmodule":  "portage_db_flat.database",
-				"eclass_cache.dbmodule":  "portage_db_cpickle.database",
+				"portdbapi.metadbmodule": "cache.metadata.database",
+				"portdbapi.auxdbmodule":  "cache.flat_hash.database",
 			}
 
 			self.usemask=[]
@@ -5017,116 +5018,6 @@ class vartree(packagetree):
 	def populate(self):
 		self.populated=1
 
-# ----------------------------------------------------------------------------
-class eclass_cache:
-	"""Maintains the cache information about eclasses used in ebuild."""
-	def __init__(self,porttree_root,settings):
-		self.porttree_root = porttree_root
-		self.settings = settings
-		self.depcachedir = self.settings.depcachedir[:]
-
-		self.dbmodule = self.settings.load_best_module("eclass_cache.dbmodule")
-
-		self.packages = {} # {"PV": {"eclass1": ["location", "_mtime_"]}}
-		self.eclasses = {} # {"Name": ["location","_mtime_"]}
-
-		# don't fool with porttree ordering unless you *ensure* that ebuild.sh's inherit
-		# ordering is *exactly* the same
-		self.porttrees=[self.porttree_root]
-		self.porttrees.extend(self.settings["PORTDIR_OVERLAY"].split())
-		#normalize the path now, so it's not required later.
-		self.porttrees = [os.path.normpath(x) for x in self.porttrees]
-		self.update_eclasses()
-
-	def close_caches(self):
-		for x in self.packages.keys():
-			for y in self.packages[x].keys():
-				try:
-					self.packages[x][y].sync()
-					self.packages[x][y].close()
-				except SystemExit, e:
-					raise
-				except Exception,e:
-					writemsg("Exception when closing DB: %s: %s\n" % (Exception,e))
-				del self.packages[x][y]
-			del self.packages[x]
-
-	def flush_cache(self):
-		self.packages = {}
-		self.eclasses = {}
-		self.update_eclasses()
-
-	def update_eclasses(self):
-		self.eclasses = {}
-		for x in suffix_array(self.porttrees, "/eclass"):
-			if x and os.path.exists(x):
-				dirlist = listdir(x)
-				for y in dirlist:
-					if y[-len(".eclass"):]==".eclass":
-						try:
-							ys=y[:-len(".eclass")]
-							ymtime=os.stat(x+"/"+y)[stat.ST_MTIME]
-						except SystemExit, e:
-							raise
-						except:
-							continue
-						self.eclasses[ys] = [x, ymtime]
-
-	def setup_package(self, location, cat, pkg):
-		if not self.packages.has_key(location):
-			self.packages[location] = {}
-
-		if not self.packages[location].has_key(cat):
-			try:
-				self.packages[location][cat] = self.dbmodule(self.depcachedir+"/"+location, cat+"-eclass", [], uid, portage_gid)
-			except SystemExit, e:
-				raise
-			except Exception, e:
-				writemsg("\n!!! Failed to open the dbmodule for eclass caching.\n")
-				writemsg("!!! Generally these are permission problems. Caught exception follows:\n")
-				writemsg("!!! "+str(e)+"\n")
-				writemsg("!!! Dirname:  "+str(self.depcachedir+"/"+location)+"\n")
-				writemsg("!!! Basename: "+str(cat+"-eclass")+"\n\n")
-				sys.exit(123)
-
-	def sync(self, location, cat, pkg):
-		if self.packages[location].has_key(cat):
-			self.packages[location][cat].sync()
-
-	def update_package(self, location, cat, pkg, eclass_list):
-		self.setup_package(location, cat, pkg)
-		if not eclass_list:
-			return 1
-
-		data = {}
-		for x in eclass_list:
-			if x not in self.eclasses:
-				writemsg("Eclass '%s' does not exist for '%s'\n" % (x, cat+"/"+pkg))
-				return 0
-			data[x] = [self.eclasses[x][0],self.eclasses[x][1]]
-
-		self.packages[location][cat][pkg] = data
-		self.sync(location,cat,pkg)
-		return 1
-
-	def is_current(self, location, cat, pkg, eclass_list):
-		self.setup_package(location, cat, pkg)
-
-		if not eclass_list:
-			return 1
-
-		if not (self.packages[location][cat].has_key(pkg) and self.packages[location][cat][pkg] and eclass_list):
-			return 0
-
-		myp = self.packages[location][cat][pkg]
-		for x in eclass_list:
-			if not (x in self.eclasses and x in myp and myp[x] == self.eclasses[x]):
-				return 0
-
-		return 1
-
-# ----------------------------------------------------------------------------
-
 auxdbkeys=[
   'DEPEND',    'RDEPEND',   'SLOT',      'SRC_URI',
 	'RESTRICT',  'HOMEPAGE',  'LICENSE',   'DESCRIPTION',
@@ -5140,6 +5031,8 @@ auxdbkeylen=len(auxdbkeys)
 def close_portdbapi_caches():
 	for i in portdbapi.portdbapi_instances:
 		i.close_caches()
+
+
 class portdbapi(dbapi):
 	"""this tree will scan a portage directory located at root (passed to init)"""
 	portdbapi_instances = []
@@ -5182,33 +5075,34 @@ class portdbapi(dbapi):
 		if self.tmpfs and not os.access(self.tmpfs, os.R_OK):
 			self.tmpfs = None
 
-		self.eclassdb = eclass_cache(self.porttree_root, self.mysettings)
+		self.eclassdb = eclass_cache.cache(self.porttree_root, overlays=settings["PORTDIR_OVERLAY"].split())
 
 		self.metadb       = {}
 		self.metadbmodule = self.mysettings.load_best_module("portdbapi.metadbmodule")
 
-		self.auxdb        = {}
-		self.auxdbmodule  = self.mysettings.load_best_module("portdbapi.auxdbmodule")
-
 		#if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening)
 		self.xcache={}
 		self.frozen=0
 
 		self.porttrees=[self.porttree_root]+self.mysettings["PORTDIR_OVERLAY"].split()
+		self.auxdbmodule  = self.mysettings.load_best_module("portdbapi.auxdbmodule")
+		self.auxdb        = {}
 
+		# XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
+		# ~harring
+		filtered_auxdbkeys = filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys)
+		for x in self.porttrees:
+			# location, label, auxdbkeys
+			self.auxdb[x] = self.auxdbmodule(portage_const.DEPCACHE_PATH, x, filtered_auxdbkeys, gid=portage_gid)
+			
 	def close_caches(self):
 		for x in self.auxdb.keys():
-			for y in self.auxdb[x].keys():
-				self.auxdb[x][y].sync()
-				self.auxdb[x][y].close()
-				del self.auxdb[x][y]
-			del self.auxdb[x]
-		self.eclassdb.close_caches()
+			self.auxdb[x].sync()
+		self.auxdb.clear()
 
 	def flush_cache(self):
 		self.metadb = {}
 		self.auxdb  = {}
-		self.eclassdb.flush_cache()
 
 	def finddigest(self,mycpv):
 		try:
@@ -5261,7 +5155,7 @@ class portdbapi(dbapi):
 		# when not found
 		return None, 0
 
-	def aux_get(self,mycpv,mylist,strict=0,metacachedir=None,debug=0):
+	def aux_get(self, mycpv, mylist):
 		"stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
 		'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
 		'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error'
@@ -5269,10 +5163,6 @@ class portdbapi(dbapi):
 
 		cat,pkg = string.split(mycpv, "/", 1)
 
-		if metacachedir:
-			if cat not in self.metadb:
-				self.metadb[cat] = self.metadbmodule(metacachedir,cat,auxdbkeys,uid,portage_gid)
-
 		myebuild, mylocation=self.findname2(mycpv)
 
 		if not myebuild:
@@ -5314,11 +5204,6 @@ class portdbapi(dbapi):
 					raise portage_exception.SecurityViolation, "Error in verification of signatures: %(errormsg)s" % {"errormsg":str(e)}
 				writemsg("!!! Manifest is missing or inaccessable: %(manifest)s\n" % {"manifest":myManifestPath})
 
-		if mylocation not in self.auxdb:
-			self.auxdb[mylocation] = {}
-
-		if not self.auxdb[mylocation].has_key(cat):
-			self.auxdb[mylocation][cat] = self.auxdbmodule(self.depcachedir+"/"+mylocation,cat,auxdbkeys,uid,portage_gid)
 
 		if os.access(myebuild, os.R_OK):
 			emtime=os.stat(myebuild)[stat.ST_MTIME]
@@ -5328,47 +5213,24 @@ class portdbapi(dbapi):
 			raise KeyError
 
 		try:
-			auxdb_is_valid = self.auxdb[mylocation][cat].has_key(pkg) and \
-			                 self.auxdb[mylocation][cat][pkg].has_key("_mtime_") and \
-			                 self.auxdb[mylocation][cat][pkg]["_mtime_"] == emtime
-		except SystemExit, e:
-			raise
-		except Exception, e:
-			auxdb_is_valid = 0
-			if not metacachedir:
-				writemsg("auxdb exception: [%(loc)s]: %(exception)s\n" % {"loc":mylocation+"::"+cat+"/"+pkg, "exception":str(e)})
-			if self.auxdb[mylocation][cat].has_key(pkg):
-				self.auxdb[mylocation][cat].del_key(pkg)
-				self.auxdb[mylocation][cat].sync()
-
-		writemsg("auxdb is valid: "+str(auxdb_is_valid)+" "+str(pkg)+"\n", 2)
-		doregen = not (auxdb_is_valid and self.eclassdb.is_current(mylocation,cat,pkg,self.auxdb[mylocation][cat][pkg]["INHERITED"].split()))
-
-		# when mylocation is not overlay directorys and metacachedir is set,
-		# we use cache files, which is usually on /usr/portage/metadata/cache/.
-		if doregen and mylocation==self.mysettings["PORTDIR"] and metacachedir and self.metadb[cat].has_key(pkg):
-			metadata=self.metadb[cat][pkg]
-
-			if "EAPI" not in metadata or not metadata["EAPI"].strip():
-				metadata["EAPI"] = "0"
-
-			if not eapi_is_supported(metadata["EAPI"]):
-				# intentionally wipe keys.
-				eapi = metadata["EAPI"]
-				mtime = metadata.get("_mtime_", 0)
-				metadata = {}
-				map(lambda x: metadata.setdefault(x, ''), auxdbkeys)
-				metadata["_mtime_"] = long(mtime)
-				metadata["EAPI"] == "-"+eapi
-
+			mydata = self.auxdb[mylocation][mycpv]
+			if emtime != long(mydata.get("_mtime_", 0)):
+				doregen = True
+			elif len(mydata.get("_eclasses_", [])) > 0:
+				doregen = not self.eclassdb.is_eclass_data_valid(mydata["_eclasses_"])
 			else:
-				# eclass updates only if we haven't nuked the entry.
-				self.eclassdb.update_package(mylocation,cat,pkg,metadata["INHERITED"].split())
+				doregen = False
+				
+		except KeyError:
+			doregen = True
+		except CacheError:
+			doregen = True
+			try:				del self.auxdb[mylocation][mycpv]
+			except KeyError:	pass
 
-			self.auxdb[mylocation][cat][pkg] = metadata
-			self.auxdb[mylocation][cat].sync()
+		writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2)
 
-		elif doregen:
+		if doregen:
 			writemsg("doregen: %s %s\n" % (doregen,mycpv), 2)
 			writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n",1)
 
@@ -5389,9 +5251,7 @@ class portdbapi(dbapi):
 			if os.path.exists(mydbkey):
 				try:
 					os.unlink(mydbkey)
-				except SystemExit, e:
-					raise
-				except Exception, e:
+				except (IOError, OSError), e:
 					portage_locks.unlockfile(mylock)
 					self.lock_held = 0
 					writemsg("Uncaught handled exception: %(exception)s\n" % {"exception":str(e)})
@@ -5411,19 +5271,13 @@ class portdbapi(dbapi):
 				os.unlink(mydbkey)
 				mylines=mycent.readlines()
 				mycent.close()
-			except SystemExit, e:
-				raise
+
 			except (IOError, OSError):
 				portage_locks.unlockfile(mylock)
 				self.lock_held = 0
 				writemsg(str(red("\naux_get():")+" (1) Error in "+mycpv+" ebuild.\n"
 				  "               Check for syntax error or corruption in the ebuild. (--debug)\n\n"))
 				raise KeyError
-			except Exception, e:
-				portage_locks.unlockfile(mylock)
-				self.lock_held = 0
-				writemsg("Uncaught handled exception: %(exception)s\n" % {"exception":str(e)})
-				raise
 
 			portage_locks.unlockfile(mylock)
 			self.lock_held = 0
@@ -5444,18 +5298,24 @@ class portdbapi(dbapi):
 				map(lambda x:mydata.setdefault(x, ""), auxdbkeys)
 				mydata["EAPI"] = "-"+eapi
 
+			if mydata.get("INHERITED", False):
+				mydata["_eclasses_"] = self.eclassdb.get_eclass_data(mydata["INHERITED"].split())
+			else:
+				mydata["_eclasses_"] = {}
+			
+			del mydata["INHERITED"]
+
 			mydata["_mtime_"] = emtime
 
-			self.auxdb[mylocation][cat][pkg] = mydata
-			self.auxdb[mylocation][cat].sync()
-			if not self.eclassdb.update_package(mylocation, cat, pkg, mylines[auxdbkeys.index("INHERITED")].split()):
-				sys.exit(1)
+			self.auxdb[mylocation][mycpv] = mydata
 
 		#finally, we look at our internal cache entry and return the requested data.
-		mydata   = self.auxdb[mylocation][cat][pkg]
 		returnme = []
 		for x in mylist:
-			returnme.append(mydata.get(x,""))
+			if x == "INHERITED":
+				returnme.append(' '.join(mydata.get("_eclasses_", {}).keys()))
+			else:
+				returnme.append(mydata.get(x,""))
 
 		if "EAPI" in mylist:
 			idx = mylist.index("EAPI")