Add manifest2 support.

svn path=/main/trunk/; revision=2990

3 files changed, 113 insertions, 175 deletions

diff --git a/pym/portage.py b/pym/portage.py
index 2a4000044..2673c7063 100644
--- a/pym/portage.py
+++ b/pym/portage.py
@@ -73,6 +73,7 @@ try:
 
 	from portage_data import ostype, lchown, userland, secpass, uid, wheelgid, \
 	                         portage_uid, portage_gid
+	from portage_manifest import Manifest
 
 	import portage_util
 	from portage_util import atomic_ofstream, apply_secpass_permissions, apply_recursive_permissions, \
@@ -2022,179 +2023,67 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 			return 0
 	return 1
 
-
-def digestCreate(myfiles,basedir,oldDigest={}):
-	"""Takes a list of files and the directory they are in and returns the
-	dict of dict[filename][CHECKSUM_KEY] = hash
-	returns None on error."""
-	mydigests={}
-	for x in myfiles:
-		print "<<<",x
-		myfile=os.path.normpath(basedir+"///"+x)
-		if os.path.exists(myfile):
-			if not os.access(myfile, os.R_OK):
-				print "!!! Given file does not appear to be readable. Does it exist?"
-				print "!!! File:",myfile
-				return None
-			mydigests[x] = portage_checksum.perform_multiple_checksums(myfile, hashes=portage_const.MANIFEST1_HASH_FUNCTIONS)
-			mysize       = os.stat(myfile)[stat.ST_SIZE]
-		else:
-			if x in oldDigest:
-				# DeepCopy because we might not have a unique reference.
-				mydigests[x] = copy.deepcopy(oldDigest[x])
-				mysize       = copy.deepcopy(oldDigest[x]["size"])
-			else:
-				print "!!! We have a source URI, but no file..."
-				print "!!! File:",myfile
-				return None
-
-		if mydigests[x].has_key("size") and (mydigests[x]["size"] != mysize):
-			raise portage_exception.DigestException, "Size mismatch during checksums"
-		mydigests[x]["size"] = copy.deepcopy(mysize)
-	return mydigests
-
-def digestCreateLines(filelist, mydict):
-	mylines = []
-	mydigests = copy.deepcopy(mydict)
-	for myarchive in filelist:
-		mysize = mydigests[myarchive]["size"]
-		if len(mydigests[myarchive]) == 0:
-			raise portage_exception.DigestException, "No generate digest for '%(file)s'" % {"file":myarchive}
-		for sumName in mydigests[myarchive].keys():
-			if sumName not in portage_checksum.get_valid_checksum_keys():
-				continue
-			mysum = mydigests[myarchive][sumName]
-
-			myline  = sumName[:]
-			myline += " "+mysum
-			myline += " "+myarchive
-			myline += " "+str(mysize)
-			mylines.append(myline)
-	return mylines
-
-def digestgen(myarchives,mysettings,overwrite=1,manifestonly=0):
+def digestgen(myarchives,mysettings,db=None,overwrite=1,manifestonly=0):
 	"""generates digest file if missing.  Assumes all files are available.	If
-	overwrite=0, the digest will only be created if it doesn't already exist."""
-
-	# archive files
-	basedir=mysettings["DISTDIR"]+"/"
-	digestfn=mysettings["FILESDIR"]+"/digest-"+mysettings["PF"]
-
-	# portage files -- p(ortagefiles)basedir
-	pbasedir=mysettings["O"]+"/"
-	manifestfn=pbasedir+"Manifest"
-
-	if not manifestonly:
-		if not os.path.isdir(mysettings["FILESDIR"]):
-			os.makedirs(mysettings["FILESDIR"])
-		mycvstree=cvstree.getentries(pbasedir, recursive=1)
-
-		if ("cvs" in features) and os.path.exists(pbasedir+"/CVS"):
-			if not cvstree.isadded(mycvstree,"files"):
-				if "autoaddcvs" in features:
-					print ">>> Auto-adding files/ dir to CVS..."
-					spawn("cd "+pbasedir+"; cvs add files",mysettings,free=1)
-				else:
-					print "--- Warning: files/ is not added to cvs."
-
-		if (not overwrite) and os.path.exists(digestfn):
-			return 1
-
-		print green(">>> Generating the digest file...")
-
-		# Track the old digest so we can assume checksums without requiring
-		# all files to be downloaded. 'Assuming'
-		myolddigest = {}
-		if os.path.exists(digestfn):
-			myolddigest = digestParseFile(digestfn)
-
-		myarchives.sort()
-		try:
-			mydigests=digestCreate(myarchives, basedir, oldDigest=myolddigest)
-		except portage_exception.DigestException, s:
-			print "!!!",s
-			return 0
-		if mydigests==None: # There was a problem, exit with an errorcode.
-			return 0
-
-		if mydigests != myolddigest:
-			digest_lines = digestCreateLines(myarchives, mydigests)
-			digest_success = True
-			try:
-				write_atomic(digestfn, "\n".join(digest_lines) + "\n")
-				digest_success = apply_secpass_permissions(
-					digestfn, gid=portage_gid, mode=0664)
-			except (IOError, OSError), e:
-				writemsg("!!! %s\n" % str(e))
-				digest_success = False
-			except portage_exception.PortageException:
-				writemsg("!!! %s\n" % str(e))
-				digest_success = False
-			if not digest_success:
-				writemsg("!!! Filesystem error, skipping generation.\n")
-				return 0
-
-	print green(">>> Generating the manifest file...")
-	mypfiles=listdir(pbasedir,recursive=1,filesonly=1,ignorecvs=1,EmptyOnError=1)
-	mypfiles=cvstree.apply_cvsignore_filter(mypfiles)
-	mypfiles.sort()
-	for x in ["Manifest"]:
-		if x in mypfiles:
-			mypfiles.remove(x)
-
-	mydigests=digestCreate(mypfiles, pbasedir)
-	if mydigests==None: # There was a problem, exit with an errorcode.
-		return 0
-
-	try:
-		outfile=open(manifestfn, "w+")
-	except SystemExit, e:
-		raise
-	except Exception, e:
-		print "!!! Filesystem error skipping generation. (Read-Only?)"
-		print "!!!",e
-		return 0
-	for x in digestCreateLines(mypfiles, mydigests):
-		outfile.write(x+"\n")
-	outfile.close()
-	try:
-		os.chown(manifestfn,os.getuid(),portage_gid)
-		os.chmod(manifestfn,0664)
-	except SystemExit, e:
-		raise
-	except Exception,e:
-		print e
-
-	if "cvs" in features and os.path.exists(pbasedir+"/CVS"):
-		mycvstree=cvstree.getentries(pbasedir, recursive=1)
-		myunaddedfiles=""
-		if not manifestonly and not cvstree.isadded(mycvstree,digestfn):
-			if digestfn[:len(pbasedir)]==pbasedir:
-				myunaddedfiles=digestfn[len(pbasedir):]+" "
-			else:
-				myunaddedfiles=digestfn+" "
-		if not cvstree.isadded(mycvstree,manifestfn[len(pbasedir):]):
-			if manifestfn[:len(pbasedir)]==pbasedir:
-				myunaddedfiles+=manifestfn[len(pbasedir):]+" "
-			else:
-				myunaddedfiles+=manifestfn
-		if myunaddedfiles:
-			if "autoaddcvs" in features:
-				print blue(">>> Auto-adding digest file(s) to CVS...")
-				spawn("cd "+pbasedir+"; cvs add "+myunaddedfiles,mysettings,free=1)
-			else:
-				print "--- Warning: digests are not yet added into CVS."
-	print darkgreen(">>> Computed message digests.")
-	print
+	overwrite=0, the digest will only be created if it doesn't already exist.
+	DEPRECATED: this now only is a compability wrapper for 
+	            portage_manifest.Manifest()"""
+
+	# NOTE: manifestonly is useless with manifest2 and therefore ignored
+	# NOTE: the old code contains a lot of crap that should really be elsewhere 
+	#       (e.g. cvs stuff should be in ebuild(1) and/or repoman)
+	# TODO: error/exception handling
+
+	if db == None:
+		db = portagetree().dbapi
+
+	mf = Manifest(mysettings["O"], db, mysettings)
+	for f in myarchives:
+		# the whole type evaluation is only for the case that myarchives isn't a 
+		# DIST file as create() determines the type on its own
+		mytype = mf.guessType(f)
+		if mytype == "AUX":
+			f = f[5:]
+		elif mytype == None:
+			continue
+		myrealtype = mf.findFile(f)
+		if myrealtype != None:
+			mytype = myrealtype
+		mf.create(assumeDistfileHashes=True)
+		mf.updateFileHashes(mytype, f, checkExisting=False)
+	# NOTE: overwrite=0 is only used by emerge --digest, not sure we wanna keep that
+	if overwrite or not os.path.exists(mf.getFullname()):
+		mf.write(sign=False)
+	
 	return 1
 
-
-def digestParseFile(myfilename):
+def digestParseFile(myfilename,mysettings=None,db=None):
 	"""(filename) -- Parses a given file for entries matching:
 	<checksumkey> <checksum_hex_string> <filename> <filesize>
 	Ignores lines that don't start with a valid checksum identifier
 	and returns a dict with the filenames as keys and {checksumkey:checksum}
-	as the values."""
+	as the values.
+	DEPRECATED: this function is now only a compability wrapper for
+	            portage_manifest.Manifest()."""
+
+	mysplit = myfilename.split(os.sep)
+	if mysplit[-2] == "files" and mysplit[-1].startswith("digest-"):
+		pkgdir = os.sep+os.sep.join(mysplit[:-2])
+	elif mysplit[-1] == "Manifest":
+		pkgdir = os.sep+os.sep.join(mysplit[:-1])
+
+	if db == None:
+		db = portagetree().dbapi
+	if mysettings == None:
+		mysettings = config(clone=settings)
+
+	mf = Manifest(pkgdir, db, mysettings)
+
+	return mf.getDigests()
+	
+	#########################################
+	# Old code that's replaced by the above #
+	#########################################
 
 	if not os.path.exists(myfilename):
 		return None
@@ -2228,7 +2117,11 @@ def digestCheckFiles(myfiles, mydigests, basedir, note="", strict=0):
 	"""(fileslist, digestdict, basedir) -- Takes a list of files and a dict
 	of their digests and checks the digests against the indicated files in
 	the basedir given. Returns 1 only if all files exist and match the checksums.
+	DEPRECATED: this function isn't compatible with manifest2, use
+	            portage_manifest.Manifest() instead for any digest related tasks.
 	"""
+	print "!!! use of deprecated function digestCheckFiles(), use portage_manifest instead"""
+	return 0
 	for x in myfiles:
 		if not mydigests.has_key(x):
 			print
@@ -2260,8 +2153,46 @@ def digestCheckFiles(myfiles, mydigests, basedir, note="", strict=0):
 	return 1
 
 
-def digestcheck(myfiles, mysettings, strict=0, justmanifest=0):
-	"""Verifies checksums.  Assumes all files have been downloaded."""
+def digestcheck(myfiles, mysettings, strict=0, justmanifest=0, db=None):
+	"""Verifies checksums.  Assumes all files have been downloaded.
+	DEPRECATED: this is now only a compability wrapper for 
+	            portage_manifest.Manifest()."""
+	
+	pkgdir = mysettings["O"]
+	if db == None:
+		db = portagetree().dbapi
+	mf = Manifest(pkgdir, db, mysettings)
+	try:
+		if strict:
+			print ">>> checking ebuild checksums",
+			mf.checkTypeHashes("EBUILD")
+			print ":-)"
+			print ">>> checking auxfile checksums",
+			mf.checkTypeHashes("AUX")
+			print ":-)"
+			print ">>> checking miscfile checksums",
+			mf.checkTypeHashes("MISC", ignoreMissingFiles=True)
+			print ":-)"
+		for f in myfiles:
+			if f.startswith("files/"):
+				f = f[5:]
+			print ">>> checking %s checksums" % f,
+			mf.checkFileHashes(mf.findFile(f), f)	
+			print ":-)"
+	except portage_exception.DigestException, e:
+		print e.value
+		print red("!!! ")+"Digest verification failed:"
+		print red("!!! ")+"    "+e.value[0]
+		print red("!!! ")+"Reason: "+e.value[1]
+		print red("!!! ")+"Got: "+str(e.value[2])
+		print red("!!! ")+"Expected: "+str(e.value[3])
+		return 0
+	return 1
+	
+	#########################################
+	# Old code that's replaced by the above	#
+	#########################################   
+	
 	# archive files
 	basedir=mysettings["DISTDIR"]+"/"
 	digestfn=mysettings["FILESDIR"]+"/digest-"+mysettings["PF"]
diff --git a/pym/portage_checksum.py b/pym/portage_checksum.py
index 8196628ca..9ea1cc636 100644
--- a/pym/portage_checksum.py
+++ b/pym/portage_checksum.py
@@ -58,6 +58,11 @@ try:
 except ImportError:
 	pass
 
+def getsize(filename):
+	size = os.stat(filename).st_size
+	return (size, size)
+hashfunc_map["size"] = getsize
+
 # end actual hash functions
 
 prelink_capable = False
@@ -68,7 +73,7 @@ if os.path.exists(PRELINK_BINARY):
 	del results
 
 def perform_md5(x, calc_prelink=0):
-	return perform_checksum(x, md5hash, calc_prelink)[0]
+	return perform_checksum(x, "MD5", calc_prelink)[0]
 
 def perform_all(x, calc_prelink=0):
 	mydict = {}
@@ -94,7 +99,7 @@ def verify_all(filename, mydict, calc_prelink=0, strict=0):
 		if   x == "size":
 			continue
 		elif x in hashfunc_map.keys():
-			myhash = perform_checksum(filename, hashfunc_map[x], calc_prelink=calc_prelink)[0]
+			myhash = perform_checksum(filename, x, calc_prelink=calc_prelink)[0]
 			if mydict[x] != myhash:
 				if strict:
 					raise portage_exception.DigestException, "Failed to verify '$(file)s' on checksum type '%(type)s'" % {"file":filename, "type":x}
@@ -118,7 +123,7 @@ def pyhash(filename, hashobject):
 
 	return (sum.hexdigest(), size)
 
-def perform_checksum(filename, hash_function=md5hash, calc_prelink=0):
+def perform_checksum(filename, hashname="MD5", calc_prelink=0):
 	myfilename      = filename[:]
 	prelink_tmpfile = os.path.join("/", PRIVATE_PATH, "prelink-checksum.tmp." + str(os.getpid()))
 	mylock          = None
@@ -132,7 +137,9 @@ def perform_checksum(filename, hash_function=md5hash, calc_prelink=0):
 			#portage_util.writemsg(">>> prelink checksum '"+str(filename)+"'.\n")
 			myfilename=prelink_tmpfile
 	try:
-		myhash, mysize = hash_function(myfilename)
+		if hashname not in hashfunc_map:
+			raise portage_exception.DigestException, hashname+" hash function not available (needs dev-python/pycrypto)"
+		myhash, mysize = hashfunc_map[hashname](myfilename)
 	except (OSError, IOError), e:
 		if e.errno == errno.ENOENT:
 			raise portage_exception.FileNotFound(e)
@@ -155,5 +162,5 @@ def perform_multiple_checksums(filename, hashes=["MD5"], calc_prelink=0):
 	for x in hashes:
 		if x not in hashfunc_map:
 			raise portage_exception.DigestException, x+" hash function not available (needs dev-python/pycrypto)"
-		rVal[x] = perform_checksum(filename, hashfunc_map[x], calc_prelink)[0]
+		rVal[x] = perform_checksum(filename, x, calc_prelink)[0]
 	return rVal
diff --git a/pym/portage_const.py b/pym/portage_const.py
index 04b61c4b7..af8aac8b7 100644
--- a/pym/portage_const.py
+++ b/pym/portage_const.py
@@ -47,10 +47,10 @@ EBUILD_PHASES			= ["setup","unpack","compile","test","install","preinst","postin
 EAPI = 0
 
 HASHING_BLOCKSIZE		= 32768
-# Disabling until behaviour when missing the relevant python module is
-# corrected.  #116485
 MANIFEST1_HASH_FUNCTIONS = ["MD5","SHA256","RMD160"]
+MANIFEST2_HASH_FUNCTIONS = ["SHA1","SHA256","RMD160"]
 
+MANIFEST2_IDENTIFIERS = ["AUX","MISC","DIST","EBUILD"]
 # ===========================================================================
 # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT
 # ===========================================================================