adding multi-hash support

svn path=/main/trunk/; revision=2410

3 files changed, 81 insertions, 65 deletions

diff --git a/pym/portage.py b/pym/portage.py
index 452707349..a5295f21e 100644
--- a/pym/portage.py
+++ b/pym/portage.py
@@ -1917,7 +1917,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 							if not fetchonly:
 								fetched=2
 							else:
-								# Check md5sum's at each fetch for fetchonly.
+								# Verify checksums at each fetch for fetchonly.
 								verified_ok,reason = portage_checksum.verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile])
 								if not verified_ok:
 									writemsg("!!! Previously fetched file: "+str(myfile)+"\n!!! Reason: "+reason+"\nRefetching...\n\n")
@@ -1990,7 +1990,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 						try:
 							mystat=os.stat(mysettings["DISTDIR"]+"/"+myfile)
 							# no exception?  file exists. let digestcheck() report
-							# an appropriately for size or md5 errors
+							# an appropriately for size or checksum errors
 							if (mystat[stat.ST_SIZE]<mydigests[myfile]["size"]):
 								# Fetch failed... Try the next one... Kill 404 files though.
 								if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")):
@@ -2013,7 +2013,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 								fetched=2
 								break
 							else:
-								# File is the correct size--check the MD5 sum for the fetched
+								# File is the correct size--check the checksums for the fetched
 								# file NOW, for those users who don't have a stable/continuous
 								# net connection. This way we have a chance to try to download
 								# from another mirror...
@@ -2061,7 +2061,7 @@ def digestCreate(myfiles,basedir,oldDigest={}):
 				print "!!! Given file does not appear to be readable. Does it exist?"
 				print "!!! File:",myfile
 				return None
-			mydigests[x] = portage_checksum.perform_all(myfile)
+			mydigests[x] = portage_checksum.perform_multiple_checksums(myfile, hashes=portage_const.MANIFEST1_HASH_FUNCTIONS)
 			mysize       = os.stat(myfile)[stat.ST_SIZE]
 		else:
 			if x in oldDigest:
@@ -2094,11 +2094,6 @@ def digestCreateLines(filelist, mydict):
 			myline += " "+mysum
 			myline += " "+myarchive
 			myline += " "+str(mysize)
-			if sumName != "MD5":
-				# XXXXXXXXXXXXXXXX This cannot be used!
-				# Older portage make very dumb assumptions about the formats.
-				# We need a lead-in period before we break everything.
-				continue
 			mylines.append(myline)
 	return mylines
 
@@ -2219,9 +2214,10 @@ def digestgen(myarchives,mysettings,overwrite=1,manifestonly=0):
 
 def digestParseFile(myfilename):
 	"""(filename) -- Parses a given file for entries matching:
-	MD5 MD5_STRING_OF_HEX_CHARS FILE_NAME FILE_SIZE
-	Ignores lines that do not begin with 'MD5' and returns a
-	dict with the filenames as keys and [md5,size] as the values."""
+	<checksumkey> <checksum_hex_string> <filename> <filesize>
+	Ignores lines that don't start with a valid checksum identifier
+	and returns a dict with the filenames as keys and {checksumkey:checksum}
+	as the values."""
 
 	if not os.path.exists(myfilename):
 		return None
@@ -2254,7 +2250,7 @@ def digestParseFile(myfilename):
 def digestCheckFiles(myfiles, mydigests, basedir, note="", strict=0):
 	"""(fileslist, digestdict, basedir) -- Takes a list of files and a dict
 	of their digests and checks the digests against the indicated files in
-	the basedir given. Returns 1 only if all files exist and match the md5s.
+	the basedir given. Returns 1 only if all files exist and match the checksums.
 	"""
 	for x in myfiles:
 		if not mydigests.has_key(x):
@@ -2281,12 +2277,12 @@ def digestCheckFiles(myfiles, mydigests, basedir, note="", strict=0):
 			print
 			return 0
 		else:
-			print ">>> md5 "+note+" ;-)",x
+			print ">>> checksums "+note+" ;-)",x
 	return 1
 
 
 def digestcheck(myfiles, mysettings, strict=0, justmanifest=0):
-	"""Checks md5sums.  Assumes all files have been downloaded."""
+	"""Verifies checksums.  Assumes all files have been downloaded."""
 	# archive files
 	basedir=mysettings["DISTDIR"]+"/"
 	digestfn=mysettings["FILESDIR"]+"/digest-"+mysettings["PF"]
@@ -2328,6 +2324,7 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0):
 		# Check the portage-related files here.
 		mymfiles=listdir(pbasedir,recursive=1,filesonly=1,ignorecvs=1,EmptyOnError=1)
 		manifest_files = mymdigests.keys()
+		# Files unrelated to the build process are ignored for verification by default
 		for x in ["Manifest", "ChangeLog", "metadata.xml"]:
 			while x in mymfiles:
 				mymfiles.remove(x)
@@ -4925,8 +4922,8 @@ class portdbapi(dbapi):
 	def getfetchsizes(self,mypkg,useflags=None,debug=0):
 		# returns a filename:size dictionnary of remaining downloads
 		mydigest=self.finddigest(mypkg)
-		mymd5s=digestParseFile(mydigest)
-		if not mymd5s:
+		checksums=digestParseFile(mydigest)
+		if not checksums:
 			if debug: print "[empty/missing/bad digest]: "+mypkg
 			return None
 		filesdict={}
@@ -4935,14 +4932,14 @@ class portdbapi(dbapi):
 		else:
 			myuris, myfiles = self.getfetchlist(mypkg,useflags=useflags)
 		#XXX: maybe this should be improved: take partial downloads
-		# into account? check md5sums?
+		# into account? check checksums?
 		for myfile in myfiles:
-			if debug and myfile not in mymd5s.keys():
+			if debug and myfile not in checksums.keys():
 				print "[bad digest]: missing",myfile,"for",mypkg
-			elif myfile in mymd5s.keys():
+			elif myfile in checksums.keys():
 				distfile=settings["DISTDIR"]+"/"+myfile
 				if not os.access(distfile, os.R_OK):
-					filesdict[myfile]=int(mymd5s[myfile]["size"])
+					filesdict[myfile]=int(checksums[myfile]["size"])
 		return filesdict
 
 	def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False):
diff --git a/pym/portage_checksum.py b/pym/portage_checksum.py
index 66f6f181d..5625f2322 100644
--- a/pym/portage_checksum.py
+++ b/pym/portage_checksum.py
@@ -4,7 +4,7 @@
 # $Id: /var/cvsroot/gentoo-src/portage/pym/portage_checksum.py,v 1.10.2.2 2005/08/10 05:42:03 ferringb Exp $
 
 
-from portage_const import PRIVATE_PATH,PRELINK_BINARY
+from portage_const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE
 import os
 import shutil
 import stat
@@ -14,6 +14,50 @@ import portage_locks
 import commands
 import sha
 
+
+# actual hash functions first
+
+#dict of all available hash functions
+hashfunc_map = {}
+
+# We _try_ to load this module. If it fails we do the slightly slower fallback.
+try:
+	import fchksum
+	
+	def md5hash(filename):
+		return fchksum.fmd5t(filename)
+
+except ImportError:
+	import md5
+	def md5hash(filename):
+		return pyhash(filename, md5)
+hashfunc_map["MD5"] = md5hash
+
+def sha1hash(filename):
+	return pyhash(filename, sha)
+hashfunc_map["SHA1"] = sha1hash
+	
+# Keep pycrypto optional for now, there are no internal fallbacks for these
+try:
+	import Crypto.Hash.SHA256
+	
+	def sha256hash(filename):
+		return pyhash(filename, Crypto.Hash.SHA256)
+	hashfunc_map["SHA256"] = sha256hash
+except ImportError:
+	pass
+
+try:
+	import Crypto.Hash.RIPEMD
+	
+	def rmd160hash(filename):
+		return pyhash(filename, Crypto.Hash.RIPEMD)
+	hashfunc_map["RMD160"] = rmd160hash
+except ImportError:
+	pass
+
+# end actual hash functions
+
 prelink_capable = False
 if os.path.exists(PRELINK_BINARY):
 	results = commands.getstatusoutput(PRELINK_BINARY+" --version > /dev/null 2>&1")
@@ -24,17 +68,14 @@ if os.path.exists(PRELINK_BINARY):
 def perform_md5(x, calc_prelink=0):
 	return perform_checksum(x, md5hash, calc_prelink)[0]
 
-def perform_sha1(x, calc_prelink=0):
-	return perform_checksum(x, sha1hash, calc_prelink)[0]
-
 def perform_all(x, calc_prelink=0):
 	mydict = {}
-	mydict["SHA1"] = perform_sha1(x, calc_prelink)
-	mydict["MD5"] = perform_md5(x, calc_prelink)
+	for k in hashfunc_map.keys():
+		mydict[k] = perform_checksum(x, hashfunc_map[k], calc_prelink)[0]
 	return mydict
 
 def get_valid_checksum_keys():
-	return ["SHA1", "MD5"]
+	return hashfunc_map.keys()
 
 def verify_all(filename, mydict, calc_prelink=0, strict=0):
 	# Dict relates to single file only.
@@ -49,16 +90,8 @@ def verify_all(filename, mydict, calc_prelink=0, strict=0):
 	for x in mydict.keys():
 		if   x == "size":
 			continue
-		elif x == "SHA1":
-			if mydict[x] != perform_sha1(filename, calc_prelink=calc_prelink):
-				if strict:
-					raise portage_exception.DigestException, "Failed to verify '$(file)s' on checksum type '%(type)s'" % {"file":filename, "type":x}
-				else:
-					file_is_ok = False
-					reason     = "Failed on %s verification" % (x,)
-					break
-		elif x == "MD5":
-			if mydict[x] != perform_md5(filename, calc_prelink=calc_prelink):
+		elif x in hashfunc_map.keys():
+			if mydict[x] != perform_checksum(filename, hashfunc_map[x], calc_prelink=calc_prelink)[0]:
 				if strict:
 					raise portage_exception.DigestException, "Failed to verify '$(file)s' on checksum type '%(type)s'" % {"file":filename, "type":x}
 				else:
@@ -67,42 +100,19 @@ def verify_all(filename, mydict, calc_prelink=0, strict=0):
 					break
 	return file_is_ok,reason
 
-# We _try_ to load this module. If it fails we do the slow fallback.
-try:
-	import fchksum
-	
-	def md5hash(filename):
-		return fchksum.fmd5t(filename)
-
-except ImportError:
-	import md5
-	def md5hash(filename):
-		f = open(filename, 'rb')
-		blocksize=32768
-		data = f.read(blocksize)
-		size = 0L
-		sum = md5.new()
-		while data:
-			sum.update(data)
-			size = size + len(data)
-			data = f.read(blocksize)
-		f.close()
-
-		return (sum.hexdigest(),size)
-
-def sha1hash(filename):
+def pyhash(filename, hashobject):
 	f = open(filename, 'rb')
-	blocksize=32768
+	blocksize = HASHING_BLOCKSIZE
 	data = f.read(blocksize)
 	size = 0L
-	sum = sha.new()
+	sum = hashobject.new()
 	while data:
 		sum.update(data)
 		size = size + len(data)
 		data = f.read(blocksize)
 	f.close()
 
-	return (sum.hexdigest(),size)
+	return (sum.hexdigest(), size)
 
 def perform_checksum(filename, hash_function=md5hash, calc_prelink=0):
 	myfilename      = filename[:]
@@ -126,3 +136,9 @@ def perform_checksum(filename, hash_function=md5hash, calc_prelink=0):
 		portage_locks.unlockfile(mylock)
 
 	return (myhash,mysize)
+
+def perform_multiple_checksums(filename, hashes=["MD5"], calc_prelink=0):
+	rVal = {}
+	for x in hashes:
+		rVal[x] = perform_checksum(filename, hashfunc_map[x], calc_prelink)[0]
+	return rVal
diff --git a/pym/portage_const.py b/pym/portage_const.py
index b58c62696..6ef92ccd0 100644
--- a/pym/portage_const.py
+++ b/pym/portage_const.py
@@ -45,6 +45,9 @@ EBUILD_PHASES			= ["setup","unpack","compile","test","install","preinst","postin
 
 EAPI = 0
 
+HASHING_BLOCKSIZE		= 32768
+MANIFEST1_HASH_FUNCTIONS = ["MD5","SHA256","RMD160"]
+
 # ===========================================================================
 # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT
 # ===========================================================================