diff options
| -rw-r--r-- | pym/portage.py | 39 | ||||
| -rw-r--r-- | pym/portage_checksum.py | 104 | ||||
| -rw-r--r-- | pym/portage_const.py | 3 |
3 files changed, 81 insertions, 65 deletions
diff --git a/pym/portage.py b/pym/portage.py index 452707349..a5295f21e 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -1917,7 +1917,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if not fetchonly: fetched=2 else: - # Check md5sum's at each fetch for fetchonly. + # Verify checksums at each fetch for fetchonly. verified_ok,reason = portage_checksum.verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile]) if not verified_ok: writemsg("!!! Previously fetched file: "+str(myfile)+"\n!!! Reason: "+reason+"\nRefetching...\n\n") @@ -1990,7 +1990,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", try: mystat=os.stat(mysettings["DISTDIR"]+"/"+myfile) # no exception? file exists. let digestcheck() report - # an appropriately for size or md5 errors + # an appropriately for size or checksum errors if (mystat[stat.ST_SIZE]<mydigests[myfile]["size"]): # Fetch failed... Try the next one... Kill 404 files though. if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")): @@ -2013,7 +2013,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", fetched=2 break else: - # File is the correct size--check the MD5 sum for the fetched + # File is the correct size--check the checksums for the fetched # file NOW, for those users who don't have a stable/continuous # net connection. This way we have a chance to try to download # from another mirror... @@ -2061,7 +2061,7 @@ def digestCreate(myfiles,basedir,oldDigest={}): print "!!! Given file does not appear to be readable. Does it exist?" print "!!! File:",myfile return None - mydigests[x] = portage_checksum.perform_all(myfile) + mydigests[x] = portage_checksum.perform_multiple_checksums(myfile, hashes=portage_const.MANIFEST1_HASH_FUNCTIONS) mysize = os.stat(myfile)[stat.ST_SIZE] else: if x in oldDigest: @@ -2094,11 +2094,6 @@ def digestCreateLines(filelist, mydict): myline += " "+mysum myline += " "+myarchive myline += " "+str(mysize) - if sumName != "MD5": - # XXXXXXXXXXXXXXXX This cannot be used! - # Older portage make very dumb assumptions about the formats. - # We need a lead-in period before we break everything. - continue mylines.append(myline) return mylines @@ -2219,9 +2214,10 @@ def digestgen(myarchives,mysettings,overwrite=1,manifestonly=0): def digestParseFile(myfilename): """(filename) -- Parses a given file for entries matching: - MD5 MD5_STRING_OF_HEX_CHARS FILE_NAME FILE_SIZE - Ignores lines that do not begin with 'MD5' and returns a - dict with the filenames as keys and [md5,size] as the values.""" + <checksumkey> <checksum_hex_string> <filename> <filesize> + Ignores lines that don't start with a valid checksum identifier + and returns a dict with the filenames as keys and {checksumkey:checksum} + as the values.""" if not os.path.exists(myfilename): return None @@ -2254,7 +2250,7 @@ def digestParseFile(myfilename): def digestCheckFiles(myfiles, mydigests, basedir, note="", strict=0): """(fileslist, digestdict, basedir) -- Takes a list of files and a dict of their digests and checks the digests against the indicated files in - the basedir given. Returns 1 only if all files exist and match the md5s. + the basedir given. Returns 1 only if all files exist and match the checksums. """ for x in myfiles: if not mydigests.has_key(x): @@ -2281,12 +2277,12 @@ def digestCheckFiles(myfiles, mydigests, basedir, note="", strict=0): print return 0 else: - print ">>> md5 "+note+" ;-)",x + print ">>> checksums "+note+" ;-)",x return 1 def digestcheck(myfiles, mysettings, strict=0, justmanifest=0): - """Checks md5sums. Assumes all files have been downloaded.""" + """Verifies checksums. Assumes all files have been downloaded.""" # archive files basedir=mysettings["DISTDIR"]+"/" digestfn=mysettings["FILESDIR"]+"/digest-"+mysettings["PF"] @@ -2328,6 +2324,7 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0): # Check the portage-related files here. mymfiles=listdir(pbasedir,recursive=1,filesonly=1,ignorecvs=1,EmptyOnError=1) manifest_files = mymdigests.keys() + # Files unrelated to the build process are ignored for verification by default for x in ["Manifest", "ChangeLog", "metadata.xml"]: while x in mymfiles: mymfiles.remove(x) @@ -4925,8 +4922,8 @@ class portdbapi(dbapi): def getfetchsizes(self,mypkg,useflags=None,debug=0): # returns a filename:size dictionnary of remaining downloads mydigest=self.finddigest(mypkg) - mymd5s=digestParseFile(mydigest) - if not mymd5s: + checksums=digestParseFile(mydigest) + if not checksums: if debug: print "[empty/missing/bad digest]: "+mypkg return None filesdict={} @@ -4935,14 +4932,14 @@ class portdbapi(dbapi): else: myuris, myfiles = self.getfetchlist(mypkg,useflags=useflags) #XXX: maybe this should be improved: take partial downloads - # into account? check md5sums? + # into account? check checksums? for myfile in myfiles: - if debug and myfile not in mymd5s.keys(): + if debug and myfile not in checksums.keys(): print "[bad digest]: missing",myfile,"for",mypkg - elif myfile in mymd5s.keys(): + elif myfile in checksums.keys(): distfile=settings["DISTDIR"]+"/"+myfile if not os.access(distfile, os.R_OK): - filesdict[myfile]=int(mymd5s[myfile]["size"]) + filesdict[myfile]=int(checksums[myfile]["size"]) return filesdict def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False): diff --git a/pym/portage_checksum.py b/pym/portage_checksum.py index 66f6f181d..5625f2322 100644 --- a/pym/portage_checksum.py +++ b/pym/portage_checksum.py @@ -4,7 +4,7 @@ # $Id: /var/cvsroot/gentoo-src/portage/pym/portage_checksum.py,v 1.10.2.2 2005/08/10 05:42:03 ferringb Exp $ -from portage_const import PRIVATE_PATH,PRELINK_BINARY +from portage_const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE import os import shutil import stat @@ -14,6 +14,50 @@ import portage_locks import commands import sha + +# actual hash functions first + +#dict of all available hash functions +hashfunc_map = {} + +# We _try_ to load this module. If it fails we do the slightly slower fallback. +try: + import fchksum + + def md5hash(filename): + return fchksum.fmd5t(filename) + +except ImportError: + import md5 + def md5hash(filename): + return pyhash(filename, md5) +hashfunc_map["MD5"] = md5hash + +def sha1hash(filename): + return pyhash(filename, sha) +hashfunc_map["SHA1"] = sha1hash + +# Keep pycrypto optional for now, there are no internal fallbacks for these +try: + import Crypto.Hash.SHA256 + + def sha256hash(filename): + return pyhash(filename, Crypto.Hash.SHA256) + hashfunc_map["SHA256"] = sha256hash +except ImportError: + pass + +try: + import Crypto.Hash.RIPEMD + + def rmd160hash(filename): + return pyhash(filename, Crypto.Hash.RIPEMD) + hashfunc_map["RMD160"] = rmd160hash +except ImportError: + pass + +# end actual hash functions + prelink_capable = False if os.path.exists(PRELINK_BINARY): results = commands.getstatusoutput(PRELINK_BINARY+" --version > /dev/null 2>&1") @@ -24,17 +68,14 @@ if os.path.exists(PRELINK_BINARY): def perform_md5(x, calc_prelink=0): return perform_checksum(x, md5hash, calc_prelink)[0] -def perform_sha1(x, calc_prelink=0): - return perform_checksum(x, sha1hash, calc_prelink)[0] - def perform_all(x, calc_prelink=0): mydict = {} - mydict["SHA1"] = perform_sha1(x, calc_prelink) - mydict["MD5"] = perform_md5(x, calc_prelink) + for k in hashfunc_map.keys(): + mydict[k] = perform_checksum(x, hashfunc_map[k], calc_prelink)[0] return mydict def get_valid_checksum_keys(): - return ["SHA1", "MD5"] + return hashfunc_map.keys() def verify_all(filename, mydict, calc_prelink=0, strict=0): # Dict relates to single file only. @@ -49,16 +90,8 @@ def verify_all(filename, mydict, calc_prelink=0, strict=0): for x in mydict.keys(): if x == "size": continue - elif x == "SHA1": - if mydict[x] != perform_sha1(filename, calc_prelink=calc_prelink): - if strict: - raise portage_exception.DigestException, "Failed to verify '$(file)s' on checksum type '%(type)s'" % {"file":filename, "type":x} - else: - file_is_ok = False - reason = "Failed on %s verification" % (x,) - break - elif x == "MD5": - if mydict[x] != perform_md5(filename, calc_prelink=calc_prelink): + elif x in hashfunc_map.keys(): + if mydict[x] != perform_checksum(filename, hashfunc_map[x], calc_prelink=calc_prelink)[0]: if strict: raise portage_exception.DigestException, "Failed to verify '$(file)s' on checksum type '%(type)s'" % {"file":filename, "type":x} else: @@ -67,42 +100,19 @@ def verify_all(filename, mydict, calc_prelink=0, strict=0): break return file_is_ok,reason -# We _try_ to load this module. If it fails we do the slow fallback. -try: - import fchksum - - def md5hash(filename): - return fchksum.fmd5t(filename) - -except ImportError: - import md5 - def md5hash(filename): - f = open(filename, 'rb') - blocksize=32768 - data = f.read(blocksize) - size = 0L - sum = md5.new() - while data: - sum.update(data) - size = size + len(data) - data = f.read(blocksize) - f.close() - - return (sum.hexdigest(),size) - -def sha1hash(filename): +def pyhash(filename, hashobject): f = open(filename, 'rb') - blocksize=32768 + blocksize = HASHING_BLOCKSIZE data = f.read(blocksize) size = 0L - sum = sha.new() + sum = hashobject.new() while data: sum.update(data) size = size + len(data) data = f.read(blocksize) f.close() - return (sum.hexdigest(),size) + return (sum.hexdigest(), size) def perform_checksum(filename, hash_function=md5hash, calc_prelink=0): myfilename = filename[:] @@ -126,3 +136,9 @@ def perform_checksum(filename, hash_function=md5hash, calc_prelink=0): portage_locks.unlockfile(mylock) return (myhash,mysize) + +def perform_multiple_checksums(filename, hashes=["MD5"], calc_prelink=0): + rVal = {} + for x in hashes: + rVal[x] = perform_checksum(filename, hashfunc_map[x], calc_prelink)[0] + return rVal diff --git a/pym/portage_const.py b/pym/portage_const.py index b58c62696..6ef92ccd0 100644 --- a/pym/portage_const.py +++ b/pym/portage_const.py @@ -45,6 +45,9 @@ EBUILD_PHASES = ["setup","unpack","compile","test","install","preinst","postin EAPI = 0 +HASHING_BLOCKSIZE = 32768 +MANIFEST1_HASH_FUNCTIONS = ["MD5","SHA256","RMD160"] + # =========================================================================== # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT # =========================================================================== |