diff options
| -rw-r--r-- | pym/portage_manifest.py | 333 |
1 files changed, 333 insertions, 0 deletions
diff --git a/pym/portage_manifest.py b/pym/portage_manifest.py new file mode 100644 index 000000000..9dddd6982 --- /dev/null +++ b/pym/portage_manifest.py @@ -0,0 +1,333 @@ +import os, sets + +import portage, portage_exception, portage_versions, portage_const +from portage_checksum import * +from portage_exception import * + +class FileNotInManifestException(PortageException): + pass + +def manifest2AuxfileFilter(filename): + filename = filename.strip(os.sep) + return not (filename in ["CVS", ".svn"] or filename.startswith("digest-")) + +def manifest2MiscfileFilter(filename): + filename = filename.strip(os.sep) + return not (filename in ["CVS", ".svn", "files", "Manifest"] or filename.endswith(".ebuild")) + +class Manifest(object): + def __init__(self, pkgdir, db, mysettings, manifest1_compat=True, from_scratch=False): + """ create new Manifest instance for package in pkgdir, using db and mysettings for metadata lookups, + and add compability entries for old portage versions if manifest1_compat == True. + Do not parse Manifest file if from_scratch == True (only for internal use) """ + self.pkgdir = pkgdir+os.sep + self.fhashdict = {} + self.hashes = portage_const.MANIFEST2_HASH_FUNCTIONS[:] + self.hashes.append("size") + if manifest1_compat: + self.hashes.extend(portage_const.MANIFEST1_HASH_FUNCTIONS) + self.hashes = sets.Set(self.hashes) + for t in portage_const.MANIFEST2_IDENTIFIERS: + self.fhashdict[t] = {} + if not from_scratch: + self._read() + self.compat = manifest1_compat + self.db = db + self.mysettings = mysettings + if mysettings.has_key("PORTAGE_ACTUAL_DISTDIR"): + self.distdir = mysettings["PORTAGE_ACTUAL_DISTDIR"] + else: + self.distdir = mysettings["DISTDIR"] + + def guessType(self, filename): + """ Perform a best effort guess of which type the given filename is, avoid using this if possible """ + if filename.startswith("files"+os.sep+"digest-"): + return None + if filename.startswith("files"+os.sep): + return "AUX" + elif filename.endswith(".ebuild"): + return "EBUILD" + elif filename in ["ChangeLog", "metadata.xml"]: + return "MISC" + else: + return "DIST" + + def getFullname(self): + """ Returns the absolute path to the Manifest file for this instance """ + return os.path.join(self.pkgdir, "Manifest") + + def getDigests(self): + """ Compability function for old digest/manifest code, returns dict of filename:{hashfunction:hashvalue} """ + rval = {} + for t in portage_const.MANIFEST2_IDENTIFIERS: + rval.update(self.fhashdict[t]) + return rval + + def _readDigests(self): + """ Parse old style digest files for this Manifest instance """ + mycontent = "" + for d in portage.listdir(os.path.join(self.pkgdir, "files"), filesonly=True, recursive=False): + if d.startswith("digest-"): + mycontent += open(os.path.join(self.pkgdir, "files", d), "r").read() + return mycontent + + def _read(self): + """ Parse Manifest file for this instance """ + if not os.path.exists(self.getFullname()): + return + fd = open(self.getFullname(), "r") + mylines = fd.readlines() + fd.close() + mylines.extend(self._readDigests().split("\n")) + for l in mylines: + myname = "" + mysplit = l.split() + if len(mysplit) == 4 and mysplit[0] in portage_const.MANIFEST1_HASH_FUNCTIONS: + myname = mysplit[2] + mytype = self.guessType(myname) + if mytype == "AUX" and myname.startswith("files"+os.sep): + myname = myname[6:] + if mytype == None: + continue + mysize = int(mysplit[3]) + myhashes = {mysplit[0]: mysplit[1]} + if len(mysplit) > 4 and mysplit[0] in portage_const.MANIFEST2_IDENTIFIERS: + mytype = mysplit[0] + myname = mysplit[1] + mysize = int(mysplit[2]) + myhashes = dict(zip(mysplit[3::2], mysplit[4::2])) + if len(myname) == 0: + continue + if not self.fhashdict[mytype].has_key(myname): + self.fhashdict[mytype][myname] = {} + self.fhashdict[mytype][myname].update(myhashes) + self.fhashdict[mytype][myname]["size"] = mysize + + def _writeDigests(self): + """ Create old style digest files for this Manifest instance """ + cpvlist = [os.path.join(self.pkgdir.rstrip(os.sep).split(os.sep)[-2], x[:-7]) for x in portage.listdir(self.pkgdir) if x.endswith(".ebuild")] + rval = [] + for cpv in cpvlist: + dname = os.path.join(self.pkgdir, "files", "digest-"+portage.catsplit(cpv)[1]) + mylines = [] + distlist = self._getCpvDistfiles(cpv) + for f in self.fhashdict["DIST"].keys(): + if f in distlist: + for h in self.fhashdict["DIST"][f].keys(): + if h not in portage_const.MANIFEST1_HASH_FUNCTIONS: + continue + myline = " ".join([h, str(self.fhashdict["DIST"][f][h]), f, str(self.fhashdict["DIST"][f]["size"])]) + mylines.append(myline) + fd = open(dname, "w") + fd.write("\n".join(mylines)) + fd.write("\n") + fd.close() + rval.append(dname) + return rval + + def _addDigestsToManifest(self, digests, fd): + """ Add entries for old style digest files to Manifest file """ + mylines = [] + for dname in digests: + myhashes = perform_multiple_checksums(dname, portage_const.MANIFEST1_HASH_FUNCTIONS+["size"]) + for h in myhashes.keys(): + mylines.append((" ".join([h, str(myhashes[h]), os.path.join("files", os.path.basename(dname)), str(myhashes["size"])]))) + fd.write("\n".join(mylines)) + fd.write("\n") + + def _write(self, fd): + """ Actual Manifest file generator """ + mylines = [] + for t in self.fhashdict.keys(): + for f in self.fhashdict[t].keys(): + myline = " ".join([t, f, str(self.fhashdict[t][f]["size"])]) + myhashes = self.fhashdict[t][f] + for h in myhashes.keys(): + if h not in portage_const.MANIFEST2_HASH_FUNCTIONS: + continue + myline += " "+h+" "+str(myhashes[h]) + mylines.append(myline) + if self.compat and t != "DIST": + for h in myhashes.keys(): + if h not in portage_const.MANIFEST1_HASH_FUNCTIONS: + continue + mylines.append((" ".join([h, str(myhashes[h]), f, str(myhashes["size"])]))) + fd.write("\n".join(mylines)) + fd.write("\n") + + def write(self, sign=False): + """ Write Manifest instance to disk, optionally signing it """ + fd = open(self.getFullname(), "w") + self._write(fd) + if self.compat: + digests = self._writeDigests() + self._addDigestsToManifest(digests, fd) + fd.close() + if sign: + self.sign() + + def sign(self): + """ Sign the Manifest """ + raise NotImplementedError() + + def validateSignature(self): + """ Validate signature on Manifest """ + raise NotImplementedError() + + def addFile(self, ftype, fname, hashdict=None): + """ Add entry to Manifest optionally using hashdict to avoid recalculation of hashes """ + if not os.path.exists(self.pkgdir+fname): + raise FileNotFound(fname) + if not ftype in portage_const.MANIFEST2_IDENTIFIERS: + raise InvalidDataType(ftype) + self.fhashdict[ftype][fname] = {} + if hashdict != None: + self.fhashdict[ftype][fname].update(hashdict) + if not portage_const.MANIFEST2_REQUIRED_HASH in self.fhashdict[ftype][fname].keys(): + self.updateFileHashes(ftype, fname) + + def removeFile(self, ftype, fname): + """ Remove given entry from Manifest """ + del self.fhashdict[ftype][fname] + + def hasFile(self, ftype, fname): + """ Return wether the Manifest contains an entry for the given type,filename pair """ + return (fname in self.fhashdict[ftype].keys()) + + def findFile(self, fname): + """ Return entrytype of the given file if present in Manifest or None if not present """ + for t in portage_const.MANIFEST2_IDENTIFIERS: + if fname in self.fhashdict[t]: + return t + return None + + def create(self, checkExisting=False, assumeDistfileHashes=True): + """ Recreate this Manifest from scratch, not using any existing checksums + (exception: if assumeDistfileHashes is true then existing DIST checksums are + reused if the file doesn't exist in DISTDIR.""" + if checkExisting: + self.checkAllHashes() + if assumeDistfileHashes: + distfilehashes = self.fhashdict["DIST"] + else: + distfilehashes = {} + self.__init__(self.pkgdir, self.db, self.mysettings, from_scratch=True) + for f in portage.listdir(self.pkgdir, filesonly=True, recursive=False): + if f.endswith(".ebuild"): + mytype = "EBUILD" + elif manifest2MiscfileFilter(f): + mytype = "MISC" + else: + continue + self.fhashdict[mytype][f] = perform_multiple_checksums(self.pkgdir+f, self.hashes) + for f in portage.listdir(self.pkgdir+"files", filesonly=True, recursive=True): + if not manifest2AuxfileFilter(f): + continue + self.fhashdict["AUX"][f] = perform_multiple_checksums(self.pkgdir+"files"+os.sep+f, self.hashes) + cpvlist = [os.path.join(self.pkgdir.rstrip(os.sep).split(os.sep)[-2], x[:-7]) for x in portage.listdir(self.pkgdir) if x.endswith(".ebuild")] + distlist = [] + for cpv in cpvlist: + distlist.extend(self._getCpvDistfiles(cpv)) + for f in distlist: + fname = os.path.join(self.distdir, f) + if os.path.exists(fname): + self.fhashdict["DIST"][f] = perform_multiple_checksums(fname, self.hashes) + elif assumeDistfileHashes and f in distfilehashes.keys(): + self.fhashdict["DIST"][f] = distfilehashes[f] + else: + raise FileNotFound(fname) + + def _getAbsname(self, ftype, fname): + if ftype == "DIST": + absname = os.path.join(self.distdir, fname) + elif ftype == "AUX": + absname = os.path.join(self.pkgdir, "files", fname) + else: + absname = os.path.join(self.pkgdir, fname) + return absname + + def checkAllHashes(self, ignoreMissingFiles=False): + for t in portage_const.MANIFEST2_IDENTIFIERS: + self.checkTypeHashes(t, ignoreMissingFiles=ignoreMissingFiles) + + def checkTypeHashes(self, idtype, ignoreMissingFiles=False): + for f in self.fhashdict[idtype].keys(): + self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles) + + def checkFileHashes(self, ftype, fname, ignoreMissing=False): + myhashes = self.fhashdict[ftype][fname] + ok,reason = verify_all(self._getAbsname(ftype, fname), self.fhashdict[ftype][fname]) + if not ok: + raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason))) + return ok, reason + + def checkCpvHashes(self, cpv, checkDistfiles=True, onlyDistfiles=False, checkMiscfiles=False): + """ check the hashes for all files associated to the given cpv, include all + AUX files and optionally all MISC files. """ + if not onlyDistfiles: + self.checkTypeHashes("AUX", ignoreMissingFiles=False) + if checkMiscfiles: + self.checkTypeHashes("MISC", ignoreMissingFiles=False) + ebuildname = portage.catsplit(cpv)[1]+".ebuild" + self.checkFileHashes("EBUILD", ebuildname, ignoreMissing=False) + if checkDistfiles: + if onlyDistfiles: + for f in self._getCpvDistfiles(cpv): + self.checkFileHashes("DIST", f, ignoreMissing=False) + + def _getCpvDistfiles(self, cpv): + """ Get a list of all DIST files associated to the given cpv """ + return self.db.getfetchlist(cpv, mysettings=self.mysettings, all=True)[1] + + def updateFileHashes(self, ftype, fname, checkExisting=True, ignoreMissing=True): + """ Regenerate hashes for the given file """ + if checkExisting: + self.checkFileHashes(fname) + if not ignoreMissing and not self.fhashdict[ftype].has_key(fname): + raise FileNotInManifestException(fname) + if not self.fhashdict[ftype].has_key(fname): + self.fhashdict[ftype][fname] = {} + myhashes = perform_multiple_checksums(self._getAbsname(ftype, fname), self.hashes) + self.fhashdict[ftype][fname].update(myhashes) + + def updateTypeHashes(self, idtype, checkExisting=False, ignoreMissingFiles=True): + """ Regenerate all hashes for all files of the given type """ + for fname in self.fhashdict[idtype].keys(): + self.updateFileHashes(idtype, fname, checkExisting) + + def updateAllHashes(self, checkExisting=False, ignoreMissingFiles=True): + """ Regenerate all hashes for all files in this Manifest. """ + for ftype in portage_const.MANIFEST2_IDENTIFIERS: + self.updateTypeHashes(idtype, fname, checkExisting) + + def updateCpvHashes(self, cpv, ignoreMissingFiles=True): + """ Regenerate all hashes associated to the given cpv (includes all AUX and MISC + files).""" + self.updateTypeHashes("AUX", ignoreMissingFiles=ignoreMissingFiles) + self.updateTypeHashes("MISC", ignoreMissingFiles=ignoreMissingFiles) + ebuildname = portage.catsplit(cpv)[1]+".ebuild" + self.updateFileHashes("EBUILD", ebuildname, ignoreMissingFiles=ignoreMissingFiles) + for f in self._getCpvDistfiles(cpv): + self.updateFileHashes("DIST", f, ignoreMissingFiles=ignoreMissingFiles) + + def getFileData(self, ftype, fname, key): + """ Return the value of a specific (type,filename,key) triple, mainly useful + to get the size for distfiles.""" + return self.fhashdict[ftype][fname][key] + + def getVersions(self): + """ Returns a list of manifest versions present in the manifest file. """ + rVal = [] + mfname = self.getFullname() + if not os.path.exists(mfname): + return rVal + myfile = open(mfname, "r") + lines = myfile.readlines() + myfile.close() + for l in lines: + mysplit = l.split() + if len(mysplit) == 4 and mysplit[0] in portage_const.MANIFEST1_HASH_FUNCTIONS and not 1 in rVal: + rVal.append(1) + elif len(mysplit) > 4 and mysplit[0] in portage_const.MANIFEST2_IDENTIFIERS and ((len(mysplit) - 3) % 2) == 0 and not 2 in rVal: + rVal.append(2) + return rVal |