diff options
| -rw-r--r-- | NEWS | 1 | ||||
| -rwxr-xr-x | bin/mirror.py | 167 |
2 files changed, 1 insertions, 167 deletions
@@ -35,3 +35,4 @@ portage-2.1 (ongoing via pre releases) * Tests show that file locking is now more reliable. * A bash call stack is printed when an ebuild dies in ebuild.sh. * New rsync option handling by using a generic PORTAGE_RSYNC_EXTRA_OPTS variable +* Manifest2 support that will allow digest-* files to be eliminated from the tree. diff --git a/bin/mirror.py b/bin/mirror.py deleted file mode 100755 index 2ad07acdc..000000000 --- a/bin/mirror.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/python -O -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/bin/mirror.py,v 1.3 2004/10/10 10:07:20 carpaski Exp $ - -# Defines the number of threads carrying out the downloading. -maxsems=5 - -import os,sys,string -os.environ["PORTAGE_CALLER"]="mirror" -os.environ["FEATURES"]="mirror cvs" -sys.path = ["/usr/lib/portage/pym"]+sys.path - -import portage -from threading import * -from output import red,green,blue,bold -from random import shuffle -from time import sleep - - -def cstrip(mystr,mychars): - newstr = "" - for x in mystr: - if x not in mychars: - newstr += x - return newstr - -class fetcher(Thread): - def __init__(self, filename, urilist, dest, md5sum): - self.filename = filename - self.myurilist = urilist - self.myuri = None - self.mydest = dest - self.destpath = self.mydest+"/"+self.filename - self.md5sum = md5sum - self.result = None - Thread.__init__(self) - - def fetch(self): - #print "Started",self.filename - sys.stderr.write(".") - sys.stderr.flush() - portage.spawn("wget -q -P "+str(self.mydest)+" "+self.myuri, free=1) - - def finished(self): - if os.path.exists(self.destpath) and self.md5sum: - ok,reason = portage_checksum.verify_all(self.destpath, md5sum) - if not ok: - portage_util.writemsg("Failed verification:"+reason[0]+" (got "+reason[1]+", expected "+reason[2]+"\n") - return 1 - return 0 - - def delete(self): - if os.path.exists(self.destpath): - #print "Unlink:",self.destpath - os.unlink(self.destpath) - - def run(self): - if not self.finished(): - self.delete() - - while not self.finished(): - if self.myurilist: - self.myuri = self.myurilist.pop(0)+"/"+self.filename - self.fetch() - else: - self.delete() - self.result = 0 - #print "Failed:",self.filename - return 1 - - #print "Finished:",self.filename - self.result = 1 - return 0 - - -uri_list = {} -fetchers = [] -fetcher_sem = BoundedSemaphore(value=maxsems) -failures = 0 -successes = 0 - -def clean_fetchers(): - global fetcher_sem,fetchers,uri_list,failures,successes,maxsems - while len(fetchers) == maxsems: - for x in fetchers: - if not x.isAlive(): - failures += (x.result == 0) - successes += (x.result == 1) - if x.filename in uri_list.keys(): - del uri_list[x.filename] - del fetchers[fetchers.index(x)] - fetcher_sem.release() - if len(fetchers) == maxsems: - sleep(1) - - -def start_fetcher(fname, urilist, dest, md5sum): - global fetcher_sem,fetchers,uri_list,failures,successes - fetcher_sem.acquire() - fetchers.append(fetcher(fname, urilist, dest, md5sum)) - fetchers[-1].start() - - -tpm = portage.thirdpartymirrors -destdir = portage.settings["DISTDIR"][:] - -hugelist = [] -for mycp in portage.db["/"]["porttree"].dbapi.cp_all(): - hugelist += portage.db["/"]["porttree"].dbapi.cp_list(mycp) -shuffle(hugelist) - -mycount = -1 -for mycpv in hugelist: - pv = string.split(mycpv, "/")[-1] - - clean_fetchers() - - mycount += 1 - if ((mycount % 20) == 0): - sys.stdout.write("\nCompleted: %s\n" % mycount) - sys.stdout.flush() - newuri = portage.db["/"]["porttree"].dbapi.aux_get(mycpv,["SRC_URI"])[0] - newuri = string.split(newuri) - - digestpath = portage.db["/"]["porttree"].dbapi.findname(mycpv) - digestpath = os.path.dirname(digestpath)+"/files/digest-"+pv - md5sums = portage.digestParseFile(digestpath) - - for x in newuri: - clean_fetchers() - if not x: - continue - if (x in [")","(",":","||"]) or (x[-1] == "?"): - # ignore it. :) - continue - x = cstrip(x,"()|?") - if not x: - continue - mybn = os.path.basename(x) - mydn = os.path.dirname(x) - if mybn not in uri_list.keys(): - if (len(mybn) > len("mirror://")) and (mybn[:len("mirror://")] == "mirror://"): - mysite = string.split(x[len("mirror://"):], "/")[0] - shuffle(tpm[mysite]) - uri_list[mybn] = tpm[mysite][:] - else: - uri_list[mybn] = [os.path.dirname(x)] - clean_fetchers() - if (not md5sums) or (mybn not in md5sums.keys()): - start_fetcher(mybn, uri_list[mybn], destdir, None) - else: - start_fetcher(mybn, uri_list[mybn], destdir, md5sums[mybn]) - else: - break - -sys.stderr.write("\n\nWaiting last set\n") -sys.stderr.flush() -while fetchers: - if fetchers[0].isAlive(): - fetchers[0].join() - clean_fetchers() - -print -print -print "Successes:",successes -print "Failures: ",failures |