Move digestgen to portage.util.digestgen.

svn path=/main/trunk/; revision=15454

5 files changed, 214 insertions, 185 deletions

diff --git a/bin/repoman b/bin/repoman
index c1f1481d0..9f2d88c59 100755
--- a/bin/repoman
+++ b/bin/repoman
@@ -77,6 +77,7 @@ from portage.output import bold, create_color_func, darkgreen, \
 	green, nocolor, red, turquoise, yellow
 from portage.output import ConsoleStyleFile, StyleWriter
 from portage.util import cmp_sort_key, writemsg_level
+from portage.util.digestgen import digestgen
 
 if sys.hexversion >= 0x3000000:
 	basestring = str
@@ -1011,7 +1012,7 @@ for x in scanlist:
 				portage._doebuild_manifest_exempt_depend -= 1
 
 		repoman_settings["O"] = checkdir
-		if not portage.digestgen(mysettings=repoman_settings, myportdb=portdb):
+		if not digestgen(mysettings=repoman_settings, myportdb=portdb):
 			print("Unable to generate manifest.")
 			dofail = 1
 		if options.mode == "manifest":
@@ -2335,7 +2336,7 @@ else:
 		mydone=[]
 		if repolevel==3:   # In a package dir
 			repoman_settings["O"] = startdir
-			portage.digestgen(mysettings=repoman_settings, myportdb=portdb)
+			digestgen(mysettings=repoman_settings, myportdb=portdb)
 		elif repolevel==2: # In a category dir
 			for x in myfiles:
 				xs=x.split("/")
@@ -2349,7 +2350,7 @@ else:
 				repoman_settings["O"] = os.path.join(startdir, xs[0])
 				if not os.path.isdir(repoman_settings["O"]):
 					continue
-				portage.digestgen(mysettings=repoman_settings, myportdb=portdb)
+				digestgen(mysettings=repoman_settings, myportdb=portdb)
 		elif repolevel==1: # repo-cvsroot
 			print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n")
 			for x in myfiles:
@@ -2364,7 +2365,7 @@ else:
 				repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1])
 				if not os.path.isdir(repoman_settings["O"]):
 					continue
-				portage.digestgen(mysettings=repoman_settings, myportdb=portdb)
+				digestgen(mysettings=repoman_settings, myportdb=portdb)
 		else:
 			print(red("I'm confused... I don't know where I am!"))
 			sys.exit(1)
diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py
index e508e233f..071b8e8d7 100644
--- a/pym/_emerge/Scheduler.py
+++ b/pym/_emerge/Scheduler.py
@@ -24,6 +24,7 @@ bad = create_color_func("BAD")
 from portage.sets import SETPREFIX
 from portage.sets.base import InternalPackageSet
 from portage.util import writemsg, writemsg_level
+from portage.util.digestgen import digestgen
 
 from _emerge.BinpkgPrefetcher import BinpkgPrefetcher
 from _emerge.Blocker import Blocker
@@ -591,7 +592,7 @@ class Scheduler(PollScheduler):
 			if ebuild_path is None:
 				raise AssertionError("ebuild not found for '%s'" % x.cpv)
 			pkgsettings['O'] = os.path.dirname(ebuild_path)
-			if not portage.digestgen(mysettings=pkgsettings, myportdb=portdb):
+			if not digestgen(mysettings=pkgsettings, myportdb=portdb):
 				writemsg_level(
 					"!!! Unable to generate manifest for '%s'.\n" \
 					% x.cpv, level=logging.ERROR, noiselevel=-1)
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index c459c1b31..46988f3ad 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -119,6 +119,7 @@ try:
 			'pickle_read,pickle_write,stack_dictlist,stack_dicts,' + \
 			'stack_lists,unique_array,varexpand,writedict,writemsg,' + \
 			'writemsg_stdout,write_atomic',
+		'portage.util.digestgen:digestgen',
 		'portage.util.digraph:digraph',
 		'portage.util.env_update:env_update',
 		'portage.util.ExtractKernelVersion:ExtractKernelVersion',
@@ -536,185 +537,6 @@ def abssymlink(symlink):
 
 _doebuild_manifest_exempt_depend = 0
 
-def digestgen(myarchives=None, mysettings=None,
-	overwrite=None, manifestonly=None, myportdb=None):
-	"""
-	Generates a digest file if missing. Fetches files if necessary.
-	NOTE: myarchives and mysettings used to be positional arguments,
-		so their order must be preserved for backward compatibility.
-	@param mysettings: the ebuild config (mysettings["O"] must correspond
-		to the ebuild's parent directory)
-	@type mysettings: config
-	@param myportdb: a portdbapi instance
-	@type myportdb: portdbapi
-	@rtype: int
-	@returns: 1 on success and 0 on failure
-	"""
-	if mysettings is None:
-		raise TypeError("portage.digestgen(): missing" + \
-			" required 'mysettings' parameter")
-	if myportdb is None:
-		warnings.warn("portage.digestgen() called without 'myportdb' parameter",
-			DeprecationWarning, stacklevel=2)
-		global portdb
-		myportdb = portdb
-	if overwrite is not None:
-		warnings.warn("portage.digestgen() called with " + \
-			"deprecated 'overwrite' parameter",
-			DeprecationWarning, stacklevel=2)
-	if manifestonly is not None:
-		warnings.warn("portage.digestgen() called with " + \
-			"deprecated 'manifestonly' parameter",
-			DeprecationWarning, stacklevel=2)
-	global _doebuild_manifest_exempt_depend
-	try:
-		_doebuild_manifest_exempt_depend += 1
-		distfiles_map = {}
-		fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb)
-		for cpv in fetchlist_dict:
-			try:
-				for myfile in fetchlist_dict[cpv]:
-					distfiles_map.setdefault(myfile, []).append(cpv)
-			except portage.exception.InvalidDependString as e:
-				writemsg("!!! %s\n" % str(e), noiselevel=-1)
-				del e
-				return 0
-		mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
-		manifest1_compat = False
-		mf = Manifest(mysettings["O"], mysettings["DISTDIR"],
-			fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat)
-		# Don't require all hashes since that can trigger excessive
-		# fetches when sufficient digests already exist.  To ease transition
-		# while Manifest 1 is being removed, only require hashes that will
-		# exist before and after the transition.
-		required_hash_types = set()
-		required_hash_types.add("size")
-		required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH)
-		dist_hashes = mf.fhashdict.get("DIST", {})
-
-		# To avoid accidental regeneration of digests with the incorrect
-		# files (such as partially downloaded files), trigger the fetch
-		# code if the file exists and it's size doesn't match the current
-		# manifest entry. If there really is a legitimate reason for the
-		# digest to change, `ebuild --force digest` can be used to avoid
-		# triggering this code (or else the old digests can be manually
-		# removed from the Manifest).
-		missing_files = []
-		for myfile in distfiles_map:
-			myhashes = dist_hashes.get(myfile)
-			if not myhashes:
-				try:
-					st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
-				except OSError:
-					st = None
-				if st is None or st.st_size == 0:
-					missing_files.append(myfile)
-				continue
-			size = myhashes.get("size")
-
-			try:
-				st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
-			except OSError as e:
-				if e.errno != errno.ENOENT:
-					raise
-				del e
-				if size == 0:
-					missing_files.append(myfile)
-					continue
-				if required_hash_types.difference(myhashes):
-					missing_files.append(myfile)
-					continue
-			else:
-				if st.st_size == 0 or size is not None and size != st.st_size:
-					missing_files.append(myfile)
-					continue
-
-		if missing_files:
-				mytree = os.path.realpath(os.path.dirname(
-					os.path.dirname(mysettings["O"])))
-				fetch_settings = config(clone=mysettings)
-				debug = mysettings.get("PORTAGE_DEBUG") == "1"
-				for myfile in missing_files:
-					uris = set()
-					for cpv in distfiles_map[myfile]:
-						myebuild = os.path.join(mysettings["O"],
-							catsplit(cpv)[1] + ".ebuild")
-						# for RESTRICT=fetch, mirror, etc...
-						doebuild_environment(myebuild, "fetch",
-							mysettings["ROOT"], fetch_settings,
-							debug, 1, myportdb)
-						uris.update(myportdb.getFetchMap(
-							cpv, mytree=mytree)[myfile])
-
-					fetch_settings["A"] = myfile # for use by pkg_nofetch()
-
-					try:
-						st = os.stat(os.path.join(
-							mysettings["DISTDIR"],myfile))
-					except OSError:
-						st = None
-
-					if not fetch({myfile : uris}, fetch_settings):
-						writemsg(_("!!! Fetch failed for %s, can't update "
-							"Manifest\n") % myfile, noiselevel=-1)
-						if myfile in dist_hashes and \
-							st is not None and st.st_size > 0:
-							# stat result is obtained before calling fetch(),
-							# since fetch may rename the existing file if the
-							# digest does not match.
-							writemsg(_("!!! If you would like to "
-								"forcefully replace the existing "
-								"Manifest entry\n!!! for %s, use "
-								"the following command:\n") % myfile + \
-								"!!!    " + colorize("INFORM",
-								"ebuild --force %s manifest" % \
-								os.path.basename(myebuild)) + "\n",
-								noiselevel=-1)
-						return 0
-		writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"])
-		try:
-			mf.create(assumeDistHashesSometimes=True,
-				assumeDistHashesAlways=(
-				"assume-digests" in mysettings.features))
-		except portage.exception.FileNotFound as e:
-			writemsg(_("!!! File %s doesn't exist, can't update "
-				"Manifest\n") % e, noiselevel=-1)
-			return 0
-		except portage.exception.PortagePackageException as e:
-			writemsg(("!!! %s\n") % (e,), noiselevel=-1)
-			return 0
-		try:
-			mf.write(sign=False)
-		except portage.exception.PermissionDenied as e:
-			writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1)
-			return 0
-		if "assume-digests" not in mysettings.features:
-			distlist = list(mf.fhashdict.get("DIST", {}))
-			distlist.sort()
-			auto_assumed = []
-			for filename in distlist:
-				if not os.path.exists(
-					os.path.join(mysettings["DISTDIR"], filename)):
-					auto_assumed.append(filename)
-			if auto_assumed:
-				mytree = os.path.realpath(
-					os.path.dirname(os.path.dirname(mysettings["O"])))
-				cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:])
-				pkgs = myportdb.cp_list(cp, mytree=mytree)
-				pkgs.sort()
-				writemsg_stdout("  digest.assumed" + portage.output.colorize("WARN",
-					str(len(auto_assumed)).rjust(18)) + "\n")
-				for pkg_key in pkgs:
-					fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree)
-					pv = pkg_key.split("/")[1]
-					for filename in auto_assumed:
-						if filename in fetchlist:
-							writemsg_stdout(
-								"   %s::%s\n" % (pv, filename))
-		return 1
-	finally:
-		_doebuild_manifest_exempt_depend -= 1
-
 def digestParseFile(myfilename, mysettings=None):
 	"""(filename) -- Parses a given file for entries matching:
 	<checksumkey> <checksum_hex_string> <filename> <filesize>
diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
index edeb5aaa6..6a00a1e61 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -21,11 +21,12 @@ import time
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.package.ebuild.config:check_config_instance',
+	'portage.util.digestgen:digestgen',
 	'portage.util.ExtractKernelVersion:ExtractKernelVersion'
 )
 
 from portage import auxdbkeys, bsd_chflags, dep_check, digestcheck, \
-	digestgen, eapi_is_supported, merge, os, selinux, StringIO, \
+	eapi_is_supported, merge, os, selinux, StringIO, \
 	unmerge, _encodings, _parse_eapi_ebuild_head, _os_merge, \
 	_shell_quote, _split_ebuild_name_glep55, _unicode_decode, _unicode_encode
 from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_BINARY, \
diff --git a/pym/portage/util/digestgen.py b/pym/portage/util/digestgen.py
new file mode 100644
index 000000000..2dd0caef4
--- /dev/null
+++ b/pym/portage/util/digestgen.py
@@ -0,0 +1,204 @@
+# Copyright 2010 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id$
+
+__all__ = ['digestgen']
+
+import errno
+import warnings
+
+import portage
+portage.proxy.lazyimport.lazyimport(globals(),
+	'portage.package.ebuild.doebuild:doebuild_environment',
+)
+
+from portage import os
+from portage.const import MANIFEST2_REQUIRED_HASH
+from portage.dbapi.porttree import FetchlistDict
+from portage.exception import InvalidDependString, FileNotFound, \
+	PermissionDenied, PortagePackageException
+from portage.localization import _
+from portage.manifest import Manifest
+from portage.output import colorize
+from portage.package.ebuild.config import config
+from portage.package.ebuild.fetch import fetch
+from portage.util import writemsg, writemsg_stdout
+from portage.versions import catsplit
+
+def digestgen(myarchives=None, mysettings=None,
+	overwrite=None, manifestonly=None, myportdb=None):
+	"""
+	Generates a digest file if missing. Fetches files if necessary.
+	NOTE: myarchives and mysettings used to be positional arguments,
+		so their order must be preserved for backward compatibility.
+	@param mysettings: the ebuild config (mysettings["O"] must correspond
+		to the ebuild's parent directory)
+	@type mysettings: config
+	@param myportdb: a portdbapi instance
+	@type myportdb: portdbapi
+	@rtype: int
+	@returns: 1 on success and 0 on failure
+	"""
+	if mysettings is None:
+		raise TypeError("portage.digestgen(): missing" + \
+			" required 'mysettings' parameter")
+	if myportdb is None:
+		warnings.warn("portage.digestgen() called without 'myportdb' parameter",
+			DeprecationWarning, stacklevel=2)
+		myportdb = portage.portdb
+	if overwrite is not None:
+		warnings.warn("portage.digestgen() called with " + \
+			"deprecated 'overwrite' parameter",
+			DeprecationWarning, stacklevel=2)
+	if manifestonly is not None:
+		warnings.warn("portage.digestgen() called with " + \
+			"deprecated 'manifestonly' parameter",
+			DeprecationWarning, stacklevel=2)
+
+	try:
+		portage._doebuild_manifest_exempt_depend += 1
+		distfiles_map = {}
+		fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb)
+		for cpv in fetchlist_dict:
+			try:
+				for myfile in fetchlist_dict[cpv]:
+					distfiles_map.setdefault(myfile, []).append(cpv)
+			except InvalidDependString as e:
+				writemsg("!!! %s\n" % str(e), noiselevel=-1)
+				del e
+				return 0
+		mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
+		manifest1_compat = False
+		mf = Manifest(mysettings["O"], mysettings["DISTDIR"],
+			fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat)
+		# Don't require all hashes since that can trigger excessive
+		# fetches when sufficient digests already exist.  To ease transition
+		# while Manifest 1 is being removed, only require hashes that will
+		# exist before and after the transition.
+		required_hash_types = set()
+		required_hash_types.add("size")
+		required_hash_types.add(MANIFEST2_REQUIRED_HASH)
+		dist_hashes = mf.fhashdict.get("DIST", {})
+
+		# To avoid accidental regeneration of digests with the incorrect
+		# files (such as partially downloaded files), trigger the fetch
+		# code if the file exists and it's size doesn't match the current
+		# manifest entry. If there really is a legitimate reason for the
+		# digest to change, `ebuild --force digest` can be used to avoid
+		# triggering this code (or else the old digests can be manually
+		# removed from the Manifest).
+		missing_files = []
+		for myfile in distfiles_map:
+			myhashes = dist_hashes.get(myfile)
+			if not myhashes:
+				try:
+					st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
+				except OSError:
+					st = None
+				if st is None or st.st_size == 0:
+					missing_files.append(myfile)
+				continue
+			size = myhashes.get("size")
+
+			try:
+				st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
+			except OSError as e:
+				if e.errno != errno.ENOENT:
+					raise
+				del e
+				if size == 0:
+					missing_files.append(myfile)
+					continue
+				if required_hash_types.difference(myhashes):
+					missing_files.append(myfile)
+					continue
+			else:
+				if st.st_size == 0 or size is not None and size != st.st_size:
+					missing_files.append(myfile)
+					continue
+
+		if missing_files:
+				mytree = os.path.realpath(os.path.dirname(
+					os.path.dirname(mysettings["O"])))
+				fetch_settings = config(clone=mysettings)
+				debug = mysettings.get("PORTAGE_DEBUG") == "1"
+				for myfile in missing_files:
+					uris = set()
+					for cpv in distfiles_map[myfile]:
+						myebuild = os.path.join(mysettings["O"],
+							catsplit(cpv)[1] + ".ebuild")
+						# for RESTRICT=fetch, mirror, etc...
+						doebuild_environment(myebuild, "fetch",
+							mysettings["ROOT"], fetch_settings,
+							debug, 1, myportdb)
+						uris.update(myportdb.getFetchMap(
+							cpv, mytree=mytree)[myfile])
+
+					fetch_settings["A"] = myfile # for use by pkg_nofetch()
+
+					try:
+						st = os.stat(os.path.join(
+							mysettings["DISTDIR"],myfile))
+					except OSError:
+						st = None
+
+					if not fetch({myfile : uris}, fetch_settings):
+						writemsg(_("!!! Fetch failed for %s, can't update "
+							"Manifest\n") % myfile, noiselevel=-1)
+						if myfile in dist_hashes and \
+							st is not None and st.st_size > 0:
+							# stat result is obtained before calling fetch(),
+							# since fetch may rename the existing file if the
+							# digest does not match.
+							writemsg(_("!!! If you would like to "
+								"forcefully replace the existing "
+								"Manifest entry\n!!! for %s, use "
+								"the following command:\n") % myfile + \
+								"!!!    " + colorize("INFORM",
+								"ebuild --force %s manifest" % \
+								os.path.basename(myebuild)) + "\n",
+								noiselevel=-1)
+						return 0
+		writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"])
+		try:
+			mf.create(assumeDistHashesSometimes=True,
+				assumeDistHashesAlways=(
+				"assume-digests" in mysettings.features))
+		except FileNotFound as e:
+			writemsg(_("!!! File %s doesn't exist, can't update "
+				"Manifest\n") % e, noiselevel=-1)
+			return 0
+		except PortagePackageException as e:
+			writemsg(("!!! %s\n") % (e,), noiselevel=-1)
+			return 0
+		try:
+			mf.write(sign=False)
+		except PermissionDenied as e:
+			writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1)
+			return 0
+		if "assume-digests" not in mysettings.features:
+			distlist = list(mf.fhashdict.get("DIST", {}))
+			distlist.sort()
+			auto_assumed = []
+			for filename in distlist:
+				if not os.path.exists(
+					os.path.join(mysettings["DISTDIR"], filename)):
+					auto_assumed.append(filename)
+			if auto_assumed:
+				mytree = os.path.realpath(
+					os.path.dirname(os.path.dirname(mysettings["O"])))
+				cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:])
+				pkgs = myportdb.cp_list(cp, mytree=mytree)
+				pkgs.sort()
+				writemsg_stdout("  digest.assumed" + colorize("WARN",
+					str(len(auto_assumed)).rjust(18)) + "\n")
+				for pkg_key in pkgs:
+					fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree)
+					pv = pkg_key.split("/")[1]
+					for filename in auto_assumed:
+						if filename in fetchlist:
+							writemsg_stdout(
+								"   %s::%s\n" % (pv, filename))
+		return 1
+	finally:
+		portage._doebuild_manifest_exempt_depend -= 1