Move random functions out of repoman and into utilties.py

svn path=/main/trunk/; revision=9221

2 files changed, 208 insertions, 132 deletions

diff --git a/bin/repoman b/bin/repoman
index 6e0b82b7d..58b8ba67e 100755
--- a/bin/repoman
+++ b/bin/repoman
@@ -8,7 +8,6 @@
 # that last one is tricky because multiple profiles need to be checked.
 
 import codecs
-import commands
 import errno
 import formatter
 import logging
@@ -52,10 +51,12 @@ del os.environ["PORTAGE_LEGACY_GLOBALS"]
 
 try:
 	from repoman.checks import run_checks
+	from repoman import utilities
 except ImportError:
 	from os import path as osp
 	sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), 'pym'))
 	from repoman.checks import run_checks
+	from repoman import utilities
 
 import portage.checksum
 import portage.const
@@ -63,8 +64,8 @@ import portage.dep
 portage.dep._dep_check_strict = True
 import portage.exception
 from portage import cvstree, normalize_path
-from portage.manifest import Manifest
 from portage.exception import ParseError
+from portage.manifest import Manifest
 from portage.process import find_binary, spawn
 from portage.util import initialize_logger
 from portage.output import bold, create_color_func, darkgreen, \
@@ -318,7 +319,6 @@ qawarnings=[
 "ebuild.allmasked",
 "ebuild.autotools",
 "ebuild.nesteddie",
-"desktop.invalid",
 "digest.assumed",
 "digest.missing",
 "digestentry.unused",
@@ -353,7 +353,7 @@ for x in missingvars:
 		qawarnings.append(x)
 
 valid_restrict = frozenset(["binchecks", "bindist",
-	"fetch", "installsources", "mirror",
+	"fetch", "installsources", "mirror", 
 	"primaryuri", "strip", "test", "userpriv"])
 
 # file.executable
@@ -547,74 +547,11 @@ if options.mode == 'commit' and not options.pretend and not isCvs:
 	logging.info("Not in a CVS repository; enabling pretend mode.")
 	options.pretend = True
 
-def have_profile_dir(path, maxdepth=3):
-	while path != "/" and maxdepth:
-		if os.path.exists(path + "/profiles/package.mask"):
-			return normalize_path(path)
-		path = normalize_path(path + "/..")
-		maxdepth -= 1
-
-portdir=None
-portdir_overlay=None
-mydir=os.getcwd()
-if "PWD" in os.environ and os.environ["PWD"] != mydir and \
-	os.path.realpath(os.environ["PWD"]) == mydir:
-	# getcwd() returns the canonical path but that makes it hard for repoman to
-	# orient itself if the user has symlinks in their portage tree structure.
-	# We use os.environ["PWD"], if available, to get the non-canonical path of
-	# the current working directory (from the shell).
-	mydir = os.environ["PWD"]
-mydir = normalize_path(mydir)
-path_ids = set()
-p = mydir
-s = None
-while True:
-	s = os.stat(p)
-	path_ids.add((s.st_dev, s.st_ino))
-	if p == "/":
-		break
-	p = os.path.dirname(p)
-if mydir[-1] != "/":
-	mydir += "/"
-
-for overlay in repoman_settings["PORTDIR_OVERLAY"].split():
-	overlay = os.path.realpath(overlay)
-	try:
-		s = os.stat(overlay)
-	except OSError:
-		continue
-	overlay_id = (s.st_dev, s.st_ino)
-	if overlay[-1] != "/":
-		overlay += "/"
-	if overlay_id in path_ids:
-		portdir_overlay = overlay
-		subdir = mydir[len(overlay):]
-		if subdir and subdir[-1] != "/":
-			subdir += "/"
-		if have_profile_dir(mydir, subdir.count("/")):
-			portdir = portdir_overlay
-		break
-
-del p, s, path_ids
-
-if not portdir_overlay:
-	if (repoman_settings["PORTDIR"] + os.path.sep).startswith(mydir):
-		portdir_overlay = repoman_settings["PORTDIR"]
-	else:
-		portdir_overlay = have_profile_dir(mydir)
-	portdir = portdir_overlay
-
-if not portdir_overlay:
-	sys.stderr.write("Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY from" + \
-		" the current\nworking directory.\n")
+try:
+	portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
+except ValueError:
 	sys.exit(1)
 
-if not portdir:
-	portdir = repoman_settings["PORTDIR"]
-
-portdir = normalize_path(portdir)
-portdir_overlay = normalize_path(portdir_overlay)
-
 os.environ["PORTDIR"] = portdir
 if portdir_overlay != portdir:
 	os.environ["PORTDIR_OVERLAY"] = portdir_overlay
@@ -635,16 +572,15 @@ portdb.mysettings = repoman_settings
 # dep_zapdeps looks at the vardbapi, but it shouldn't for repoman.
 del trees["/"]["vartree"]
 
-if not myreporoot:
-	myreporoot = os.path.basename(portdir_overlay)
-	myreporoot += mydir[len(portdir_overlay):-1]
+myreporoot = os.path.basename(portdir_overlay)
+myreporoot += mydir[len(portdir_overlay):]
 
-reposplit=myreporoot.split("/")
-repolevel=len(reposplit)
+reposplit = myreporoot.split(os.path.sep)
+repolevel = len(reposplit)
 
 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
-# this check ensure that repoman knows where it is, and the manifest recommit is at least possible.
+# this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
 if options.mode == 'commit' and repolevel not in [1,2,3]:
 	print red("***")+" Commit attempts *must* be from within a cvs co, category, or package directory."
 	print red("***")+" Attempting to commit from a packages files directory will be blocked for instance."
@@ -654,40 +590,14 @@ if options.mode == 'commit' and repolevel not in [1,2,3]:
 
 startdir = normalize_path(mydir)
 repodir = startdir
-for x in range(0,repolevel-1):
+for x in range(0, repolevel - 1):
 	repodir = os.path.dirname(repodir)
 
-def caterror(mycat):
-	warn(mycat + " is not an official category. " + \
-		"Skipping QA checks in this directory.\n" + \
-		"Please ensure that you add " + catdir + \
-		" to " + repodir + "/profiles/categories\nif it is a new category.")
-
-def parse_use_local_desc(mylines, usedict=None):
-	"""returns a dict of the form {cpv:set(flags)}"""
-	if usedict is None:
-		usedict = {}
-	lineno = 0
-	for l in mylines:
-		lineno += 1
-		if not l or l.startswith("#"):
-			continue
-		mysplit = l.split(None, 1)
-		if not mysplit:
-			continue
-		mysplit = mysplit[0].split(":")
-		if len(mysplit) != 2:
-			raise ParseError("line %d: Malformed input: '%s'" % \
-				(lineno, l.rstrip("\n")))
-		usedict.setdefault(mysplit[0], set())
-		usedict[mysplit[0]].add(mysplit[1])
-	return usedict
-
 # retreive local USE list
 luselist={}
 try:
 	f = open(os.path.join(portdir, "profiles", "use.local.desc"))
-	parse_use_local_desc(f, luselist)
+	utilities.parse_use_local_desc(f, luselist)
 	f.close()
 except (IOError, OSError, ParseError), e:
 	logging.exception("Couldn't read from use.local.desc", e)
@@ -741,33 +651,9 @@ if portdir_overlay != portdir:
 	manifest1_compat = not os.path.exists(
 		os.path.join(portdir_overlay, "manifest1_obsolete"))
 
-scanlist=[]
-if repolevel==2:
-	#we are inside a category directory
-	catdir=reposplit[-1]
-	if catdir not in repoman_settings.categories:
-		caterror(catdir)
-	mydirlist=os.listdir(startdir)
-	for x in mydirlist:
-		if x == "CVS" or x.startswith("."):
-			continue
-		if os.path.isdir(startdir+"/"+x):
-			scanlist.append(catdir+"/"+x)
-elif repolevel==1:
-	for x in repoman_settings.categories:
-		if not os.path.isdir(startdir+"/"+x):
-			continue
-		for y in os.listdir(startdir+"/"+x):
-			if y == "CVS" or y.startswith("."):
-				continue
-			if os.path.isdir(startdir+"/"+x+"/"+y):
-				scanlist.append(x+"/"+y)
-elif repolevel==3:
-	catdir = reposplit[-2]
-	if catdir not in repoman_settings.categories:
-		caterror(catdir)
-	scanlist.append(catdir+"/"+reposplit[-1])
+scanlist = utilities.FindPackagesToScan(repoman_settings, startdir, reposplit)
 scanlist.sort()
+logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
 
 profiles={}
 descfile=portdir+"/profiles/profiles.desc"
@@ -1089,7 +975,7 @@ for x in scanlist:
 				full_path = os.path.join(repodir, relative_path)
 				if stat.S_IMODE(os.stat(full_path).st_mode) & 0111:
 					stats["file.executable"] += 1
-					fails["file.executable"].append(relative_path)
+					fails["file.executable"].append(x+"/files/"+y)
 				
 				mykey = catdir + "/" + y[7:]
 				if y[7:] not in ebuildlist:
@@ -1225,7 +1111,7 @@ for x in scanlist:
 		full_path = os.path.join(repodir, relative_path)
 		if stat.S_IMODE(os.stat(full_path).st_mode) & 0111:
 			stats["file.executable"] += 1
-			fails["file.executable"].append(relative_path)
+			fails["file.executable"].append(x+"/"+y+".ebuild")
 		if isCvs and y not in eadded:
 			#ebuild not added to cvs
 			stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
@@ -1507,6 +1393,8 @@ for x in scanlist:
 				for mybad in mybadrestrict:
 					fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
 		# Syntax Checks
+		relative_path = os.path.join(x, y + ".ebuild")
+		full_path = os.path.join(repodir, relative_path)
 		f = open(full_path, 'rb')
 		try:
 			for check_name, e in run_checks(f, os.stat(full_path).st_mtime):
diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py
new file mode 100644
index 000000000..26863ecf1
--- /dev/null
+++ b/pym/repoman/utilities.py
@@ -0,0 +1,188 @@
+# repoman: Utilities
+# Copyright 2007 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id$
+
+import logging
+import os
+
+from portage import util
+from portage import exception
+
+normalize_path = util.normalize_path
+util.initialize_logger()
+
+def have_profile_dir(path, maxdepth=3):
+	""" Try to figure out if 'path' has a /profiles dir in it by checking for a package.mask file
+	"""
+	while path != "/" and maxdepth:
+		if os.path.exists(path + "/profiles/package.mask"):
+			return normalize_path(path)
+		path = normalize_path(path + "/..")
+		maxdepth -= 1
+
+def parse_use_local_desc(mylines, usedict=None):
+	"""
+	Records are of the form PACKAGE:FLAG - DESC
+	returns a dict of the form {cpv:set(flags)}"""
+	if usedict is None:
+		usedict = {}
+	for line_num, l in enumerate(mylines):
+		if not l or l.startswith('#'):
+			continue
+		pkg_flag = l.split(None, 1) # None implies splitting on whitespace
+		if not pkg_flag:
+			continue
+		try:
+			pkg, flag = pkg_flag[0].split(":")
+		except ValueError:
+			raise exception,ParseError("line %d: Malformed input: '%s'" % \
+				(linenum + 1, l.rstrip("\n")))
+		usedict.setdefault(pkg, set())
+		usedict[pkg].add(flag)
+	return usedict
+
+def FindPackagesToScan(settings, startdir, reposplit):
+	""" Try to find packages that need to be scanned
+	
+	Args:
+		settings - portage.config instance, preferably repoman_settings
+		startdir - directory that repoman was run in
+		reposplit - root of the repository
+	Returns:
+		A list of directories to scan
+	"""
+	
+	def AddPackagesInDir(path):
+		""" Given a list of dirs, add any packages in it """
+		ret = []
+		pkgdirs = os.listdir(path)
+		for d in pkgdirs:
+			if d == 'CVS' or d.startswith('.'):
+				continue
+			p = os.path.join(path, d)
+
+			if os.path.isdir(p):
+				cat_pkg_dir = os.path.join(p.split(os.path.sep)[-2:])
+				logging.debug('adding %s to scanlist' % cat_pkg_dir)
+				ret.append(cat_pkg_dir)
+		return ret
+	
+	scanlist = []
+	repolevel = len(reposplit)
+	if repolevel == 1: # root of the tree, startdir = repodir
+		for cat in settings.categories:
+			path = os.path.join(startdir, cat)
+			if not os.path.isdir(path):
+				continue
+			pkgdirs = os.listdir(path)
+			scanlist.extend(AddPackagesInDir(path))
+	elif repolevel == 2: # category level, startdir = catdir
+		# we only want 1 segment of the directory, is why we use catdir instead of startdir
+		catdir = reposplit[-2]
+		if catdir not in settings.categories:
+			logging.warn('%s is not a valid category according to profiles/categories, ' \
+				'skipping checks in %s' % (catdir, catdir))
+		else:
+			scanlist = AddPackagesInDir(catdir)
+	elif repolevel == 3: # pkgdir level, startdir = pkgdir
+		catdir = reposplit[-2]
+		pkgdir = reposplit[-1]
+		if catdir not in settings.categories:
+			logging.warn('%s is not a valid category according to profiles/categories, ' \
+			'skipping checks in %s' % (catdir, catdir))
+		else:
+			scanlist.append(os.path.join(catdir, pkgdir))
+	return scanlist
+
+def FindPortdir(settings):
+	""" Try to figure out what repo we are in and whether we are in a regular
+	tree or an overlay.
+	
+	Basic logic is:
+	
+	1. Determine what directory we are in (supports symlinks).
+	2. Build a list of directories from / to our current location
+	3. Iterate over PORTDIR_OVERLAY, if we find a match, search for a profiles directory
+		 in the overlay.  If it has one, make it portdir, otherwise make it portdir_overlay.
+	4. If we didn't find an overlay in PORTDIR_OVERLAY, see if we are in PORTDIR; if so, set
+		 portdir_overlay to PORTDIR.  If we aren't in PORTDIR, see if PWD has a profiles dir, if
+		 so, set portdir_overlay and portdir to PWD, else make them False.
+	5. If we haven't found portdir_overlay yet, it means the user is doing something odd, report
+		 an error.
+	6. If we haven't found a portdir yet, set portdir to PORTDIR.
+	
+	Args:
+		settings - portage.config instance, preferably repoman_settings
+	Returns:
+		tuple(portdir, portdir_overlay, location)
+	"""
+
+	portdir = None
+	portdir_overlay = None
+	location = os.getcwd()
+	pwd = os.environ.get('PWD', '')
+	if pwd != location and os.path.realpath(pwd) == location:
+		# getcwd() returns the canonical path but that makes it hard for repoman to
+		# orient itself if the user has symlinks in their portage tree structure.
+		# We use os.environ["PWD"], if available, to get the non-canonical path of
+		# the current working directory (from the shell).
+		location = pwd
+
+	location = normalize_path(location)
+
+	path_ids = set()
+	p = location
+	s = None
+	while True:
+		s = os.stat(p)
+		path_ids.add((s.st_dev, s.st_ino))
+		if p == "/":
+			break
+		p = os.path.dirname(p)
+	if location[-1] != "/":
+		location += "/"
+
+	for overlay in settings["PORTDIR_OVERLAY"].split():
+		overlay = os.path.realpath(overlay)
+		try:
+			s = os.stat(overlay)
+		except OSError:
+			continue
+		overlay_id = (s.st_dev, s.st_ino)
+		if overlay[-1] != "/":
+			overlay += "/"
+		if overlay_id in path_ids:
+			portdir_overlay = overlay
+			subdir = location[len(overlay):]
+			if subdir and subdir[-1] != "/":
+				subdir += "/"
+			if have_profile_dir(location, subdir.count("/")):
+				portdir = portdir_overlay
+			break
+	
+	del p, s, path_ids
+	
+	if not portdir_overlay:
+		if (settings["PORTDIR"] + os.path.sep).startswith(location):
+			portdir_overlay = settings["PORTDIR"]
+		else:
+			portdir_overlay = have_profile_dir(location)
+		portdir = portdir_overlay
+	
+	if not portdir_overlay:
+		msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \
+			' from the current working directory'
+		logging.critical(msg)
+		raise ValueError(msg)
+
+	if not portdir:
+		portdir = settings["PORTDIR"]
+
+	if not portdir_overlay.endswith('/'):
+		portdir_overlay += '/'
+	
+	if not portdir.endswith('/'):
+		portdir += '/'
+
+	return map(normalize_path, (portdir, portdir_overlay, location))