Add set configuration framework in preparation for emerge integration

svn path=/main/trunk/; revision=7952

6 files changed, 216 insertions, 77 deletions

diff --git a/pym/portage/sets/__init__.py b/pym/portage/sets/__init__.py
index e62a9d750..746a7e7a9 100644
--- a/pym/portage/sets/__init__.py
+++ b/pym/portage/sets/__init__.py
@@ -3,7 +3,8 @@
 # $Id$
 
 import os
-from portage import flatten
+from ConfigParser import SafeConfigParser, NoOptionError
+from portage import flatten, load_mod
 from portage.dep import isvalidatom, match_from_list, \
      best_match_to_list, dep_getkey, use_reduce, paren_reduce
 from portage.exception import InvalidAtom
@@ -145,7 +146,6 @@ class EditablePackageSet(PackageSet):
 		# This method must be overwritten in subclasses that should be editable
 		raise NotImplementedError()
 
-
 class InternalPackageSet(EditablePackageSet):
 	def __init__(self, initial_atoms=None):
 		super(InternalPackageSet, self).__init__()
@@ -163,78 +163,99 @@ class InternalPackageSet(EditablePackageSet):
 		pass
 
 
+class SetConfigError(Exception):
+	pass
+
+class SetConfig(SafeConfigParser):
+	def __init__(self, paths, settings, trees):
+		SafeConfigParser.__init__(self)
+		self.read(paths)
+		self.errors = []
+		self.psets = {}
+		self.trees = trees
+		self.settings = settings
+	
+	def _parse(self):
+		for sname in self.sections():
+			# find classname for current section, default to file based sets
+			if not self.has_option(sname, "class"):
+				classname = "portage.sets.files.StaticFileSet"
+			else:
+				classname = self.get(sname, "class")
+			
+			# try to import the specified class
+			try:
+				setclass = load_mod(classname)
+			except (ImportError, AttributeError):
+				self.errors.append("Could not import '%s' for section '%s'" % (classname, sname))
+				continue
+			# prepare option dict for the current section
+			optdict = {}
+			for oname in self.options(sname):
+				optdict[oname] = self.get(sname, oname)
+			
+			# create single or multiple instances of the given class depending on configuration
+			if self.has_option(sname, "multiset") and self.getboolean(sname, "multiset"):
+				if hasattr(setclass, "multiBuilder"):
+					try:
+						self.psets.update(setclass.multiBuilder(optdict, self.settings, self.trees))
+					except SetConfigError, e:
+						self.errors.append("Configuration error in section '%s': %s" % (sname, str(e)))
+						continue
+				else:
+					self.errors.append("Section '%s' is configured as multiset, but '%s' doesn't support that configuration" % (sname, classname))
+					continue
+			else:
+				try:
+					setname = self.get(sname, "name")
+				except NoOptionError:
+					setname = "sets/"+sname
+				if hasattr(setclass, "singleBuilder"):
+					try:
+						self.psets[setname] = setclass.singleBuilder(optdict, self.settings, self.trees)
+					except SetConfigError, e:
+						self.errors.append("Configuration error in section '%s': %s" % (sname, str(e)))
+						continue
+				else:
+					self.errors.append("'%s' does not support individual set creation, section '%s' must be configured as multiset" % (classname, sname))
+					continue
+	
+	def getSets(self):
+		self._parse()
+		return (self.psets, self.errors)
+
+def make_default_config(settings, trees):
+	sc = SetConfig([], settings, trees)
+	sc.add_section("security")
+	sc.set("security", "class", "portage.sets.security.NewAffectedSet")
+	
+	sc.add_section("system")
+	sc.set("system", "class", "portage.sets.profiles.PackagesSystemSet")
+	
+	sc.add_section("world")
+	sc.set("world", "class", "portage.sets.files.WorldSet")
+	
+	sc.add_section("everything")
+	sc.set("everything", "class", "portage.sets.dbapi.EverythingSet")
 
-def make_default_sets(configroot, root, profile_paths, settings=None, 
-		vdbapi=None, portdbapi=None):
-	from portage.sets.files import StaticFileSet, ConfigFileSet
-	from portage.sets.profiles import PackagesSystemSet
-	from portage.sets.security import NewAffectedSet
-	from portage.sets.dbapi import EverythingSet
-	from portage.const import PRIVATE_PATH, USER_CONFIG_PATH
-	
-	rValue = {}
-	worldset = StaticFileSet(os.path.join(root, PRIVATE_PATH, "world"))
-	worldset.description = "Set of packages that were directly installed"
-	rValue["world"] = worldset
-	for suffix in ["mask", "unmask", "keywords", "use"]:
-		myname = "package_"+suffix
-		myset = ConfigFileSet(os.path.join(configroot, USER_CONFIG_PATH.lstrip(os.sep), "package."+suffix))
-		rValue[myname] = myset
-	rValue["system"] = PackagesSystemSet(profile_paths)
-	if settings != None and portdbapi != None:
-		rValue["security"] = NewAffectedSet(settings, vdbapi, portdbapi)
-	else:
-		rValue["security"] = InternalPackageSet()
-	if vdbapi != None:
-		rValue["everything"] = EverythingSet(vdbapi)
-	else:
-		rValue["everything"] = InternalPackageSet()
-
-	return rValue
-
-def make_extra_static_sets(configroot):
-	from portage.sets.files import StaticFileSet
-	from portage.const import PRIVATE_PATH, USER_CONFIG_PATH
-	
-	rValue = {}
-	mydir = os.path.join(configroot, USER_CONFIG_PATH.lstrip(os.sep), "sets")
-	try:
-		mysets = os.listdir(mydir)
-	except (OSError, IOError):
-		return rValue
-	for myname in mysets:
-		if myname in DEFAULT_SETS:
-			continue
-		rValue[myname] = StaticFileSet(os.path.join(mydir, myname))
-	return rValue
-
-def make_category_sets(portdbapi, settings, only_visible=True):
-	from portage.sets.dbapi import CategorySet
-	rValue = {}
-	for c in settings.categories:
-		rValue["category_%s" % c] = CategorySet(c, portdbapi, only_visible=only_visible)
-	return rValue
+	sc.add_section("config")
+	sc.set("config", "class", "portage.sets.files.ConfigFileSet")
+	sc.set("config", "multiset", "true")
+	
+	sc.add_section("categories_installed")
+	sc.set("categories_installed", "class", "portage.sets.dbapi.CategorySet")
+	sc.set("categories_installed", "multiset", "true")
+	sc.set("categories_installed", "repository", "vartree")
+	sc.set("categories_installed", "name_pattern", "installed/$category")
+	
+	return sc
 
 # adhoc test code
 if __name__ == "__main__":
-	import portage, sys, os
-	from portage.sets.dbapi import CategorySet
-	from portage.sets.files import StaticFileSet
-	l = make_default_sets("/", "/", portage.settings.profiles, portage.settings, portage.db["/"]["vartree"].dbapi, portage.db["/"]["porttree"].dbapi)
-	l.update(make_extra_static_sets("/"))
-	if len(sys.argv) > 1:
-		for s in sys.argv[1:]:
-			if s.startswith("category_"):
-				c = s[9:]
-				l["category_%s" % c] = CategorySet(c, portage.db['/']['porttree'].dbapi, only_visible=False)
-			elif os.path.exists(s):
-				l[os.path.basename(s)] = StaticFileSet(s)
-			elif s != "*":
-				print "ERROR: could not create set '%s'" % s
-		if not "*" in sys.argv:
-			for n in l:
-				if n not in sys.argv[1:]:
-					del l[n]
+	import portage
+	sc = make_default_config(portage.settings, portage.db["/"])
+	l, e = sc.getSets()
+	print l, e
 	for x in l:
 		print x+":"
 		print "DESCRIPTION = %s" % l[x].getMetadata("Description")
diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py
index 5c16b1701..1a7a66907 100644
--- a/pym/portage/sets/dbapi.py
+++ b/pym/portage/sets/dbapi.py
@@ -3,7 +3,7 @@
 # $Id$
 
 from portage.versions import catsplit
-from portage.sets import PackageSet
+from portage.sets import PackageSet, SetConfigError
 
 class EverythingSet(PackageSet):
 	_operations = ["merge", "unmerge"]
@@ -23,6 +23,10 @@ class EverythingSet(PackageSet):
 			else:
 				myatoms.append(cp)
 		self._setAtoms(myatoms)
+	
+	def singleBuilder(self, options, settings, trees):
+		return EverythingSet(trees["vartree"].dbapi)
+	singleBuilder = classmethod(singleBuilder)
 
 class CategorySet(PackageSet):
 	_operations = ["merge", "unmerge"]
@@ -46,3 +50,56 @@ class CategorySet(PackageSet):
 					myatoms.append(cp)
 		self._setAtoms(myatoms)
 	
+	def _builderGetRepository(cls, options, repositories):
+		repository = options.get("repository", "porttree")
+		if not repository in repositories:
+			raise SetConfigError("invalid repository class '%s'" % repository)
+		return repository
+	_builderGetRepository = classmethod(_builderGetRepository)
+
+	def _builderGetVisible(cls, options):
+		visible = options.get("only_visible", "true").lower()
+		if visible not in ["1", "0", "yes", "no", "true", "false", "on", "off"]:
+			raise SetConfigError("invalid value for only_visible: %s" % visible)
+		return bool(visible in ["1", "yes", "true", "on"])
+	_builderGetVisible = classmethod(_builderGetVisible)
+		
+	def singleBuilder(cls, options, settings, trees):
+		if not "category" in options:
+			raise SetConfigError("no category given")
+
+		category = options["category"]
+		if not category in categories:
+			raise SetConfigError("invalid category name '%s'" % category)
+
+		repository = cls._builderGetRepository(options, trees.keys())
+		visible = cls._builderGetVisible(options)
+		
+		return CategorySet(category, dbapi=trees[repository].dbapi, only_visible=visible)
+	singleBuilder = classmethod(singleBuilder)
+
+	def multiBuilder(cls, options, settings, trees):
+		rValue = {}
+	
+		if "categories" in options:
+			categories = options["categories"].split()
+			invalid = set(categories).difference(settings.categories)
+			if invalid:
+				raise SetConfigError("invalid categories: %s" % ", ".join(list(invalid)))
+		else:
+			categories = settings.categories
+	
+		repository = cls._builderGetRepository(options, trees.keys())
+		visible = cls._builderGetVisible(options)
+		name_pattern = options.get("name_pattern", "$category/*")
+	
+		if not "$category" in name_pattern and not "${category}" in name_pattern:
+			raise SetConfigError("name_pattern doesn't include $category placeholder")
+	
+		for cat in categories:
+			myset = CategorySet(cat, trees[repository].dbapi, only_visible=visible)
+			myname = name_pattern.replace("$category", cat)
+			myname = myname.replace("${category}", cat)
+			rValue[myname] = myset
+		return rValue
+	multiBuilder = classmethod(multiBuilder)
diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py
index 6379e2641..8a7f39549 100644
--- a/pym/portage/sets/files.py
+++ b/pym/portage/sets/files.py
@@ -6,10 +6,10 @@ import errno
 import os
 
 from portage.util import grabfile, write_atomic, ensure_dirs
-from portage.const import PRIVATE_PATH
+from portage.const import PRIVATE_PATH, USER_CONFIG_PATH
 from portage.locks import lockfile, unlockfile
 from portage import portage_gid
-from portage.sets import PackageSet, EditablePackageSet
+from portage.sets import PackageSet, EditablePackageSet, SetConfigError
 from portage.env.loaders import ItemFileLoader, KeyListFileLoader
 from portage.env.validators import ValidAtomValidator
 
@@ -60,6 +60,25 @@ class StaticFileSet(EditablePackageSet):
 				data = {}
 			self._setAtoms(data.keys())
 			self._mtime = mtime
+		
+	def singleBuilder(self, options, settings, trees):
+		if not "filename" in options:
+			raise SetConfigError("no filename specified")
+		return ConfigFileSet(options[filename])
+	singleBuilder = classmethod(singleBuilder)
+	
+	def multiBuilder(self, options, settings, trees):
+		rValue = {}
+		directory = options.get("directory", os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH.lstrip(os.sep), "sets"))
+		name_pattern = options.get("name_pattern", "sets/$name")
+		if not "$name" in name_pattern and not "${name}" in name_pattern:
+			raise SetConfigError("name_pattern doesn't include $name placeholder")
+		for filename in os.listdir(directory):
+			myname = name_pattern.replace("$name", filename)
+			myname = myname.replace("${name}", filename)
+			rValue[myname] = StaticFileSet(os.path.join(directory, filename))
+		return rValue
+	multiBuilder = classmethod(multiBuilder)
 	
 class ConfigFileSet(PackageSet):
 	def __init__(self, filename):
@@ -71,12 +90,31 @@ class ConfigFileSet(PackageSet):
 	def load(self):
 		data, errors = self.loader.load()
 		self._setAtoms(data.keys())
+	
+	def singleBuilder(self, options, settings, trees):
+		if not "filename" in options:
+			raise SetConfigError("no filename specified")
+		return ConfigFileSet(options[filename])
+	singleBuilder = classmethod(singleBuilder)
+	
+	def multiBuilder(self, options, settings, trees):
+		rValue = {}
+		directory = options.get("directory", os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH.lstrip(os.sep)))
+		name_pattern = options.get("name_pattern", "sets/package_$suffix")
+		if not "$suffix" in name_pattern and not "${suffix}" in name_pattern:
+			raise SetConfigError("name_pattern doesn't include $suffix placeholder")
+		for suffix in ["keywords", "use", "mask", "unmask"]:
+			myname = name_pattern.replace("$suffix", suffix)
+			myname = myname.replace("${suffix}", suffix)
+			rValue[myname] = ConfigFileSet(os.path.join(directory, "package."+suffix))
+		return rValue
+	multiBuilder = classmethod(multiBuilder)
 
 class WorldSet(StaticFileSet):
 	description = "Set of packages that were directly installed by the user"
 	
 	def __init__(self, root):
-		super(WorldSet, self).__init__(os.path.join(os.sep, root, PRIVATE_PATH, "world"))
+		super(WorldSet, self).__init__(os.path.join(os.sep, root, PRIVATE_PATH.lstrip(os.sep), "world"))
 		self._lock = None
 
 	def _ensure_dirs(self):
@@ -89,3 +127,8 @@ class WorldSet(StaticFileSet):
 	def unlock(self):
 		unlockfile(self._lock)
 		self._lock = None
+
+	def singleBuilder(self, options, settings, trees):
+		print "world.build"
+		return WorldSet(settings["ROOT"])
+	singleBuilder = classmethod(singleBuilder)
diff --git a/pym/portage/sets/profiles.py b/pym/portage/sets/profiles.py
index 7dbc35e36..6d629fc67 100644
--- a/pym/portage/sets/profiles.py
+++ b/pym/portage/sets/profiles.py
@@ -18,3 +18,7 @@ class PackagesSystemSet(PackageSet):
 		mylist = [grabfile_package(os.path.join(x, "packages")) for x in self._profile_paths]
 		mylist = stack_lists(mylist, incremental=1)
 		self._setAtoms([x[1:] for x in mylist if x[0] == "*"])
+
+	def singleBuilder(self, options, settings, trees):
+		return PackagesSystemSet(settings.profiles)
+	singleBuilder = classmethod(singleBuilder)
diff --git a/pym/portage/sets/security.py b/pym/portage/sets/security.py
index 6c463df9a..576009364 100644
--- a/pym/portage/sets/security.py
+++ b/pym/portage/sets/security.py
@@ -14,12 +14,13 @@ class SecuritySet(PackageSet):
 	
 	description = "package set that includes all packages possibly affected by a GLSA"
 		
-	def __init__(self, settings, vardbapi, portdbapi):
+	def __init__(self, settings, vardbapi, portdbapi, least_change=True):
 		super(SecuritySet, self).__init__()
 		self._settings = settings
 		self._vardbapi = vardbapi
 		self._portdbapi = portdbapi
 		self._checkfile = os.path.join(os.sep, self._settings["ROOT"], CACHE_PATH.lstrip(os.sep), "glsa")
+		self._least_change = least_change
 
 	def getGlsaList(self, skip_applied):
 		glsaindexlist = glsa.get_glsa_list(self._settings)
@@ -37,7 +38,7 @@ class SecuritySet(PackageSet):
 			myglsa = glsa.Glsa(glsaid, self._settings, self._vardbapi, self._portdbapi)
 			#print glsaid, myglsa.isVulnerable(), myglsa.isApplied(), myglsa.getMergeList()
 			if self.useGlsa(myglsa):
-				atomlist += myglsa.getMergeList(least_change=False)
+				atomlist += myglsa.getMergeList(least_change=self._least_change)
 		self._setAtoms(atomlist)
 	
 	def useGlsa(self, myglsa):
@@ -52,6 +53,15 @@ class SecuritySet(PackageSet):
 				applied_list.append(glsaid)
 		write_atomic(self._checkfile, "\n".join(applied_list))
 	
+	def singleBuilder(cls, options, settings, trees):
+		if "use_emerge_resoler" in options \
+				and options.get("use_emerge_resolver").lower() in ["1", "yes", "true", "on"]:
+			least_change = False
+		else:
+			least_change = True
+		return cls(settings, trees["vartree"].dbapi, trees["porttree"].dbapi, least_change=least_change)
+	singleBuilder = classmethod(singleBuilder)
+	
 class NewGlsaSet(SecuritySet):
 	_skip_applied = True
 	description = "Package set that includes all packages possibly affected by an unapplied GLSA"
diff --git a/pym/portage/sets/shell.py b/pym/portage/sets/shell.py
index 7bfaeaecb..d61c116ce 100644
--- a/pym/portage/sets/shell.py
+++ b/pym/portage/sets/shell.py
@@ -5,7 +5,7 @@
 import subprocess
 import os
 
-from portage.sets import PackageSet
+from portage.sets import PackageSet, SetConfigError
 
 class CommandOutputSet(PackageSet):
 	"""This class creates a PackageSet from the output of a shell command.
@@ -34,3 +34,7 @@ class CommandOutputSet(PackageSet):
 			text = pipe.stdout.read()
 			self._setAtoms(text.split("\n"))
 		
+	def singleBuilder(self, options, settings, trees):
+		if not command in options:
+			raise SetConfigError("no command specified")
+		return CommandOutputSet(options["command"])