From a3b036604dfe8ae0d1c256ca9d768c1071740241 Mon Sep 17 00:00:00 2001 From: Marius Mauch Date: Fri, 5 Oct 2007 17:27:34 +0000 Subject: Add set configuration framework in preparation for emerge integration svn path=/main/trunk/; revision=7952 --- pym/portage/sets/__init__.py | 161 ++++++++++++++++++++++++------------------- pym/portage/sets/dbapi.py | 59 +++++++++++++++- pym/portage/sets/files.py | 49 ++++++++++++- pym/portage/sets/profiles.py | 4 ++ pym/portage/sets/security.py | 14 +++- pym/portage/sets/shell.py | 6 +- 6 files changed, 216 insertions(+), 77 deletions(-) (limited to 'pym') diff --git a/pym/portage/sets/__init__.py b/pym/portage/sets/__init__.py index e62a9d750..746a7e7a9 100644 --- a/pym/portage/sets/__init__.py +++ b/pym/portage/sets/__init__.py @@ -3,7 +3,8 @@ # $Id$ import os -from portage import flatten +from ConfigParser import SafeConfigParser, NoOptionError +from portage import flatten, load_mod from portage.dep import isvalidatom, match_from_list, \ best_match_to_list, dep_getkey, use_reduce, paren_reduce from portage.exception import InvalidAtom @@ -145,7 +146,6 @@ class EditablePackageSet(PackageSet): # This method must be overwritten in subclasses that should be editable raise NotImplementedError() - class InternalPackageSet(EditablePackageSet): def __init__(self, initial_atoms=None): super(InternalPackageSet, self).__init__() @@ -163,78 +163,99 @@ class InternalPackageSet(EditablePackageSet): pass +class SetConfigError(Exception): + pass + +class SetConfig(SafeConfigParser): + def __init__(self, paths, settings, trees): + SafeConfigParser.__init__(self) + self.read(paths) + self.errors = [] + self.psets = {} + self.trees = trees + self.settings = settings + + def _parse(self): + for sname in self.sections(): + # find classname for current section, default to file based sets + if not self.has_option(sname, "class"): + classname = "portage.sets.files.StaticFileSet" + else: + classname = self.get(sname, "class") + + # try to import the specified class + try: + setclass = load_mod(classname) + except (ImportError, AttributeError): + self.errors.append("Could not import '%s' for section '%s'" % (classname, sname)) + continue + # prepare option dict for the current section + optdict = {} + for oname in self.options(sname): + optdict[oname] = self.get(sname, oname) + + # create single or multiple instances of the given class depending on configuration + if self.has_option(sname, "multiset") and self.getboolean(sname, "multiset"): + if hasattr(setclass, "multiBuilder"): + try: + self.psets.update(setclass.multiBuilder(optdict, self.settings, self.trees)) + except SetConfigError, e: + self.errors.append("Configuration error in section '%s': %s" % (sname, str(e))) + continue + else: + self.errors.append("Section '%s' is configured as multiset, but '%s' doesn't support that configuration" % (sname, classname)) + continue + else: + try: + setname = self.get(sname, "name") + except NoOptionError: + setname = "sets/"+sname + if hasattr(setclass, "singleBuilder"): + try: + self.psets[setname] = setclass.singleBuilder(optdict, self.settings, self.trees) + except SetConfigError, e: + self.errors.append("Configuration error in section '%s': %s" % (sname, str(e))) + continue + else: + self.errors.append("'%s' does not support individual set creation, section '%s' must be configured as multiset" % (classname, sname)) + continue + + def getSets(self): + self._parse() + return (self.psets, self.errors) + +def make_default_config(settings, trees): + sc = SetConfig([], settings, trees) + sc.add_section("security") + sc.set("security", "class", "portage.sets.security.NewAffectedSet") + + sc.add_section("system") + sc.set("system", "class", "portage.sets.profiles.PackagesSystemSet") + + sc.add_section("world") + sc.set("world", "class", "portage.sets.files.WorldSet") + + sc.add_section("everything") + sc.set("everything", "class", "portage.sets.dbapi.EverythingSet") -def make_default_sets(configroot, root, profile_paths, settings=None, - vdbapi=None, portdbapi=None): - from portage.sets.files import StaticFileSet, ConfigFileSet - from portage.sets.profiles import PackagesSystemSet - from portage.sets.security import NewAffectedSet - from portage.sets.dbapi import EverythingSet - from portage.const import PRIVATE_PATH, USER_CONFIG_PATH - - rValue = {} - worldset = StaticFileSet(os.path.join(root, PRIVATE_PATH, "world")) - worldset.description = "Set of packages that were directly installed" - rValue["world"] = worldset - for suffix in ["mask", "unmask", "keywords", "use"]: - myname = "package_"+suffix - myset = ConfigFileSet(os.path.join(configroot, USER_CONFIG_PATH.lstrip(os.sep), "package."+suffix)) - rValue[myname] = myset - rValue["system"] = PackagesSystemSet(profile_paths) - if settings != None and portdbapi != None: - rValue["security"] = NewAffectedSet(settings, vdbapi, portdbapi) - else: - rValue["security"] = InternalPackageSet() - if vdbapi != None: - rValue["everything"] = EverythingSet(vdbapi) - else: - rValue["everything"] = InternalPackageSet() - - return rValue - -def make_extra_static_sets(configroot): - from portage.sets.files import StaticFileSet - from portage.const import PRIVATE_PATH, USER_CONFIG_PATH - - rValue = {} - mydir = os.path.join(configroot, USER_CONFIG_PATH.lstrip(os.sep), "sets") - try: - mysets = os.listdir(mydir) - except (OSError, IOError): - return rValue - for myname in mysets: - if myname in DEFAULT_SETS: - continue - rValue[myname] = StaticFileSet(os.path.join(mydir, myname)) - return rValue - -def make_category_sets(portdbapi, settings, only_visible=True): - from portage.sets.dbapi import CategorySet - rValue = {} - for c in settings.categories: - rValue["category_%s" % c] = CategorySet(c, portdbapi, only_visible=only_visible) - return rValue + sc.add_section("config") + sc.set("config", "class", "portage.sets.files.ConfigFileSet") + sc.set("config", "multiset", "true") + + sc.add_section("categories_installed") + sc.set("categories_installed", "class", "portage.sets.dbapi.CategorySet") + sc.set("categories_installed", "multiset", "true") + sc.set("categories_installed", "repository", "vartree") + sc.set("categories_installed", "name_pattern", "installed/$category") + + return sc # adhoc test code if __name__ == "__main__": - import portage, sys, os - from portage.sets.dbapi import CategorySet - from portage.sets.files import StaticFileSet - l = make_default_sets("/", "/", portage.settings.profiles, portage.settings, portage.db["/"]["vartree"].dbapi, portage.db["/"]["porttree"].dbapi) - l.update(make_extra_static_sets("/")) - if len(sys.argv) > 1: - for s in sys.argv[1:]: - if s.startswith("category_"): - c = s[9:] - l["category_%s" % c] = CategorySet(c, portage.db['/']['porttree'].dbapi, only_visible=False) - elif os.path.exists(s): - l[os.path.basename(s)] = StaticFileSet(s) - elif s != "*": - print "ERROR: could not create set '%s'" % s - if not "*" in sys.argv: - for n in l: - if n not in sys.argv[1:]: - del l[n] + import portage + sc = make_default_config(portage.settings, portage.db["/"]) + l, e = sc.getSets() + print l, e for x in l: print x+":" print "DESCRIPTION = %s" % l[x].getMetadata("Description") diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py index 5c16b1701..1a7a66907 100644 --- a/pym/portage/sets/dbapi.py +++ b/pym/portage/sets/dbapi.py @@ -3,7 +3,7 @@ # $Id$ from portage.versions import catsplit -from portage.sets import PackageSet +from portage.sets import PackageSet, SetConfigError class EverythingSet(PackageSet): _operations = ["merge", "unmerge"] @@ -23,6 +23,10 @@ class EverythingSet(PackageSet): else: myatoms.append(cp) self._setAtoms(myatoms) + + def singleBuilder(self, options, settings, trees): + return EverythingSet(trees["vartree"].dbapi) + singleBuilder = classmethod(singleBuilder) class CategorySet(PackageSet): _operations = ["merge", "unmerge"] @@ -46,3 +50,56 @@ class CategorySet(PackageSet): myatoms.append(cp) self._setAtoms(myatoms) + def _builderGetRepository(cls, options, repositories): + repository = options.get("repository", "porttree") + if not repository in repositories: + raise SetConfigError("invalid repository class '%s'" % repository) + return repository + _builderGetRepository = classmethod(_builderGetRepository) + + def _builderGetVisible(cls, options): + visible = options.get("only_visible", "true").lower() + if visible not in ["1", "0", "yes", "no", "true", "false", "on", "off"]: + raise SetConfigError("invalid value for only_visible: %s" % visible) + return bool(visible in ["1", "yes", "true", "on"]) + _builderGetVisible = classmethod(_builderGetVisible) + + def singleBuilder(cls, options, settings, trees): + if not "category" in options: + raise SetConfigError("no category given") + + category = options["category"] + if not category in categories: + raise SetConfigError("invalid category name '%s'" % category) + + repository = cls._builderGetRepository(options, trees.keys()) + visible = cls._builderGetVisible(options) + + return CategorySet(category, dbapi=trees[repository].dbapi, only_visible=visible) + singleBuilder = classmethod(singleBuilder) + + def multiBuilder(cls, options, settings, trees): + rValue = {} + + if "categories" in options: + categories = options["categories"].split() + invalid = set(categories).difference(settings.categories) + if invalid: + raise SetConfigError("invalid categories: %s" % ", ".join(list(invalid))) + else: + categories = settings.categories + + repository = cls._builderGetRepository(options, trees.keys()) + visible = cls._builderGetVisible(options) + name_pattern = options.get("name_pattern", "$category/*") + + if not "$category" in name_pattern and not "${category}" in name_pattern: + raise SetConfigError("name_pattern doesn't include $category placeholder") + + for cat in categories: + myset = CategorySet(cat, trees[repository].dbapi, only_visible=visible) + myname = name_pattern.replace("$category", cat) + myname = myname.replace("${category}", cat) + rValue[myname] = myset + return rValue + multiBuilder = classmethod(multiBuilder) diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index 6379e2641..8a7f39549 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -6,10 +6,10 @@ import errno import os from portage.util import grabfile, write_atomic, ensure_dirs -from portage.const import PRIVATE_PATH +from portage.const import PRIVATE_PATH, USER_CONFIG_PATH from portage.locks import lockfile, unlockfile from portage import portage_gid -from portage.sets import PackageSet, EditablePackageSet +from portage.sets import PackageSet, EditablePackageSet, SetConfigError from portage.env.loaders import ItemFileLoader, KeyListFileLoader from portage.env.validators import ValidAtomValidator @@ -60,6 +60,25 @@ class StaticFileSet(EditablePackageSet): data = {} self._setAtoms(data.keys()) self._mtime = mtime + + def singleBuilder(self, options, settings, trees): + if not "filename" in options: + raise SetConfigError("no filename specified") + return ConfigFileSet(options[filename]) + singleBuilder = classmethod(singleBuilder) + + def multiBuilder(self, options, settings, trees): + rValue = {} + directory = options.get("directory", os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH.lstrip(os.sep), "sets")) + name_pattern = options.get("name_pattern", "sets/$name") + if not "$name" in name_pattern and not "${name}" in name_pattern: + raise SetConfigError("name_pattern doesn't include $name placeholder") + for filename in os.listdir(directory): + myname = name_pattern.replace("$name", filename) + myname = myname.replace("${name}", filename) + rValue[myname] = StaticFileSet(os.path.join(directory, filename)) + return rValue + multiBuilder = classmethod(multiBuilder) class ConfigFileSet(PackageSet): def __init__(self, filename): @@ -71,12 +90,31 @@ class ConfigFileSet(PackageSet): def load(self): data, errors = self.loader.load() self._setAtoms(data.keys()) + + def singleBuilder(self, options, settings, trees): + if not "filename" in options: + raise SetConfigError("no filename specified") + return ConfigFileSet(options[filename]) + singleBuilder = classmethod(singleBuilder) + + def multiBuilder(self, options, settings, trees): + rValue = {} + directory = options.get("directory", os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH.lstrip(os.sep))) + name_pattern = options.get("name_pattern", "sets/package_$suffix") + if not "$suffix" in name_pattern and not "${suffix}" in name_pattern: + raise SetConfigError("name_pattern doesn't include $suffix placeholder") + for suffix in ["keywords", "use", "mask", "unmask"]: + myname = name_pattern.replace("$suffix", suffix) + myname = myname.replace("${suffix}", suffix) + rValue[myname] = ConfigFileSet(os.path.join(directory, "package."+suffix)) + return rValue + multiBuilder = classmethod(multiBuilder) class WorldSet(StaticFileSet): description = "Set of packages that were directly installed by the user" def __init__(self, root): - super(WorldSet, self).__init__(os.path.join(os.sep, root, PRIVATE_PATH, "world")) + super(WorldSet, self).__init__(os.path.join(os.sep, root, PRIVATE_PATH.lstrip(os.sep), "world")) self._lock = None def _ensure_dirs(self): @@ -89,3 +127,8 @@ class WorldSet(StaticFileSet): def unlock(self): unlockfile(self._lock) self._lock = None + + def singleBuilder(self, options, settings, trees): + print "world.build" + return WorldSet(settings["ROOT"]) + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/sets/profiles.py b/pym/portage/sets/profiles.py index 7dbc35e36..6d629fc67 100644 --- a/pym/portage/sets/profiles.py +++ b/pym/portage/sets/profiles.py @@ -18,3 +18,7 @@ class PackagesSystemSet(PackageSet): mylist = [grabfile_package(os.path.join(x, "packages")) for x in self._profile_paths] mylist = stack_lists(mylist, incremental=1) self._setAtoms([x[1:] for x in mylist if x[0] == "*"]) + + def singleBuilder(self, options, settings, trees): + return PackagesSystemSet(settings.profiles) + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/sets/security.py b/pym/portage/sets/security.py index 6c463df9a..576009364 100644 --- a/pym/portage/sets/security.py +++ b/pym/portage/sets/security.py @@ -14,12 +14,13 @@ class SecuritySet(PackageSet): description = "package set that includes all packages possibly affected by a GLSA" - def __init__(self, settings, vardbapi, portdbapi): + def __init__(self, settings, vardbapi, portdbapi, least_change=True): super(SecuritySet, self).__init__() self._settings = settings self._vardbapi = vardbapi self._portdbapi = portdbapi self._checkfile = os.path.join(os.sep, self._settings["ROOT"], CACHE_PATH.lstrip(os.sep), "glsa") + self._least_change = least_change def getGlsaList(self, skip_applied): glsaindexlist = glsa.get_glsa_list(self._settings) @@ -37,7 +38,7 @@ class SecuritySet(PackageSet): myglsa = glsa.Glsa(glsaid, self._settings, self._vardbapi, self._portdbapi) #print glsaid, myglsa.isVulnerable(), myglsa.isApplied(), myglsa.getMergeList() if self.useGlsa(myglsa): - atomlist += myglsa.getMergeList(least_change=False) + atomlist += myglsa.getMergeList(least_change=self._least_change) self._setAtoms(atomlist) def useGlsa(self, myglsa): @@ -52,6 +53,15 @@ class SecuritySet(PackageSet): applied_list.append(glsaid) write_atomic(self._checkfile, "\n".join(applied_list)) + def singleBuilder(cls, options, settings, trees): + if "use_emerge_resoler" in options \ + and options.get("use_emerge_resolver").lower() in ["1", "yes", "true", "on"]: + least_change = False + else: + least_change = True + return cls(settings, trees["vartree"].dbapi, trees["porttree"].dbapi, least_change=least_change) + singleBuilder = classmethod(singleBuilder) + class NewGlsaSet(SecuritySet): _skip_applied = True description = "Package set that includes all packages possibly affected by an unapplied GLSA" diff --git a/pym/portage/sets/shell.py b/pym/portage/sets/shell.py index 7bfaeaecb..d61c116ce 100644 --- a/pym/portage/sets/shell.py +++ b/pym/portage/sets/shell.py @@ -5,7 +5,7 @@ import subprocess import os -from portage.sets import PackageSet +from portage.sets import PackageSet, SetConfigError class CommandOutputSet(PackageSet): """This class creates a PackageSet from the output of a shell command. @@ -34,3 +34,7 @@ class CommandOutputSet(PackageSet): text = pipe.stdout.read() self._setAtoms(text.split("\n")) + def singleBuilder(self, options, settings, trees): + if not command in options: + raise SetConfigError("no command specified") + return CommandOutputSet(options["command"]) -- cgit v1.2.3-1-g7c22