diff options
Diffstat (limited to 'pym/portage/_sets')
-rw-r--r-- | pym/portage/_sets/__init__.py | 203 | ||||
-rw-r--r-- | pym/portage/_sets/base.py | 253 | ||||
-rw-r--r-- | pym/portage/_sets/dbapi.py | 337 | ||||
-rw-r--r-- | pym/portage/_sets/files.py | 327 | ||||
-rw-r--r-- | pym/portage/_sets/libs.py | 91 | ||||
-rw-r--r-- | pym/portage/_sets/profiles.py | 53 | ||||
-rw-r--r-- | pym/portage/_sets/security.py | 86 | ||||
-rw-r--r-- | pym/portage/_sets/shell.py | 44 |
8 files changed, 1394 insertions, 0 deletions
diff --git a/pym/portage/_sets/__init__.py b/pym/portage/_sets/__init__.py new file mode 100644 index 000000000..66d558f6c --- /dev/null +++ b/pym/portage/_sets/__init__.py @@ -0,0 +1,203 @@ +# Copyright 2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + +__all__ = ["SETPREFIX", "get_boolean", "SetConfigError", + "SetConfig", "load_default_config"] + +try: + from configparser import SafeConfigParser, NoOptionError +except ImportError: + from ConfigParser import SafeConfigParser, NoOptionError +from portage import os +from portage import load_mod +from portage.const import USER_CONFIG_PATH, GLOBAL_CONFIG_PATH +from portage.exception import PackageSetNotFound +from portage.localization import _ + +SETPREFIX = "@" + +def get_boolean(options, name, default): + if not name in options: + return default + elif options[name].lower() in ("1", "yes", "on", "true"): + return True + elif options[name].lower() in ("0", "no", "off", "false"): + return False + else: + raise SetConfigError(_("invalid value '%(value)s' for option '%(option)s'") % {"value": options[name], "option": name}) + +class SetConfigError(Exception): + pass + +class SetConfig(object): + def __init__(self, paths, settings, trees): + self._parser = SafeConfigParser( + defaults={ + "EPREFIX" : settings["EPREFIX"], + "EROOT" : settings["EROOT"], + "PORTAGE_CONFIGROOT" : settings["PORTAGE_CONFIGROOT"], + "ROOT" : settings["ROOT"], + }) + self._parser.read(paths) + self.errors = [] + self.psets = {} + self.trees = trees + self.settings = settings + self._parsed = False + self.active = [] + + def update(self, setname, options): + parser = self._parser + self.errors = [] + if not setname in self.psets: + options["name"] = setname + options["world-candidate"] = "False" + + # for the unlikely case that there is already a section with the requested setname + import random + while setname in parser.sections(): + setname = "%08d" % random.randint(0, 10**10) + + parser.add_section(setname) + for k, v in options.items(): + parser.set(setname, k, v) + else: + section = self.psets[setname].creator + if parser.has_option(section, "multiset") and \ + parser.getboolean(section, "multiset"): + self.errors.append(_("Invalid request to reconfigure set '%(set)s' generated " + "by multiset section '%(section)s'") % {"set": setname, "section": section}) + return + for k, v in options.items(): + parser.set(section, k, v) + self._parse(update=True) + + def _parse(self, update=False): + if self._parsed and not update: + return + parser = self._parser + for sname in parser.sections(): + # find classname for current section, default to file based sets + if not parser.has_option(sname, "class"): + classname = "portage._sets.files.StaticFileSet" + else: + classname = parser.get(sname, "class") + + if classname.startswith('portage.sets.'): + # The module has been made private, but we still support + # the previous namespace for sets.conf entries. + classname = classname.replace('sets', '_sets', 1) + + # try to import the specified class + try: + setclass = load_mod(classname) + except (ImportError, AttributeError): + try: + setclass = load_mod("portage._sets." + classname) + except (ImportError, AttributeError): + self.errors.append(_("Could not import '%(class)s' for section " + "'%(section)s'") % {"class": classname, "section": sname}) + continue + # prepare option dict for the current section + optdict = {} + for oname in parser.options(sname): + optdict[oname] = parser.get(sname, oname) + + # create single or multiple instances of the given class depending on configuration + if parser.has_option(sname, "multiset") and \ + parser.getboolean(sname, "multiset"): + if hasattr(setclass, "multiBuilder"): + newsets = {} + try: + newsets = setclass.multiBuilder(optdict, self.settings, self.trees) + except SetConfigError as e: + self.errors.append(_("Configuration error in section '%s': %s") % (sname, str(e))) + continue + for x in newsets: + if x in self.psets and not update: + self.errors.append(_("Redefinition of set '%s' (sections: '%s', '%s')") % (x, self.psets[x].creator, sname)) + newsets[x].creator = sname + if parser.has_option(sname, "world-candidate") and \ + parser.getboolean(sname, "world-candidate"): + newsets[x].world_candidate = True + self.psets.update(newsets) + else: + self.errors.append(_("Section '%(section)s' is configured as multiset, but '%(class)s' " + "doesn't support that configuration") % {"section": sname, "class": classname}) + continue + else: + try: + setname = parser.get(sname, "name") + except NoOptionError: + setname = sname + if setname in self.psets and not update: + self.errors.append(_("Redefinition of set '%s' (sections: '%s', '%s')") % (setname, self.psets[setname].creator, sname)) + if hasattr(setclass, "singleBuilder"): + try: + self.psets[setname] = setclass.singleBuilder(optdict, self.settings, self.trees) + self.psets[setname].creator = sname + if parser.has_option(sname, "world-candidate") and \ + parser.getboolean(sname, "world-candidate"): + self.psets[setname].world_candidate = True + except SetConfigError as e: + self.errors.append(_("Configuration error in section '%s': %s") % (sname, str(e))) + continue + else: + self.errors.append(_("'%(class)s' does not support individual set creation, section '%(section)s' " + "must be configured as multiset") % {"class": classname, "section": sname}) + continue + self._parsed = True + + def getSets(self): + self._parse() + return self.psets.copy() + + def getSetAtoms(self, setname, ignorelist=None): + """ + This raises PackageSetNotFound if the give setname does not exist. + """ + self._parse() + try: + myset = self.psets[setname] + except KeyError: + raise PackageSetNotFound(setname) + myatoms = myset.getAtoms() + parser = self._parser + + if ignorelist is None: + ignorelist = set() + + ignorelist.add(setname) + for n in myset.getNonAtoms(): + if n.startswith(SETPREFIX): + s = n[len(SETPREFIX):] + if s in self.psets: + if s not in ignorelist: + myatoms.update(self.getSetAtoms(s, + ignorelist=ignorelist)) + else: + raise PackageSetNotFound(s) + + return myatoms + +def load_default_config(settings, trees): + global_config_path = GLOBAL_CONFIG_PATH + if settings['EPREFIX']: + global_config_path = os.path.join(settings['EPREFIX'], + GLOBAL_CONFIG_PATH.lstrip(os.sep)) + def _getfiles(): + for path, dirs, files in os.walk(os.path.join(global_config_path, "sets")): + for f in files: + yield os.path.join(path, f) + + dbapi = trees["porttree"].dbapi + for repo in dbapi.getRepositories(): + path = dbapi.getRepositoryPath(repo) + yield os.path.join(path, "sets.conf") + + yield os.path.join(settings["PORTAGE_CONFIGROOT"], + USER_CONFIG_PATH, "sets.conf") + + return SetConfig(_getfiles(), settings, trees) diff --git a/pym/portage/_sets/base.py b/pym/portage/_sets/base.py new file mode 100644 index 000000000..cd9c08d32 --- /dev/null +++ b/pym/portage/_sets/base.py @@ -0,0 +1,253 @@ +# Copyright 2007-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import sys +from portage.dep import Atom, ExtendedAtomDict, best_match_to_list, match_from_list +from portage.exception import InvalidAtom +from portage.versions import cpv_getkey + +if sys.hexversion >= 0x3000000: + basestring = str + +OPERATIONS = ["merge", "unmerge"] + +class PackageSet(object): + # Set this to operations that are supported by your subclass. While + # technically there is no difference between "merge" and "unmerge" regarding + # package sets, the latter doesn't make sense for some sets like "system" + # or "security" and therefore isn't supported by them. + _operations = ["merge"] + description = "generic package set" + + def __init__(self): + self._atoms = set() + self._atommap = ExtendedAtomDict(set) + self._loaded = False + self._loading = False + self.errors = [] + self._nonatoms = set() + self.world_candidate = False + + def __contains__(self, atom): + self._load() + return atom in self._atoms or atom in self._nonatoms + + def __iter__(self): + self._load() + for x in self._atoms: + yield x + for x in self._nonatoms: + yield x + + def __bool__(self): + self._load() + return bool(self._atoms or self._nonatoms) + + if sys.hexversion < 0x3000000: + __nonzero__ = __bool__ + + def supportsOperation(self, op): + if not op in OPERATIONS: + raise ValueError(op) + return op in self._operations + + def _load(self): + if not (self._loaded or self._loading): + self._loading = True + self.load() + self._loaded = True + self._loading = False + + def getAtoms(self): + self._load() + return self._atoms.copy() + + def getNonAtoms(self): + self._load() + return self._nonatoms.copy() + + def _setAtoms(self, atoms, allow_wildcard=False): + self._atoms.clear() + self._nonatoms.clear() + for a in atoms: + if not isinstance(a, Atom): + if isinstance(a, basestring): + a = a.strip() + if not a: + continue + try: + a = Atom(a, allow_wildcard=True) + except InvalidAtom: + self._nonatoms.add(a) + continue + if not allow_wildcard and a.extended_syntax: + raise InvalidAtom("extended atom syntax not allowed here") + self._atoms.add(a) + + self._updateAtomMap() + + def load(self): + # This method must be overwritten by subclasses + # Editable sets should use the value of self._mtime to determine if they + # need to reload themselves + raise NotImplementedError() + + def containsCPV(self, cpv): + self._load() + for a in self._atoms: + if match_from_list(a, [cpv]): + return True + return False + + def getMetadata(self, key): + if hasattr(self, key.lower()): + return getattr(self, key.lower()) + else: + return "" + + def _updateAtomMap(self, atoms=None): + """Update self._atommap for specific atoms or all atoms.""" + if not atoms: + self._atommap.clear() + atoms = self._atoms + for a in atoms: + self._atommap.setdefault(a.cp, set()).add(a) + + # Not sure if this one should really be in PackageSet + def findAtomForPackage(self, pkg, modified_use=None): + """Return the best match for a given package from the arguments, or + None if there are no matches. This matches virtual arguments against + the PROVIDE metadata. This can raise an InvalidDependString exception + if an error occurs while parsing PROVIDE.""" + + if modified_use is not None and modified_use is not pkg.use.enabled: + pkg = pkg.copy() + pkg.metadata["USE"] = " ".join(modified_use) + + # Atoms matched via PROVIDE must be temporarily transformed since + # match_from_list() only works correctly when atom.cp == pkg.cp. + rev_transform = {} + for atom in self.iterAtomsForPackage(pkg): + if atom.cp == pkg.cp: + rev_transform[atom] = atom + else: + rev_transform[Atom(atom.replace(atom.cp, pkg.cp, 1), allow_wildcard=True)] = atom + best_match = best_match_to_list(pkg, iter(rev_transform)) + if best_match: + return rev_transform[best_match] + return None + + def iterAtomsForPackage(self, pkg): + """ + Find all matching atoms for a given package. This matches virtual + arguments against the PROVIDE metadata. This will raise an + InvalidDependString exception if PROVIDE is invalid. + """ + cpv_slot_list = [pkg] + cp = cpv_getkey(pkg.cpv) + self._load() # make sure the atoms are loaded + + atoms = self._atommap.get(cp) + if atoms: + for atom in atoms: + if match_from_list(atom, cpv_slot_list): + yield atom + provides = pkg.metadata['PROVIDE'] + if not provides: + return + provides = provides.split() + for provide in provides: + try: + provided_cp = Atom(provide).cp + except InvalidAtom: + continue + atoms = self._atommap.get(provided_cp) + if atoms: + for atom in atoms: + if match_from_list(atom.replace(provided_cp, cp), + cpv_slot_list): + yield atom + +class EditablePackageSet(PackageSet): + + def __init__(self, allow_wildcard=False): + super(EditablePackageSet, self).__init__() + self._allow_wildcard = allow_wildcard + + def update(self, atoms): + self._load() + modified = False + normal_atoms = [] + for a in atoms: + if not isinstance(a, Atom): + try: + a = Atom(a, allow_wildcard=True) + except InvalidAtom: + modified = True + self._nonatoms.add(a) + continue + if not self._allow_wildcard and a.extended_syntax: + raise InvalidAtom("extended atom syntax not allowed here") + normal_atoms.append(a) + + if normal_atoms: + modified = True + self._atoms.update(normal_atoms) + self._updateAtomMap(atoms=normal_atoms) + if modified: + self.write() + + def add(self, atom): + self.update([atom]) + + def replace(self, atoms): + self._setAtoms(atoms, allow_wildcard=self._allow_wildcard) + self.write() + + def remove(self, atom): + self._load() + self._atoms.discard(atom) + self._nonatoms.discard(atom) + self._updateAtomMap() + self.write() + + def removePackageAtoms(self, cp): + self._load() + for a in list(self._atoms): + if a.cp == cp: + self.remove(a) + self.write() + + def write(self): + # This method must be overwritten in subclasses that should be editable + raise NotImplementedError() + +class InternalPackageSet(EditablePackageSet): + def __init__(self, initial_atoms=None, allow_wildcard=False): + super(InternalPackageSet, self).__init__(allow_wildcard=allow_wildcard) + if initial_atoms != None: + self.update(initial_atoms) + + def clear(self): + self._atoms.clear() + self._updateAtomMap() + + def load(self): + pass + + def write(self): + pass + +class DummyPackageSet(PackageSet): + def __init__(self, atoms=None): + super(DummyPackageSet, self).__init__() + if atoms: + self._setAtoms(atoms) + + def load(self): + pass + + def singleBuilder(cls, options, settings, trees): + atoms = options.get("packages", "").split() + return DummyPackageSet(atoms=atoms) + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/_sets/dbapi.py b/pym/portage/_sets/dbapi.py new file mode 100644 index 000000000..362cc91c6 --- /dev/null +++ b/pym/portage/_sets/dbapi.py @@ -0,0 +1,337 @@ +# Copyright 2007-2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import time + +from portage import os +from portage.versions import catpkgsplit, catsplit, pkgcmp, best +from portage.dep import Atom +from portage.localization import _ +from portage._sets.base import PackageSet +from portage._sets import SetConfigError, get_boolean +import portage + +__all__ = ["CategorySet", "DowngradeSet", + "EverythingSet", "OwnerSet", "VariableSet"] + +class EverythingSet(PackageSet): + _operations = ["merge"] + description = "Package set which contains SLOT " + \ + "atoms to match all installed packages" + _filter = None + + def __init__(self, vdbapi, **kwargs): + super(EverythingSet, self).__init__() + self._db = vdbapi + + def load(self): + myatoms = [] + db_keys = ["SLOT"] + aux_get = self._db.aux_get + cp_list = self._db.cp_list + + for cp in self._db.cp_all(): + cpv_list = cp_list(cp) + + if len(cpv_list) > 1: + for cpv in cpv_list: + slot, = aux_get(cpv, db_keys) + atom = Atom("%s:%s" % (cp, slot)) + if self._filter: + if self._filter(atom): + myatoms.append(atom) + else: + myatoms.append(atom) + + else: + atom = Atom(cp) + if self._filter: + if self._filter(atom): + myatoms.append(atom) + else: + myatoms.append(atom) + + self._setAtoms(myatoms) + + def singleBuilder(self, options, settings, trees): + return EverythingSet(trees["vartree"].dbapi) + singleBuilder = classmethod(singleBuilder) + +class OwnerSet(PackageSet): + + _operations = ["merge", "unmerge"] + + description = "Package set which contains all packages " + \ + "that own one or more files." + + def __init__(self, vardb=None, files=None): + super(OwnerSet, self).__init__() + self._db = vardb + self._files = files + + def mapPathsToAtoms(self, paths): + rValue = set() + vardb = self._db + aux_get = vardb.aux_get + aux_keys = ["SLOT"] + for link, p in vardb._owners.iter_owners(paths): + cat, pn = catpkgsplit(link.mycpv)[:2] + slot, = aux_get(link.mycpv, aux_keys) + rValue.add("%s/%s:%s" % (cat, pn, slot)) + return rValue + + def load(self): + self._setAtoms(self.mapPathsToAtoms(self._files)) + + def singleBuilder(cls, options, settings, trees): + if not "files" in options: + raise SetConfigError(_("no files given")) + + return cls(vardb=trees["vartree"].dbapi, + files=frozenset(portage.util.shlex_split(options["files"]))) + + singleBuilder = classmethod(singleBuilder) + +class VariableSet(EverythingSet): + + _operations = ["merge", "unmerge"] + + description = "Package set which contains all packages " + \ + "that match specified values of a specified variable." + + def __init__(self, vardb, metadatadb=None, variable=None, includes=None, excludes=None): + super(VariableSet, self).__init__(vardb) + self._metadatadb = metadatadb + self._variable = variable + self._includes = includes + self._excludes = excludes + + def _filter(self, atom): + ebuild = best(self._metadatadb.match(atom)) + if not ebuild: + return False + values, = self._metadatadb.aux_get(ebuild, [self._variable]) + values = values.split() + if self._includes and not self._includes.intersection(values): + return False + if self._excludes and self._excludes.intersection(values): + return False + return True + + def singleBuilder(cls, options, settings, trees): + + variable = options.get("variable") + if variable is None: + raise SetConfigError(_("missing required attribute: 'variable'")) + + includes = options.get("includes", "") + excludes = options.get("excludes", "") + + if not (includes or excludes): + raise SetConfigError(_("no includes or excludes given")) + + metadatadb = options.get("metadata-source", "vartree") + if not metadatadb in trees: + raise SetConfigError(_("invalid value '%s' for option metadata-source") % metadatadb) + + return cls(trees["vartree"].dbapi, + metadatadb=trees[metadatadb].dbapi, + excludes=frozenset(excludes.split()), + includes=frozenset(includes.split()), + variable=variable) + + singleBuilder = classmethod(singleBuilder) + +class DowngradeSet(PackageSet): + + _operations = ["merge", "unmerge"] + + description = "Package set which contains all packages " + \ + "for which the highest visible ebuild version is lower than " + \ + "the currently installed version." + + def __init__(self, portdb=None, vardb=None): + super(DowngradeSet, self).__init__() + self._portdb = portdb + self._vardb = vardb + + def load(self): + atoms = [] + xmatch = self._portdb.xmatch + xmatch_level = "bestmatch-visible" + cp_list = self._vardb.cp_list + aux_get = self._vardb.aux_get + aux_keys = ["SLOT"] + for cp in self._vardb.cp_all(): + for cpv in cp_list(cp): + slot, = aux_get(cpv, aux_keys) + slot_atom = "%s:%s" % (cp, slot) + ebuild = xmatch(xmatch_level, slot_atom) + if not ebuild: + continue + ebuild_split = catpkgsplit(ebuild)[1:] + installed_split = catpkgsplit(cpv)[1:] + if pkgcmp(installed_split, ebuild_split) > 0: + atoms.append(slot_atom) + + self._setAtoms(atoms) + + def singleBuilder(cls, options, settings, trees): + return cls(portdb=trees["porttree"].dbapi, + vardb=trees["vartree"].dbapi) + + singleBuilder = classmethod(singleBuilder) + +class UnavailableSet(EverythingSet): + + _operations = ["unmerge"] + + description = "Package set which contains all installed " + \ + "packages for which there are no visible ebuilds " + \ + "corresponding to the same $CATEGORY/$PN:$SLOT." + + def __init__(self, vardb, metadatadb=None): + super(UnavailableSet, self).__init__(vardb) + self._metadatadb = metadatadb + + def _filter(self, atom): + return not self._metadatadb.match(atom) + + def singleBuilder(cls, options, settings, trees): + + metadatadb = options.get("metadata-source", "porttree") + if not metadatadb in trees: + raise SetConfigError(_("invalid value '%s' for option " + "metadata-source") % (metadatadb,)) + + return cls(trees["vartree"].dbapi, + metadatadb=trees[metadatadb].dbapi) + + singleBuilder = classmethod(singleBuilder) + +class CategorySet(PackageSet): + _operations = ["merge", "unmerge"] + + def __init__(self, category, dbapi, only_visible=True): + super(CategorySet, self).__init__() + self._db = dbapi + self._category = category + self._check = only_visible + if only_visible: + s="visible" + else: + s="all" + self.description = "Package set containing %s packages of category %s" % (s, self._category) + + def load(self): + myatoms = [] + for cp in self._db.cp_all(): + if catsplit(cp)[0] == self._category: + if (not self._check) or len(self._db.match(cp)) > 0: + myatoms.append(cp) + self._setAtoms(myatoms) + + def _builderGetRepository(cls, options, repositories): + repository = options.get("repository", "porttree") + if not repository in repositories: + raise SetConfigError(_("invalid repository class '%s'") % repository) + return repository + _builderGetRepository = classmethod(_builderGetRepository) + + def _builderGetVisible(cls, options): + return get_boolean(options, "only_visible", True) + _builderGetVisible = classmethod(_builderGetVisible) + + def singleBuilder(cls, options, settings, trees): + if not "category" in options: + raise SetConfigError(_("no category given")) + + category = options["category"] + if not category in settings.categories: + raise SetConfigError(_("invalid category name '%s'") % category) + + repository = cls._builderGetRepository(options, trees.keys()) + visible = cls._builderGetVisible(options) + + return CategorySet(category, dbapi=trees[repository].dbapi, only_visible=visible) + singleBuilder = classmethod(singleBuilder) + + def multiBuilder(cls, options, settings, trees): + rValue = {} + + if "categories" in options: + categories = options["categories"].split() + invalid = set(categories).difference(settings.categories) + if invalid: + raise SetConfigError(_("invalid categories: %s") % ", ".join(list(invalid))) + else: + categories = settings.categories + + repository = cls._builderGetRepository(options, trees.keys()) + visible = cls._builderGetVisible(options) + name_pattern = options.get("name_pattern", "$category/*") + + if not "$category" in name_pattern and not "${category}" in name_pattern: + raise SetConfigError(_("name_pattern doesn't include $category placeholder")) + + for cat in categories: + myset = CategorySet(cat, trees[repository].dbapi, only_visible=visible) + myname = name_pattern.replace("$category", cat) + myname = myname.replace("${category}", cat) + rValue[myname] = myset + return rValue + multiBuilder = classmethod(multiBuilder) + +class AgeSet(EverythingSet): + _operations = ["merge", "unmerge"] + + def __init__(self, vardb, mode="older", age=7): + super(AgeSet, self).__init__(vardb) + self._mode = mode + self._age = age + + def _filter(self, atom): + + cpv = self._db.match(atom)[0] + path = self._db.getpath(cpv, filename="COUNTER") + age = (time.time() - os.stat(path).st_mtime) / (3600 * 24) + if ((self._mode == "older" and age <= self._age) \ + or (self._mode == "newer" and age >= self._age)): + return False + else: + return True + + def singleBuilder(cls, options, settings, trees): + mode = options.get("mode", "older") + if str(mode).lower() not in ["newer", "older"]: + raise SetConfigError(_("invalid 'mode' value %s (use either 'newer' or 'older')") % mode) + try: + age = int(options.get("age", "7")) + except ValueError as e: + raise SetConfigError(_("value of option 'age' is not an integer")) + return AgeSet(vardb=trees["vartree"].dbapi, mode=mode, age=age) + + singleBuilder = classmethod(singleBuilder) + +class RebuiltBinaries(EverythingSet): + _operations = ('merge',) + _aux_keys = ('BUILD_TIME',) + + def __init__(self, vardb, bindb=None): + super(RebuiltBinaries, self).__init__(vardb, bindb=bindb) + self._bindb = bindb + + def _filter(self, atom): + cpv = self._db.match(atom)[0] + inst_build_time, = self._db.aux_get(cpv, self._aux_keys) + try: + bin_build_time, = self._bindb.aux_get(cpv, self._aux_keys) + except KeyError: + return False + return bool(bin_build_time and (inst_build_time != bin_build_time)) + + def singleBuilder(cls, options, settings, trees): + return RebuiltBinaries(trees["vartree"].dbapi, + bindb=trees["bintree"].dbapi) + + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/_sets/files.py b/pym/portage/_sets/files.py new file mode 100644 index 000000000..8116d07cd --- /dev/null +++ b/pym/portage/_sets/files.py @@ -0,0 +1,327 @@ +# Copyright 2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import re +from itertools import chain + +from portage import os +from portage import _encodings +from portage import _unicode_decode +from portage import _unicode_encode +from portage.util import grabfile, write_atomic, ensure_dirs, normalize_path +from portage.const import USER_CONFIG_PATH, WORLD_FILE, WORLD_SETS_FILE +from portage.localization import _ +from portage.locks import lockfile, unlockfile +from portage import portage_gid +from portage._sets.base import PackageSet, EditablePackageSet +from portage._sets import SetConfigError, SETPREFIX, get_boolean +from portage.env.loaders import ItemFileLoader, KeyListFileLoader +from portage.env.validators import ValidAtomValidator +from portage import cpv_getkey + +__all__ = ["StaticFileSet", "ConfigFileSet", "WorldSelectedSet"] + +class StaticFileSet(EditablePackageSet): + _operations = ["merge", "unmerge"] + _repopath_match = re.compile(r'.*\$\{repository:(?P<reponame>.+)\}.*') + _repopath_sub = re.compile(r'\$\{repository:(?P<reponame>.+)\}') + + def __init__(self, filename, greedy=False, dbapi=None): + super(StaticFileSet, self).__init__() + self._filename = filename + self._mtime = None + self.description = "Package set loaded from file %s" % self._filename + self.loader = ItemFileLoader(self._filename, self._validate) + if greedy and not dbapi: + self.errors.append(_("%s configured as greedy set, but no dbapi instance passed in constructor") % self._filename) + greedy = False + self.greedy = greedy + self.dbapi = dbapi + + metadata = grabfile(self._filename + ".metadata") + key = None + value = [] + for line in metadata: + line = line.strip() + if len(line) == 0 and key != None: + setattr(self, key, " ".join(value)) + key = None + elif line[-1] == ":" and key == None: + key = line[:-1].lower() + value = [] + elif key != None: + value.append(line) + else: + pass + else: + if key != None: + setattr(self, key, " ".join(value)) + + def _validate(self, atom): + return bool(atom[:1] == SETPREFIX or ValidAtomValidator(atom)) + + def write(self): + write_atomic(self._filename, "".join("%s\n" % (atom,) \ + for atom in sorted(chain(self._atoms, self._nonatoms)))) + + def load(self): + try: + mtime = os.stat(self._filename).st_mtime + except (OSError, IOError): + mtime = None + if (not self._loaded or self._mtime != mtime): + try: + data, errors = self.loader.load() + for fname in errors: + for e in errors[fname]: + self.errors.append(fname+": "+e) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + del e + data = {} + if self.greedy: + atoms = [] + for a in data: + matches = self.dbapi.match(a) + for cpv in matches: + atoms.append("%s:%s" % (cpv_getkey(cpv), + self.dbapi.aux_get(cpv, ["SLOT"])[0])) + # In addition to any installed slots, also try to pull + # in the latest new slot that may be available. + atoms.append(a) + else: + atoms = iter(data) + self._setAtoms(atoms) + self._mtime = mtime + + def singleBuilder(self, options, settings, trees): + if not "filename" in options: + raise SetConfigError(_("no filename specified")) + greedy = get_boolean(options, "greedy", False) + filename = options["filename"] + # look for repository path variables + match = self._repopath_match.match(filename) + if match: + try: + filename = self._repopath_sub.sub(trees["porttree"].dbapi.treemap[match.groupdict()["reponame"]], filename) + except KeyError: + raise SetConfigError(_("Could not find repository '%s'") % match.groupdict()["reponame"]) + return StaticFileSet(filename, greedy=greedy, dbapi=trees["vartree"].dbapi) + singleBuilder = classmethod(singleBuilder) + + def multiBuilder(self, options, settings, trees): + rValue = {} + directory = options.get("directory", + os.path.join(settings["PORTAGE_CONFIGROOT"], + USER_CONFIG_PATH, "sets")) + name_pattern = options.get("name_pattern", "${name}") + if not "$name" in name_pattern and not "${name}" in name_pattern: + raise SetConfigError(_("name_pattern doesn't include ${name} placeholder")) + greedy = get_boolean(options, "greedy", False) + # look for repository path variables + match = self._repopath_match.match(directory) + if match: + try: + directory = self._repopath_sub.sub(trees["porttree"].dbapi.treemap[match.groupdict()["reponame"]], directory) + except KeyError: + raise SetConfigError(_("Could not find repository '%s'") % match.groupdict()["reponame"]) + + try: + directory = _unicode_decode(directory, + encoding=_encodings['fs'], errors='strict') + # Now verify that we can also encode it. + _unicode_encode(directory, + encoding=_encodings['fs'], errors='strict') + except UnicodeError: + directory = _unicode_decode(directory, + encoding=_encodings['fs'], errors='replace') + raise SetConfigError( + _("Directory path contains invalid character(s) for encoding '%s': '%s'") \ + % (_encodings['fs'], directory)) + + if os.path.isdir(directory): + directory = normalize_path(directory) + + for parent, dirs, files in os.walk(directory): + try: + parent = _unicode_decode(parent, + encoding=_encodings['fs'], errors='strict') + except UnicodeDecodeError: + continue + for d in dirs[:]: + if d[:1] == '.': + dirs.remove(d) + for filename in files: + try: + filename = _unicode_decode(filename, + encoding=_encodings['fs'], errors='strict') + except UnicodeDecodeError: + continue + if filename[:1] == '.': + continue + if filename.endswith(".metadata"): + continue + filename = os.path.join(parent, + filename)[1 + len(directory):] + myname = name_pattern.replace("$name", filename) + myname = myname.replace("${name}", filename) + rValue[myname] = StaticFileSet( + os.path.join(directory, filename), + greedy=greedy, dbapi=trees["vartree"].dbapi) + return rValue + multiBuilder = classmethod(multiBuilder) + +class ConfigFileSet(PackageSet): + def __init__(self, filename): + super(ConfigFileSet, self).__init__() + self._filename = filename + self.description = "Package set generated from %s" % self._filename + self.loader = KeyListFileLoader(self._filename, ValidAtomValidator) + + def load(self): + data, errors = self.loader.load() + self._setAtoms(iter(data)) + + def singleBuilder(self, options, settings, trees): + if not "filename" in options: + raise SetConfigError(_("no filename specified")) + return ConfigFileSet(options["filename"]) + singleBuilder = classmethod(singleBuilder) + + def multiBuilder(self, options, settings, trees): + rValue = {} + directory = options.get("directory", + os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH)) + name_pattern = options.get("name_pattern", "sets/package_$suffix") + if not "$suffix" in name_pattern and not "${suffix}" in name_pattern: + raise SetConfigError(_("name_pattern doesn't include $suffix placeholder")) + for suffix in ["keywords", "use", "mask", "unmask"]: + myname = name_pattern.replace("$suffix", suffix) + myname = myname.replace("${suffix}", suffix) + rValue[myname] = ConfigFileSet(os.path.join(directory, "package."+suffix)) + return rValue + multiBuilder = classmethod(multiBuilder) + +class WorldSelectedSet(EditablePackageSet): + description = "Set of packages that were directly installed by the user" + + def __init__(self, root): + super(WorldSelectedSet, self).__init__() + # most attributes exist twice as atoms and non-atoms are stored in + # separate files + self._lock = None + self._filename = os.path.join(os.sep, root, WORLD_FILE) + self.loader = ItemFileLoader(self._filename, self._validate) + self._mtime = None + + self._filename2 = os.path.join(os.sep, root, WORLD_SETS_FILE) + self.loader2 = ItemFileLoader(self._filename2, self._validate2) + self._mtime2 = None + + def _validate(self, atom): + return ValidAtomValidator(atom) + + def _validate2(self, setname): + return setname.startswith(SETPREFIX) + + def write(self): + write_atomic(self._filename, + "".join(sorted("%s\n" % x for x in self._atoms))) + write_atomic(self._filename2, "\n".join(sorted(self._nonatoms))+"\n") + + def load(self): + atoms = [] + nonatoms = [] + atoms_changed = False + # load atoms and non-atoms from different files so the worldfile is + # backwards-compatible with older versions and other PMs, even though + # it's supposed to be private state data :/ + try: + mtime = os.stat(self._filename).st_mtime + except (OSError, IOError): + mtime = None + if (not self._loaded or self._mtime != mtime): + try: + data, errors = self.loader.load() + for fname in errors: + for e in errors[fname]: + self.errors.append(fname+": "+e) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + del e + data = {} + atoms = list(data) + self._mtime = mtime + atoms_changed = True + else: + atoms.extend(self._atoms) + try: + mtime = os.stat(self._filename2).st_mtime + except (OSError, IOError): + mtime = None + if (not self._loaded or self._mtime2 != mtime): + try: + data, errors = self.loader2.load() + for fname in errors: + for e in errors[fname]: + self.errors.append(fname+": "+e) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + del e + data = {} + nonatoms = list(data) + self._mtime2 = mtime + atoms_changed = True + else: + nonatoms.extend(self._nonatoms) + if atoms_changed: + self._setAtoms(atoms+nonatoms) + + def _ensure_dirs(self): + ensure_dirs(os.path.dirname(self._filename), gid=portage_gid, mode=0o2750, mask=0o2) + + def lock(self): + self._ensure_dirs() + self._lock = lockfile(self._filename, wantnewlockfile=1) + + def unlock(self): + unlockfile(self._lock) + self._lock = None + + def cleanPackage(self, vardb, cpv): + ''' + Before calling this function you should call lock and load. + After calling this function you should call unlock. + ''' + if not self._lock: + raise AssertionError('cleanPackage needs the set to be locked') + + worldlist = list(self._atoms) + mykey = cpv_getkey(cpv) + newworldlist = [] + for x in worldlist: + if x.cp == mykey: + matches = vardb.match(x, use_cache=0) + if not matches: + #zap our world entry + pass + elif len(matches) == 1 and matches[0] == cpv: + #zap our world entry + pass + else: + #others are around; keep it. + newworldlist.append(x) + else: + #this doesn't match the package we're unmerging; keep it. + newworldlist.append(x) + + newworldlist.extend(self._nonatoms) + self.replace(newworldlist) + + def singleBuilder(self, options, settings, trees): + return WorldSelectedSet(settings["EROOT"]) + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/_sets/libs.py b/pym/portage/_sets/libs.py new file mode 100644 index 000000000..20347e452 --- /dev/null +++ b/pym/portage/_sets/libs.py @@ -0,0 +1,91 @@ +# Copyright 2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + +from portage.localization import _ +from portage._sets.base import PackageSet +from portage._sets import get_boolean +from portage.versions import catpkgsplit +import portage + +class LibraryConsumerSet(PackageSet): + _operations = ["merge", "unmerge"] + + def __init__(self, vardbapi, debug=False): + super(LibraryConsumerSet, self).__init__() + self.dbapi = vardbapi + self.debug = debug + + def mapPathsToAtoms(self, paths): + rValue = set() + for link, p in self.dbapi._owners.iter_owners(paths): + cat, pn = catpkgsplit(link.mycpv)[:2] + slot = self.dbapi.aux_get(link.mycpv, ["SLOT"])[0] + rValue.add("%s/%s:%s" % (cat, pn, slot)) + return rValue + +class LibraryFileConsumerSet(LibraryConsumerSet): + + """ + Note: This does not detect libtool archive (*.la) files that consume the + specified files (revdep-rebuild is able to detect them). + """ + + description = "Package set which contains all packages " + \ + "that consume the specified library file(s)." + + def __init__(self, vardbapi, files, **kargs): + super(LibraryFileConsumerSet, self).__init__(vardbapi, **kargs) + self.files = files + + def load(self): + consumers = set() + for lib in self.files: + consumers.update(self.dbapi._linkmap.findConsumers(lib)) + + if not consumers: + return + self._setAtoms(self.mapPathsToAtoms(consumers)) + + def singleBuilder(cls, options, settings, trees): + files = tuple(portage.util.shlex_split(options.get("files", ""))) + if not files: + raise SetConfigError(_("no files given")) + debug = get_boolean(options, "debug", False) + return LibraryFileConsumerSet(trees["vartree"].dbapi, + files, debug=debug) + singleBuilder = classmethod(singleBuilder) + +class PreservedLibraryConsumerSet(LibraryConsumerSet): + def load(self): + reg = self.dbapi._plib_registry + if reg is None: + # preserve-libs is entirely disabled + return + consumers = set() + if reg: + plib_dict = reg.getPreservedLibs() + for libs in plib_dict.values(): + for lib in libs: + if self.debug: + print(lib) + for x in sorted(self.dbapi._linkmap.findConsumers(lib)): + print(" ", x) + print("-"*40) + consumers.update(self.dbapi._linkmap.findConsumers(lib)) + # Don't rebuild packages just because they contain preserved + # libs that happen to be consumers of other preserved libs. + for libs in plib_dict.values(): + consumers.difference_update(libs) + else: + return + if not consumers: + return + self._setAtoms(self.mapPathsToAtoms(consumers)) + + def singleBuilder(cls, options, settings, trees): + debug = get_boolean(options, "debug", False) + return PreservedLibraryConsumerSet(trees["vartree"].dbapi, + debug=debug) + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/_sets/profiles.py b/pym/portage/_sets/profiles.py new file mode 100644 index 000000000..e47f08db3 --- /dev/null +++ b/pym/portage/_sets/profiles.py @@ -0,0 +1,53 @@ +# Copyright 2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import logging + +from portage import os +from portage.util import grabfile_package, stack_lists +from portage._sets.base import PackageSet +from portage._sets import get_boolean +from portage.util import writemsg_level + +__all__ = ["PackagesSystemSet"] + +class PackagesSystemSet(PackageSet): + _operations = ["merge"] + + def __init__(self, profile_paths, debug=False): + super(PackagesSystemSet, self).__init__() + self._profile_paths = profile_paths + self._debug = debug + if profile_paths: + description = self._profile_paths[-1] + if description == "/etc/portage/profile" and \ + len(self._profile_paths) > 1: + description = self._profile_paths[-2] + else: + description = None + self.description = "System packages for profile %s" % description + + def load(self): + debug = self._debug + if debug: + writemsg_level("\nPackagesSystemSet: profile paths: %s\n" % \ + (self._profile_paths,), level=logging.DEBUG, noiselevel=-1) + + mylist = [grabfile_package(os.path.join(x, "packages")) for x in self._profile_paths] + + if debug: + writemsg_level("\nPackagesSystemSet: raw packages: %s\n" % \ + (mylist,), level=logging.DEBUG, noiselevel=-1) + + mylist = stack_lists(mylist, incremental=1) + + if debug: + writemsg_level("\nPackagesSystemSet: stacked packages: %s\n" % \ + (mylist,), level=logging.DEBUG, noiselevel=-1) + + self._setAtoms([x[1:] for x in mylist if x[0] == "*"]) + + def singleBuilder(self, options, settings, trees): + debug = get_boolean(options, "debug", False) + return PackagesSystemSet(settings.profiles, debug=debug) + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/_sets/security.py b/pym/portage/_sets/security.py new file mode 100644 index 000000000..2d8fcf667 --- /dev/null +++ b/pym/portage/_sets/security.py @@ -0,0 +1,86 @@ +# Copyright 2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage.glsa as glsa +from portage._sets.base import PackageSet +from portage.versions import catpkgsplit, pkgcmp +from portage._sets import get_boolean + +__all__ = ["SecuritySet", "NewGlsaSet", "NewAffectedSet", "AffectedSet"] + +class SecuritySet(PackageSet): + _operations = ["merge"] + _skip_applied = False + + description = "package set that includes all packages possibly affected by a GLSA" + + def __init__(self, settings, vardbapi, portdbapi, least_change=True): + super(SecuritySet, self).__init__() + self._settings = settings + self._vardbapi = vardbapi + self._portdbapi = portdbapi + self._least_change = least_change + + def getGlsaList(self, skip_applied): + glsaindexlist = glsa.get_glsa_list(self._settings) + if skip_applied: + applied_list = glsa.get_applied_glsas(self._settings) + glsaindexlist = set(glsaindexlist).difference(applied_list) + glsaindexlist = list(glsaindexlist) + glsaindexlist.sort() + return glsaindexlist + + def load(self): + glsaindexlist = self.getGlsaList(self._skip_applied) + atomlist = [] + for glsaid in glsaindexlist: + myglsa = glsa.Glsa(glsaid, self._settings, self._vardbapi, self._portdbapi) + #print glsaid, myglsa.isVulnerable(), myglsa.isApplied(), myglsa.getMergeList() + if self.useGlsa(myglsa): + atomlist += ["="+x for x in myglsa.getMergeList(least_change=self._least_change)] + self._setAtoms(self._reduce(atomlist)) + + def _reduce(self, atomlist): + mydict = {} + for atom in atomlist[:]: + cpv = self._portdbapi.xmatch("match-all", atom)[0] + slot = self._portdbapi.aux_get(cpv, ["SLOT"])[0] + cps = "/".join(catpkgsplit(cpv)[0:2]) + ":" + slot + if not cps in mydict: + mydict[cps] = (atom, cpv) + else: + other_cpv = mydict[cps][1] + if pkgcmp(catpkgsplit(cpv)[1:], catpkgsplit(other_cpv)[1:]) > 0: + atomlist.remove(mydict[cps][0]) + mydict[cps] = (atom, cpv) + return atomlist + + def useGlsa(self, myglsa): + return True + + def updateAppliedList(self): + glsaindexlist = self.getGlsaList(True) + applied_list = glsa.get_applied_glsas(self._settings) + for glsaid in glsaindexlist: + myglsa = glsa.Glsa(glsaid, self._settings, self._vardbapi, self._portdbapi) + if not myglsa.isVulnerable() and not myglsa.nr in applied_list: + myglsa.inject() + + def singleBuilder(cls, options, settings, trees): + least_change = not get_boolean(options, "use_emerge_resolver", False) + return cls(settings, trees["vartree"].dbapi, trees["porttree"].dbapi, least_change=least_change) + singleBuilder = classmethod(singleBuilder) + +class NewGlsaSet(SecuritySet): + _skip_applied = True + description = "Package set that includes all packages possibly affected by an unapplied GLSA" + +class AffectedSet(SecuritySet): + description = "Package set that includes all packages affected by an unapplied GLSA" + + def useGlsa(self, myglsa): + return myglsa.isVulnerable() + +class NewAffectedSet(AffectedSet): + _skip_applied = True + description = "Package set that includes all packages affected by an unapplied GLSA" diff --git a/pym/portage/_sets/shell.py b/pym/portage/_sets/shell.py new file mode 100644 index 000000000..2c95845c8 --- /dev/null +++ b/pym/portage/_sets/shell.py @@ -0,0 +1,44 @@ +# Copyright 2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import subprocess + +from portage import os +from portage import _unicode_decode +from portage._sets.base import PackageSet +from portage._sets import SetConfigError + +__all__ = ["CommandOutputSet"] + +class CommandOutputSet(PackageSet): + """This class creates a PackageSet from the output of a shell command. + The shell command should produce one atom per line, that is: + + >>> atom1 + atom2 + ... + atomN + + Args: + name: A string that identifies the set. + command: A string or sequence identifying the command to run + (see the subprocess.Popen documentaion for the format) + """ + _operations = ["merge", "unmerge"] + + def __init__(self, command): + super(CommandOutputSet, self).__init__() + self._command = command + self.description = "Package set generated from output of '%s'" % self._command + + def load(self): + pipe = subprocess.Popen(self._command, stdout=subprocess.PIPE, shell=True) + stdout, stderr = pipe.communicate() + if pipe.wait() == os.EX_OK: + self._setAtoms(_unicode_decode(stdout).splitlines()) + + def singleBuilder(self, options, settings, trees): + if not "command" in options: + raise SetConfigError("no command specified") + return CommandOutputSet(options["command"]) + singleBuilder = classmethod(singleBuilder) |