# Copyright 1998-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ from __future__ import print_function __all__ = [ "vardbapi", "vartree", "dblink"] + \ ["write_contents", "tar_contents"] import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.checksum:_perform_md5_merge@perform_md5', 'portage.dep:dep_getkey,isjustname,match_from_list,' + \ 'use_reduce,paren_reduce,_slot_re', 'portage.elog:elog_process', 'portage.elog.filtering:filter_mergephases,filter_unmergephases', 'portage.locks:lockdir,unlockdir', 'portage.output:bold,colorize', 'portage.update:fixdbentries', 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ 'grabfile,grabdict,normalize_path,new_protect_filename,getlibpaths', 'portage.versions:best,catpkgsplit,catsplit,pkgcmp,_pkgsplit@pkgsplit', ) from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH from portage.data import portage_gid, portage_uid, secpass from portage.dbapi import dbapi from portage.exception import CommandNotFound, \ InvalidData, InvalidPackageName, \ FileNotFound, PermissionDenied, UnsupportedAPIException from portage.localization import _ from portage import listdir, dep_expand, digraph, flatten, key_expand, \ doebuild_environment, doebuild, env_update, prepare_build_dirs, \ abssymlink, movefile, _movefile, bsd_chflags, cpv_getkey # This is a special version of the os module, wrapped for unicode support. from portage import os from portage import _encodings from portage import _os_merge from portage import _selinux_merge from portage import _unicode_decode from portage import _unicode_encode from portage.cache.mappings import slot_dict_class import codecs import re, shutil, stat, errno, copy, subprocess import logging import os as _os import sys import warnings try: import cPickle as pickle except ImportError: import pickle if sys.hexversion >= 0x3000000: basestring = str long = int class vardbapi(dbapi): _excluded_dirs = ["CVS", "lost+found"] _excluded_dirs = [re.escape(x) for x in _excluded_dirs] _excluded_dirs = re.compile(r'^(\..*|-MERGING-.*|' + \ "|".join(_excluded_dirs) + r')$') _aux_cache_version = "1" _owners_cache_version = "1" # Number of uncached packages to trigger cache update, since # it's wasteful to update it for every vdb change. _aux_cache_threshold = 5 _aux_cache_keys_re = re.compile(r'^NEEDED\..*$') _aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$') def __init__(self, root, categories=None, settings=None, vartree=None): """ The categories parameter is unused since the dbapi class now has a categories property that is generated from the available packages. """ self.root = _unicode_decode(root, encoding=_encodings['content'], errors='strict') # Used by emerge to check whether any packages # have been added or removed. self._pkgs_changed = False #cache for category directory mtimes self.mtdircache = {} #cache for dependency checks self.matchcache = {} #cache for cp_list results self.cpcache = {} self.blockers = None if settings is None: from portage import settings self.settings = settings if vartree is None: from portage import db vartree = db[root]["vartree"] self.vartree = vartree self._aux_cache_keys = set( ["CHOST", "COUNTER", "DEPEND", "DESCRIPTION", "EAPI", "HOMEPAGE", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", "RESTRICT" , "SLOT", "USE"]) self._aux_cache_obj = None self._aux_cache_filename = os.path.join(self.root, CACHE_PATH, "vdb_metadata.pickle") self._counter_path = os.path.join(root, CACHE_PATH, "counter") self._owners = self._owners_db(self) def getpath(self, mykey, filename=None): # This is an optimized hotspot, so don't use unicode-wrapped # os module and don't use os.path.join(). rValue = self.root + _os.sep + VDB_PATH + _os.sep + mykey if filename is not None: # If filename is always relative, we can do just # rValue += _os.sep + filename rValue = _os.path.join(rValue, filename) return rValue def cpv_exists(self, mykey): "Tells us whether an actual ebuild exists on disk (no masking)" return os.path.exists(self.getpath(mykey)) def cpv_counter(self, mycpv): "This method will grab the COUNTER. Returns a counter value." try: return long(self.aux_get(mycpv, ["COUNTER"])[0]) except (KeyError, ValueError): pass writemsg_level(_("portage: COUNTER for %s was corrupted; " \ "resetting to value of 0\n") % (mycpv,), level=logging.ERROR, noiselevel=-1) return 0 def cpv_inject(self, mycpv): "injects a real package into our on-disk database; assumes mycpv is valid and doesn't already exist" os.makedirs(self.getpath(mycpv)) counter = self.counter_tick(self.root, mycpv=mycpv) # write local package counter so that emerge clean does the right thing write_atomic(self.getpath(mycpv, filename="COUNTER"), str(counter)) def isInjected(self, mycpv): if self.cpv_exists(mycpv): if os.path.exists(self.getpath(mycpv, filename="INJECTED")): return True if not os.path.exists(self.getpath(mycpv, filename="CONTENTS")): return True return False def move_ent(self, mylist): origcp = mylist[1] newcp = mylist[2] # sanity check for atom in (origcp, newcp): if not isjustname(atom): raise InvalidPackageName(str(atom)) origmatches = self.match(origcp, use_cache=0) moves = 0 if not origmatches: return moves for mycpv in origmatches: mycpv_cp = cpv_getkey(mycpv) if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) mynewcat = catsplit(newcp)[0] origpath = self.getpath(mycpv) if not os.path.exists(origpath): continue moves += 1 if not os.path.exists(self.getpath(mynewcat)): #create the directory os.makedirs(self.getpath(mynewcat)) newpath = self.getpath(mynewcpv) if os.path.exists(newpath): #dest already exists; keep this puppy where it is. continue _movefile(origpath, newpath, mysettings=self.settings) # We need to rename the ebuild now. old_pf = catsplit(mycpv)[1] new_pf = catsplit(mynewcpv)[1] if new_pf != old_pf: try: os.rename(os.path.join(newpath, old_pf + ".ebuild"), os.path.join(newpath, new_pf + ".ebuild")) except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") fixdbentries([mylist], newpath) return moves def cp_list(self, mycp, use_cache=1): mysplit=catsplit(mycp) if mysplit[0] == '*': mysplit[0] = mysplit[0][1:] try: mystat = os.stat(self.getpath(mysplit[0]))[stat.ST_MTIME] except OSError: mystat = 0 if use_cache and mycp in self.cpcache: cpc = self.cpcache[mycp] if cpc[0] == mystat: return cpc[1][:] cat_dir = self.getpath(mysplit[0]) try: dir_list = os.listdir(cat_dir) except EnvironmentError as e: if e.errno == PermissionDenied.errno: raise PermissionDenied(cat_dir) del e dir_list = [] returnme = [] for x in dir_list: if self._excluded_dirs.match(x) is not None: continue ps = pkgsplit(x) if not ps: self.invalidentry(os.path.join(self.getpath(mysplit[0]), x)) continue if len(mysplit) > 1: if ps[0] == mysplit[1]: returnme.append(mysplit[0]+"/"+x) self._cpv_sort_ascending(returnme) if use_cache: self.cpcache[mycp] = [mystat, returnme[:]] elif mycp in self.cpcache: del self.cpcache[mycp] return returnme def cpv_all(self, use_cache=1): """ Set use_cache=0 to bypass the portage.cachedir() cache in cases when the accuracy of mtime staleness checks should not be trusted (generally this is only necessary in critical sections that involve merge or unmerge of packages). """ returnme = [] basepath = os.path.join(self.root, VDB_PATH) + os.path.sep if use_cache: from portage import listdir else: def listdir(p, **kwargs): try: return [x for x in os.listdir(p) \ if os.path.isdir(os.path.join(p, x))] except EnvironmentError as e: if e.errno == PermissionDenied.errno: raise PermissionDenied(p) del e return [] for x in listdir(basepath, EmptyOnError=1, ignorecvs=1, dirsonly=1): if self._excluded_dirs.match(x) is not None: continue if not self._category_re.match(x): continue for y in listdir(basepath + x, EmptyOnError=1, dirsonly=1): if self._excluded_dirs.match(y) is not None: continue subpath = x + "/" + y # -MERGING- should never be a cpv, nor should files. try: if catpkgsplit(subpath) is None: self.invalidentry(self.getpath(subpath)) continue except InvalidData: self.invalidentry(self.getpath(subpath)) continue returnme.append(subpath) return returnme def cp_all(self, use_cache=1): mylist = self.cpv_all(use_cache=use_cache) d={} for y in mylist: if y[0] == '*': y = y[1:] try: mysplit = catpkgsplit(y) except InvalidData: self.invalidentry(self.getpath(y)) continue if not mysplit: self.invalidentry(self.getpath(y)) continue d[mysplit[0]+"/"+mysplit[1]] = None return list(d) def checkblockers(self, origdep): pass def _clear_cache(self): self.mtdircache.clear() self.matchcache.clear() self.cpcache.clear() self._aux_cache_obj = None def _add(self, pkg_dblink): self._pkgs_changed = True self._clear_pkg_cache(pkg_dblink) def _remove(self, pkg_dblink): self._pkgs_changed = True self._clear_pkg_cache(pkg_dblink) def _clear_pkg_cache(self, pkg_dblink): # Due to 1 second mtime granularity in = self._aux_cache_threshold and \ secpass >= 2: self._owners.populate() # index any unindexed contents valid_nodes = set(self.cpv_all()) for cpv in list(self._aux_cache["packages"]): if cpv not in valid_nodes: del self._aux_cache["packages"][cpv] del self._aux_cache["modified"] try: f = atomic_ofstream(self._aux_cache_filename, 'wb') pickle.dump(self._aux_cache, f, protocol=2) f.close() apply_secpass_permissions( self._aux_cache_filename, gid=portage_gid, mode=0o644) except (IOError, OSError) as e: pass self._aux_cache["modified"] = set() @property def _aux_cache(self): if self._aux_cache_obj is None: self._aux_cache_init() return self._aux_cache_obj def _aux_cache_init(self): aux_cache = None open_kwargs = {} if sys.hexversion >= 0x3000000: # Buffered io triggers extreme performance issues in # Unpickler.load() (problem observed with python-3.0.1). # Unfortunately, performance is still poor relative to # python-2.x, but buffering makes it much worse. open_kwargs["buffering"] = 0 try: f = open(_unicode_encode(self._aux_cache_filename, encoding=_encodings['fs'], errors='strict'), mode='rb', **open_kwargs) mypickle = pickle.Unpickler(f) try: mypickle.find_global = None except AttributeError: # TODO: If py3k, override Unpickler.find_class(). pass aux_cache = mypickle.load() f.close() del f except (IOError, OSError, EOFError, ValueError, pickle.UnpicklingError) as e: if isinstance(e, pickle.UnpicklingError): writemsg(_("!!! Error loading '%s': %s\n") % \ (self._aux_cache_filename, str(e)), noiselevel=-1) del e if not aux_cache or \ not isinstance(aux_cache, dict) or \ aux_cache.get("version") != self._aux_cache_version or \ not aux_cache.get("packages"): aux_cache = {"version": self._aux_cache_version} aux_cache["packages"] = {} owners = aux_cache.get("owners") if owners is not None: if not isinstance(owners, dict): owners = None elif "version" not in owners: owners = None elif owners["version"] != self._owners_cache_version: owners = None elif "base_names" not in owners: owners = None elif not isinstance(owners["base_names"], dict): owners = None if owners is None: owners = { "base_names" : {}, "version" : self._owners_cache_version } aux_cache["owners"] = owners aux_cache["modified"] = set() self._aux_cache_obj = aux_cache def aux_get(self, mycpv, wants): """This automatically caches selected keys that are frequently needed by emerge for dependency calculations. The cached metadata is considered valid if the mtime of the package directory has not changed since the data was cached. The cache is stored in a pickled dict object with the following format: {version:"1", "packages":{cpv1:(mtime,{k1,v1, k2,v2, ...}), cpv2...}} If an error occurs while loading the cache pickle or the version is unrecognized, the cache will simple be recreated from scratch (it is completely disposable). """ cache_these_wants = self._aux_cache_keys.intersection(wants) for x in wants: if self._aux_cache_keys_re.match(x) is not None: cache_these_wants.add(x) if not cache_these_wants: return self._aux_get(mycpv, wants) cache_these = set(self._aux_cache_keys) cache_these.update(cache_these_wants) mydir = self.getpath(mycpv) mydir_stat = None try: mydir_stat = os.stat(mydir) except OSError as e: if e.errno != errno.ENOENT: raise raise KeyError(mycpv) mydir_mtime = long(mydir_stat.st_mtime) pkg_data = self._aux_cache["packages"].get(mycpv) pull_me = cache_these.union(wants) mydata = {"_mtime_" : mydir_mtime} cache_valid = False cache_incomplete = False cache_mtime = None metadata = None if pkg_data is not None: if not isinstance(pkg_data, tuple) or len(pkg_data) != 2: pkg_data = None else: cache_mtime, metadata = pkg_data if not isinstance(cache_mtime, (long, int)) or \ not isinstance(metadata, dict): pkg_data = None if pkg_data: cache_mtime, metadata = pkg_data cache_valid = cache_mtime == mydir_mtime if cache_valid: # Migrate old metadata to unicode. for k, v in metadata.items(): metadata[k] = _unicode_decode(v, encoding=_encodings['repo.content'], errors='replace') mydata.update(metadata) pull_me.difference_update(mydata) if pull_me: # pull any needed data and cache it aux_keys = list(pull_me) for k, v in zip(aux_keys, self._aux_get(mycpv, aux_keys, st=mydir_stat)): mydata[k] = v if not cache_valid or cache_these.difference(metadata): cache_data = {} if cache_valid and metadata: cache_data.update(metadata) for aux_key in cache_these: cache_data[aux_key] = mydata[aux_key] self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data) self._aux_cache["modified"].add(mycpv) if _slot_re.match(mydata['SLOT']) is None: # Empty or invalid slot triggers InvalidAtom exceptions when # generating slot atoms for packages, so translate it to '0' here. mydata['SLOT'] = _unicode_decode('0') return [mydata[x] for x in wants] def _aux_get(self, mycpv, wants, st=None): mydir = self.getpath(mycpv) if st is None: try: st = os.stat(mydir) except OSError as e: if e.errno == errno.ENOENT: raise KeyError(mycpv) elif e.errno == PermissionDenied.errno: raise PermissionDenied(mydir) else: raise if not stat.S_ISDIR(st.st_mode): raise KeyError(mycpv) results = [] for x in wants: if x == "_mtime_": results.append(long(st.st_mtime)) continue try: myf = codecs.open( _unicode_encode(os.path.join(mydir, x), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') try: myd = myf.read() finally: myf.close() # Preserve \n for metadata that is known to # contain multiple lines. if self._aux_multi_line_re.match(x) is None: myd = " ".join(myd.split()) except IOError: myd = _unicode_decode('') if x == "EAPI" and not myd: results.append(_unicode_decode('0')) else: results.append(myd) return results def aux_update(self, cpv, values): cat, pkg = catsplit(cpv) mylink = dblink(cat, pkg, self.root, self.settings, treetype="vartree", vartree=self.vartree) if not mylink.exists(): raise KeyError(cpv) for k, v in values.items(): if v: mylink.setfile(k, v) else: try: os.unlink(os.path.join(self.getpath(cpv), k)) except EnvironmentError: pass def counter_tick(self, myroot, mycpv=None): return self.counter_tick_core(myroot, incrementing=1, mycpv=mycpv) def get_counter_tick_core(self, myroot, mycpv=None): """ Use this method to retrieve the counter instead of having to trust the value of a global counter file that can lead to invalid COUNTER generation. When cache is valid, the package COUNTER files are not read and we rely on the timestamp of the package directory to validate cache. The stat calls should only take a short time, so performance is sufficient without having to rely on a potentially corrupt global counter file. The global counter file located at $CACHE_PATH/counter serves to record the counter of the last installed package and it also corresponds to the total number of installation actions that have occurred in the history of this package database. """ cp_list = self.cp_list max_counter = 0 for cp in self.cp_all(): for cpv in cp_list(cp): try: counter = int(self.aux_get(cpv, ["COUNTER"])[0]) except (KeyError, OverflowError, ValueError): continue if counter > max_counter: max_counter = counter new_vdb = False counter = -1 try: cfile = codecs.open( _unicode_encode(self._counter_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') except EnvironmentError as e: new_vdb = not bool(self.cpv_all()) if not new_vdb: writemsg(_("!!! Unable to read COUNTER file: '%s'\n") % \ self._counter_path, noiselevel=-1) writemsg("!!! %s\n" % str(e), noiselevel=-1) del e else: try: try: counter = long(cfile.readline().strip()) finally: cfile.close() except (OverflowError, ValueError) as e: writemsg(_("!!! COUNTER file is corrupt: '%s'\n") % \ self._counter_path, noiselevel=-1) writemsg("!!! %s\n" % str(e), noiselevel=-1) del e # We must ensure that we return a counter # value that is at least as large as the # highest one from the installed packages, # since having a corrupt value that is too low # can trigger incorrect AUTOCLEAN behavior due # to newly installed packages having lower # COUNTERs than the previous version in the # same slot. if counter > max_counter: max_counter = counter if counter < 0 and not new_vdb: writemsg(_("!!! Initializing COUNTER to " \ "value of %d\n") % max_counter, noiselevel=-1) return max_counter + 1 def counter_tick_core(self, myroot, incrementing=1, mycpv=None): "This method will grab the next COUNTER value and record it back to the global file. Returns new counter value." counter = self.get_counter_tick_core(myroot, mycpv=mycpv) - 1 if incrementing: #increment counter counter += 1 # update new global counter file write_atomic(self._counter_path, str(counter)) return counter def _dblink(self, cpv): category, pf = catsplit(cpv) return dblink(category, pf, self.root, self.settings, vartree=self.vartree, treetype="vartree") def removeFromContents(self, pkg, paths, relative_paths=True): """ @param pkg: cpv for an installed package @type pkg: string @param paths: paths of files to remove from contents @type paths: iterable """ if not hasattr(pkg, "getcontents"): pkg = self._dblink(pkg) root = self.root root_len = len(root) - 1 new_contents = pkg.getcontents().copy() removed = 0 for filename in paths: filename = _unicode_decode(filename, encoding=_encodings['content'], errors='strict') filename = normalize_path(filename) if relative_paths: relative_filename = filename else: relative_filename = filename[root_len:] contents_key = pkg._match_contents(relative_filename, root) if contents_key: del new_contents[contents_key] removed += 1 if removed: f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) write_contents(new_contents, root, f) f.close() pkg._clear_contents_cache() class _owners_cache(object): """ This class maintains an hash table that serves to index package contents by mapping the basename of file to a list of possible packages that own it. This is used to optimize owner lookups by narrowing the search down to a smaller number of packages. """ try: from hashlib import md5 as _new_hash except ImportError: from md5 import new as _new_hash _hash_bits = 16 _hex_chars = int(_hash_bits / 4) def __init__(self, vardb): self._vardb = vardb def add(self, cpv): root_len = len(self._vardb.root) contents = self._vardb._dblink(cpv).getcontents() pkg_hash = self._hash_pkg(cpv) if not contents: # Empty path is a code used to represent empty contents. self._add_path("", pkg_hash) for x in contents: self._add_path(x[root_len:], pkg_hash) self._vardb._aux_cache["modified"].add(cpv) def _add_path(self, path, pkg_hash): """ Empty path is a code that represents empty contents. """ if path: name = os.path.basename(path.rstrip(os.path.sep)) if not name: return else: name = path name_hash = self._hash_str(name) base_names = self._vardb._aux_cache["owners"]["base_names"] pkgs = base_names.get(name_hash) if pkgs is None: pkgs = {} base_names[name_hash] = pkgs pkgs[pkg_hash] = None def _hash_str(self, s): h = self._new_hash() # Always use a constant utf_8 encoding here, since # the "default" encoding can change. h.update(_unicode_encode(s, encoding=_encodings['repo.content'], errors='backslashreplace')) h = h.hexdigest() h = h[-self._hex_chars:] h = int(h, 16) return h def _hash_pkg(self, cpv): counter, mtime = self._vardb.aux_get( cpv, ["COUNTER", "_mtime_"]) try: counter = int(counter) except ValueError: counter = 0 return (cpv, counter, mtime) class _owners_db(object): def __init__(self, vardb): self._vardb = vardb def populate(self): self._populate() def _populate(self): owners_cache = vardbapi._owners_cache(self._vardb) cached_hashes = set() base_names = self._vardb._aux_cache["owners"]["base_names"] # Take inventory of all cached package hashes. for name, hash_values in list(base_names.items()): if not isinstance(hash_values, dict): del base_names[name] continue cached_hashes.update(hash_values) # Create sets of valid package hashes and uncached packages. uncached_pkgs = set() hash_pkg = owners_cache._hash_pkg valid_pkg_hashes = set() for cpv in self._vardb.cpv_all(): hash_value = hash_pkg(cpv) valid_pkg_hashes.add(hash_value) if hash_value not in cached_hashes: uncached_pkgs.add(cpv) # Cache any missing packages. for cpv in uncached_pkgs: owners_cache.add(cpv) # Delete any stale cache. stale_hashes = cached_hashes.difference(valid_pkg_hashes) if stale_hashes: for base_name_hash, bucket in list(base_names.items()): for hash_value in stale_hashes.intersection(bucket): del bucket[hash_value] if not bucket: del base_names[base_name_hash] return owners_cache def get_owners(self, path_iter): """ @return the owners as a dblink -> set(files) mapping. """ owners = {} for owner, f in self.iter_owners(path_iter): owned_files = owners.get(owner) if owned_files is None: owned_files = set() owners[owner] = owned_files owned_files.add(f) return owners def getFileOwnerMap(self, path_iter): owners = self.get_owners(path_iter) file_owners = {} for pkg_dblink, files in owners.items(): for f in files: owner_set = file_owners.get(f) if owner_set is None: owner_set = set() file_owners[f] = owner_set owner_set.add(pkg_dblink) return file_owners def iter_owners(self, path_iter): """ Iterate over tuples of (dblink, path). In order to avoid consuming too many resources for too much time, resources are only allocated for the duration of a given iter_owners() call. Therefore, to maximize reuse of resources when searching for multiple files, it's best to search for them all in a single call. """ owners_cache = self._populate() vardb = self._vardb root = vardb.root hash_pkg = owners_cache._hash_pkg hash_str = owners_cache._hash_str base_names = self._vardb._aux_cache["owners"]["base_names"] dblink_cache = {} def dblink(cpv): x = dblink_cache.get(cpv) if x is None: x = self._vardb._dblink(cpv) dblink_cache[cpv] = x return x for path in path_iter: is_basename = os.sep != path[:1] if is_basename: name = path else: name = os.path.basename(path.rstrip(os.path.sep)) if not name: continue name_hash = hash_str(name) pkgs = base_names.get(name_hash) if pkgs is not None: for hash_value in pkgs: if not isinstance(hash_value, tuple) or \ len(hash_value) != 3: continue cpv, counter, mtime = hash_value if not isinstance(cpv, basestring): continue try: current_hash = hash_pkg(cpv) except KeyError: continue if current_hash != hash_value: continue if is_basename: for p in dblink(cpv).getcontents(): if os.path.basename(p) == name: yield dblink(cpv), p[len(root):] else: if dblink(cpv).isowner(path, root): yield dblink(cpv), path class vartree(object): "this tree will scan a var/db/pkg database located at root (passed to init)" def __init__(self, root="/", virtual=None, clone=None, categories=None, settings=None): if clone: writemsg("vartree.__init__(): deprecated " + \ "use of clone parameter\n", noiselevel=-1) self.root = clone.root[:] self.dbapi = copy.deepcopy(clone.dbapi) self.populated = 1 from portage import config self.settings = config(clone=clone.settings) else: self.root = root[:] if settings is None: from portage import settings self.settings = settings # for key_expand calls if categories is None: categories = settings.categories self.dbapi = vardbapi(self.root, categories=categories, settings=settings, vartree=self) self.populated = 1 def getpath(self, mykey, filename=None): return self.dbapi.getpath(mykey, filename=filename) def zap(self, mycpv): return def inject(self, mycpv): return def get_provide(self, mycpv): myprovides = [] mylines = None try: mylines, myuse = self.dbapi.aux_get(mycpv, ["PROVIDE", "USE"]) if mylines: myuse = myuse.split() mylines = flatten(use_reduce(paren_reduce(mylines), uselist=myuse)) for myprovide in mylines: mys = catpkgsplit(myprovide) if not mys: mys = myprovide.split("/") myprovides += [mys[0] + "/" + mys[1]] return myprovides except SystemExit as e: raise except Exception as e: mydir = os.path.join(self.root, VDB_PATH, mycpv) writemsg(_("\nParse Error reading PROVIDE and USE in '%s'\n") % mydir, noiselevel=-1) if mylines: writemsg(_("Possibly Invalid: '%s'\n") % str(mylines), noiselevel=-1) writemsg(_("Exception: %s\n\n") % str(e), noiselevel=-1) return [] def get_all_provides(self): myprovides = {} for node in self.getallcpv(): for mykey in self.get_provide(node): if mykey in myprovides: myprovides[mykey] += [node] else: myprovides[mykey] = [node] return myprovides def dep_bestmatch(self, mydep, use_cache=1): "compatibility method -- all matches, not just visible ones" #mymatch=best(match(dep_expand(mydep,self.dbapi),self.dbapi)) mymatch = best(self.dbapi.match( dep_expand(mydep, mydb=self.dbapi, settings=self.settings), use_cache=use_cache)) if mymatch is None: return "" else: return mymatch def dep_match(self, mydep, use_cache=1): "compatibility method -- we want to see all matches, not just visible ones" #mymatch = match(mydep,self.dbapi) mymatch = self.dbapi.match(mydep, use_cache=use_cache) if mymatch is None: return [] else: return mymatch def exists_specific(self, cpv): return self.dbapi.cpv_exists(cpv) def getallcpv(self): """temporary function, probably to be renamed --- Gets a list of all category/package-versions installed on the system.""" return self.dbapi.cpv_all() def getallnodes(self): """new behavior: these are all *unmasked* nodes. There may or may not be available masked package for nodes in this nodes list.""" return self.dbapi.cp_all() def exists_specific_cat(self, cpv, use_cache=1): warnings.warn( "portage.dbapi.vartree.vartree.exists_specific_cat() is deprecated", DeprecationWarning) cpv = key_expand(cpv, mydb=self.dbapi, use_cache=use_cache, settings=self.settings) a = catpkgsplit(cpv) if not a: return 0 mylist = listdir(self.getpath(a[0]), EmptyOnError=1) for x in mylist: b = pkgsplit(x) if not b: self.dbapi.invalidentry(self.getpath(a[0], filename=x)) continue if a[1] == b[0]: return 1 return 0 def getebuildpath(self, fullpackage): cat, package = catsplit(fullpackage) return self.getpath(fullpackage, filename=package+".ebuild") def getnode(self, mykey, use_cache=1): warnings.warn( "portage.dbapi.vartree.vartree.getnode() is deprecated", DeprecationWarning) mykey = key_expand(mykey, mydb=self.dbapi, use_cache=use_cache, settings=self.settings) if not mykey: return [] mysplit = catsplit(mykey) mydirlist = listdir(self.getpath(mysplit[0]),EmptyOnError=1) returnme = [] for x in mydirlist: mypsplit = pkgsplit(x) if not mypsplit: self.dbapi.invalidentry(self.getpath(mysplit[0], filename=x)) continue if mypsplit[0] == mysplit[1]: appendme = [mysplit[0]+"/"+x, [mysplit[0], mypsplit[0], mypsplit[1], mypsplit[2]]] returnme.append(appendme) return returnme def getslot(self, mycatpkg): "Get a slot for a catpkg; assume it exists." try: return self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] except KeyError: return "" def hasnode(self, mykey, use_cache): """Does the particular node (cat/pkg key) exist?""" warnings.warn( "portage.dbapi.vartree.vartree.hadnode() is deprecated", DeprecationWarning) mykey = key_expand(mykey, mydb=self.dbapi, use_cache=use_cache, settings=self.settings) mysplit = catsplit(mykey) mydirlist = listdir(self.getpath(mysplit[0]), EmptyOnError=1) for x in mydirlist: mypsplit = pkgsplit(x) if not mypsplit: self.dbapi.invalidentry(self.getpath(mysplit[0], filename=x)) continue if mypsplit[0] == mysplit[1]: return 1 return 0 def populate(self): self.populated=1 class dblink(object): """ This class provides an interface to the installed package database At present this is implemented as a text backend in /var/db/pkg. """ import re _normalize_needed = re.compile(r'.*//.*|^[^/]|.+/$|(^|.*/)\.\.?(/.*|$)') _contents_split_counts = { "dev": 2, "dir": 2, "fif": 2, "obj": 4, "sym": 5 } # When looping over files for merge/unmerge, temporarily yield to the # scheduler each time this many files are processed. _file_merge_yield_interval = 20 def __init__(self, cat, pkg, myroot, mysettings, treetype=None, vartree=None, blockers=None, scheduler=None): """ Creates a DBlink object for a given CPV. The given CPV may not be present in the database already. @param cat: Category @type cat: String @param pkg: Package (PV) @type pkg: String @param myroot: Typically ${ROOT} @type myroot: String (Path) @param mysettings: Typically portage.config @type mysettings: An instance of portage.config @param treetype: one of ['porttree','bintree','vartree'] @type treetype: String @param vartree: an instance of vartree corresponding to myroot. @type vartree: vartree """ self.cat = cat self.pkg = pkg self.mycpv = self.cat + "/" + self.pkg self.mysplit = list(catpkgsplit(self.mycpv)[1:]) self.mysplit[0] = "%s/%s" % (self.cat, self.mysplit[0]) self.treetype = treetype if vartree is None: from portage import db vartree = db[myroot]["vartree"] self.vartree = vartree self._blockers = blockers self._scheduler = scheduler self.dbroot = normalize_path(os.path.join(myroot, VDB_PATH)) self.dbcatdir = self.dbroot+"/"+cat self.dbpkgdir = self.dbcatdir+"/"+pkg self.dbtmpdir = self.dbcatdir+"/-MERGING-"+pkg self.dbdir = self.dbpkgdir self._lock_vdb = None self.settings = mysettings self._verbose = self.settings.get("PORTAGE_VERBOSE") == "1" self.myroot=myroot protect_obj = ConfigProtect(myroot, portage.util.shlex_split(mysettings.get("CONFIG_PROTECT", "")), portage.util.shlex_split( mysettings.get("CONFIG_PROTECT_MASK", ""))) self.updateprotect = protect_obj.updateprotect self.isprotected = protect_obj.isprotected self._installed_instance = None self.contentscache = None self._contents_inodes = None self._contents_basenames = None self._md5_merge_map = {} def lockdb(self): if self._lock_vdb: raise AssertionError("Lock already held.") # At least the parent needs to exist for the lock file. ensure_dirs(self.dbroot) self._lock_vdb = lockdir(self.dbroot) def unlockdb(self): if self._lock_vdb: unlockdir(self._lock_vdb) self._lock_vdb = None def getpath(self): "return path to location of db information (for >>> informational display)" return self.dbdir def exists(self): "does the db entry exist? boolean." return os.path.exists(self.dbdir) def delete(self): """ Remove this entry from the database """ if not os.path.exists(self.dbdir): return # Check validity of self.dbdir before attempting to remove it. if not self.dbdir.startswith(self.dbroot): writemsg(_("portage.dblink.delete(): invalid dbdir: %s\n") % \ self.dbdir, noiselevel=-1) return shutil.rmtree(self.dbdir) self.vartree.dbapi._remove(self) def clearcontents(self): """ For a given db entry (self), erase the CONTENTS values. """ if os.path.exists(self.dbdir+"/CONTENTS"): os.unlink(self.dbdir+"/CONTENTS") def _clear_contents_cache(self): self.contentscache = None self._contents_inodes = None self._contents_basenames = None def getcontents(self): """ Get the installed files of a given package (aka what that package installed) """ contents_file = os.path.join(self.dbdir, "CONTENTS") if self.contentscache is not None: return self.contentscache pkgfiles = {} try: myc = codecs.open(_unicode_encode(contents_file, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e self.contentscache = pkgfiles return pkgfiles mylines = myc.readlines() myc.close() null_byte = "\0" normalize_needed = self._normalize_needed contents_split_counts = self._contents_split_counts myroot = self.myroot if myroot == os.path.sep: myroot = None pos = 0 errors = [] for pos, line in enumerate(mylines): if null_byte in line: # Null bytes are a common indication of corruption. errors.append((pos + 1, _("Null byte found in CONTENTS entry"))) continue line = line.rstrip("\n") # Split on " " so that even file paths that # end with spaces can be handled. mydat = line.split(" ") entry_type = mydat[0] # empty string if line is empty correct_split_count = contents_split_counts.get(entry_type) if correct_split_count and len(mydat) > correct_split_count: # Apparently file paths contain spaces, so reassemble # the split have the correct_split_count. newsplit = [entry_type] spaces_total = len(mydat) - correct_split_count if entry_type == "sym": try: splitter = mydat.index("->", 2, len(mydat) - 2) except ValueError: errors.append((pos + 1, _("Unrecognized CONTENTS entry"))) continue spaces_in_path = splitter - 2 spaces_in_target = spaces_total - spaces_in_path newsplit.append(" ".join(mydat[1:splitter])) newsplit.append("->") target_end = splitter + spaces_in_target + 2 newsplit.append(" ".join(mydat[splitter + 1:target_end])) newsplit.extend(mydat[target_end:]) else: path_end = spaces_total + 2 newsplit.append(" ".join(mydat[1:path_end])) newsplit.extend(mydat[path_end:]) mydat = newsplit # we do this so we can remove from non-root filesystems # (use the ROOT var to allow maintenance on other partitions) try: if normalize_needed.match(mydat[1]): mydat[1] = normalize_path(mydat[1]) if not mydat[1].startswith(os.path.sep): mydat[1] = os.path.sep + mydat[1] if myroot: mydat[1] = os.path.join(myroot, mydat[1].lstrip(os.path.sep)) if mydat[0] == "obj": #format: type, mtime, md5sum pkgfiles[mydat[1]] = [mydat[0], mydat[3], mydat[2]] elif mydat[0] == "dir": #format: type pkgfiles[mydat[1]] = [mydat[0]] elif mydat[0] == "sym": #format: type, mtime, dest pkgfiles[mydat[1]] = [mydat[0], mydat[4], mydat[3]] elif mydat[0] == "dev": #format: type pkgfiles[mydat[1]] = [mydat[0]] elif mydat[0]=="fif": #format: type pkgfiles[mydat[1]] = [mydat[0]] else: errors.append((pos + 1, _("Unrecognized CONTENTS entry"))) except (KeyError, IndexError): errors.append((pos + 1, _("Unrecognized CONTENTS entry"))) if errors: writemsg(_("!!! Parse error in '%s'\n") % contents_file, noiselevel=-1) for pos, e in errors: writemsg(_("!!! line %d: %s\n") % (pos, e), noiselevel=-1) self.contentscache = pkgfiles return pkgfiles def unmerge(self, pkgfiles=None, trimworld=1, cleanup=1, ldpath_mtimes=None, others_in_slot=None): """ Calls prerm Unmerges a given package (CPV) calls postrm calls cleanrm calls env_update @param pkgfiles: files to unmerge (generally self.getcontents() ) @type pkgfiles: Dictionary @param trimworld: Remove CPV from world file if True, not if False @type trimworld: Boolean @param cleanup: cleanup to pass to doebuild (see doebuild) @type cleanup: Boolean @param ldpath_mtimes: mtimes to pass to env_update (see env_update) @type ldpath_mtimes: Dictionary @param others_in_slot: all dblink instances in this slot, excluding self @type others_in_slot: list @rtype: Integer @returns: 1. os.EX_OK if everything went well. 2. return code of the failed phase (for prerm, postrm, cleanrm) Notes: The caller must ensure that lockdb() and unlockdb() are called before and after this method. """ showMessage = self._display_merge if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None # When others_in_slot is supplied, the security check has already been # done for this slot, so it shouldn't be repeated until the next # replacement or unmerge operation. if others_in_slot is None: slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0] slot_matches = self.vartree.dbapi.match( "%s:%s" % (dep_getkey(self.mycpv), slot)) others_in_slot = [] for cur_cpv in slot_matches: if cur_cpv == self.mycpv: continue others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], self.vartree.root, self.settings, vartree=self.vartree, treetype="vartree")) retval = self._security_check([self] + others_in_slot) if retval: return retval contents = self.getcontents() # Now, don't assume that the name of the ebuild is the same as the # name of the dir; the package may have been moved. myebuildpath = None ebuild_phase = "prerm" log_path = None mystuff = os.listdir(self.dbdir) for x in mystuff: if x.endswith(".ebuild"): myebuildpath = os.path.join(self.dbdir, self.pkg + ".ebuild") if x[:-7] != self.pkg: # Clean up after vardbapi.move_ent() breakage in # portage versions before 2.1.2 os.rename(os.path.join(self.dbdir, x), myebuildpath) write_atomic(os.path.join(self.dbdir, "PF"), self.pkg+"\n") break self.settings.setcpv(self.mycpv, mydb=self.vartree.dbapi) if myebuildpath: try: doebuild_environment(myebuildpath, "prerm", self.myroot, self.settings, 0, 0, self.vartree.dbapi) except UnsupportedAPIException as e: # Sometimes this happens due to corruption of the EAPI file. writemsg(_("!!! FAILED prerm: %s\n") % \ os.path.join(self.dbdir, "EAPI"), noiselevel=-1) writemsg("%s\n" % str(e), noiselevel=-1) myebuildpath = None else: catdir = os.path.dirname(self.settings["PORTAGE_BUILDDIR"]) ensure_dirs(os.path.dirname(catdir), uid=portage_uid, gid=portage_gid, mode=0o70, mask=0) builddir_lock = None catdir_lock = None scheduler = self._scheduler retval = -1 failures = 0 try: if myebuildpath: catdir_lock = lockdir(catdir) ensure_dirs(catdir, uid=portage_uid, gid=portage_gid, mode=0o70, mask=0) builddir_lock = lockdir( self.settings["PORTAGE_BUILDDIR"]) try: unlockdir(catdir_lock) finally: catdir_lock = None prepare_build_dirs(self.myroot, self.settings, 1) log_path = self.settings.get("PORTAGE_LOG_FILE") if scheduler is None: retval = doebuild(myebuildpath, ebuild_phase, self.myroot, self.settings, cleanup=cleanup, use_cache=0, mydbapi=self.vartree.dbapi, tree=self.treetype, vartree=self.vartree) else: retval = scheduler.dblinkEbuildPhase( self, self.vartree.dbapi, myebuildpath, ebuild_phase) # XXX: Decide how to handle failures here. if retval != os.EX_OK: failures += 1 writemsg(_("!!! FAILED prerm: %s\n") % retval, noiselevel=-1) self._unmerge_pkgfiles(pkgfiles, others_in_slot) self._clear_contents_cache() if myebuildpath: ebuild_phase = "postrm" if scheduler is None: retval = doebuild(myebuildpath, ebuild_phase, self.myroot, self.settings, use_cache=0, tree=self.treetype, mydbapi=self.vartree.dbapi, vartree=self.vartree) else: retval = scheduler.dblinkEbuildPhase( self, self.vartree.dbapi, myebuildpath, ebuild_phase) # XXX: Decide how to handle failures here. if retval != os.EX_OK: failures += 1 writemsg(_("!!! FAILED postrm: %s\n") % retval, noiselevel=-1) finally: if builddir_lock: try: if myebuildpath: if retval != os.EX_OK: msg_lines = [] msg = _("The '%(ebuild_phase)s' " "phase of the '%(cpv)s' package " "has failed with exit value %(retval)s.") % \ {"ebuild_phase":ebuild_phase, "cpv":self.mycpv, "retval":retval} from textwrap import wrap msg_lines.extend(wrap(msg, 72)) msg_lines.append("") ebuild_name = os.path.basename(myebuildpath) ebuild_dir = os.path.dirname(myebuildpath) msg = _("The problem occurred while executing " "the ebuild file named '%(ebuild_name)s' " "located in the '%(ebuild_dir)s' directory. " "If necessary, manually remove " "the environment.bz2 file and/or the " "ebuild file located in that directory.") % \ {"ebuild_name":ebuild_name, "ebuild_dir":ebuild_dir} msg_lines.extend(wrap(msg, 72)) msg_lines.append("") msg = _("Removal " "of the environment.bz2 file is " "preferred since it may allow the " "removal phases to execute successfully. " "The ebuild will be " "sourced and the eclasses " "from the current portage tree will be used " "when necessary. Removal of " "the ebuild file will cause the " "pkg_prerm() and pkg_postrm() removal " "phases to be skipped entirely.") msg_lines.extend(wrap(msg, 72)) self._eerror(ebuild_phase, msg_lines) # process logs created during pre/postrm elog_process(self.mycpv, self.settings, phasefilter=filter_unmergephases) if retval == os.EX_OK: if scheduler is None: doebuild(myebuildpath, "cleanrm", self.myroot, self.settings, tree="vartree", mydbapi=self.vartree.dbapi, vartree=self.vartree) else: scheduler.dblinkEbuildPhase( self, self.vartree.dbapi, myebuildpath, "cleanrm") finally: unlockdir(builddir_lock) try: if myebuildpath and not catdir_lock: # Lock catdir for removal if empty. catdir_lock = lockdir(catdir) finally: if catdir_lock: try: os.rmdir(catdir) except OSError as e: if e.errno not in (errno.ENOENT, errno.ENOTEMPTY, errno.EEXIST): raise del e unlockdir(catdir_lock) if log_path is not None: if not failures and 'unmerge-logs' not in self.settings.features: try: os.unlink(log_path) except OSError: pass try: st = os.stat(log_path) except OSError: pass else: if st.st_size == 0: try: os.unlink(log_path) except OSError: pass if log_path is not None and os.path.exists(log_path): # Restore this since it gets lost somewhere above and it # needs to be set for _display_merge() to be able to log. # Note that the log isn't necessarily supposed to exist # since if PORT_LOGDIR is unset then it's a temp file # so it gets cleaned above. self.settings["PORTAGE_LOG_FILE"] = log_path else: self.settings.pop("PORTAGE_LOG_FILE", None) env_update(target_root=self.myroot, prev_mtimes=ldpath_mtimes, contents=contents, env=self.settings.environ(), writemsg_level=self._display_merge) return os.EX_OK def _display_merge(self, msg, level=0, noiselevel=0): if not self._verbose and noiselevel >= 0 and level < logging.WARN: return if self._scheduler is not None: self._scheduler.dblinkDisplayMerge(self, msg, level=level, noiselevel=noiselevel) return writemsg_level(msg, level=level, noiselevel=noiselevel) def _unmerge_pkgfiles(self, pkgfiles, others_in_slot): """ Unmerges the contents of a package from the liveFS Removes the VDB entry for self @param pkgfiles: typically self.getcontents() @type pkgfiles: Dictionary { filename: [ 'type', '?', 'md5sum' ] } @param others_in_slot: all dblink instances in this slot, excluding self @type others_in_slot: list @rtype: None """ os = _os_merge perf_md5 = perform_md5 showMessage = self._display_merge scheduler = self._scheduler if not pkgfiles: showMessage(_("No package files given... Grabbing a set.\n")) pkgfiles = self.getcontents() if others_in_slot is None: others_in_slot = [] slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0] slot_matches = self.vartree.dbapi.match( "%s:%s" % (dep_getkey(self.mycpv), slot)) for cur_cpv in slot_matches: if cur_cpv == self.mycpv: continue others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], self.vartree.root, self.settings, vartree=self.vartree, treetype="vartree")) dest_root = normalize_path(self.vartree.root).rstrip(os.path.sep) + \ os.path.sep dest_root_len = len(dest_root) - 1 conf_mem_file = os.path.join(dest_root, CONFIG_MEMORY_FILE) cfgfiledict = grabdict(conf_mem_file) stale_confmem = [] unmerge_orphans = "unmerge-orphans" in self.settings.features if pkgfiles: self.updateprotect() mykeys = list(pkgfiles) mykeys.sort() mykeys.reverse() #process symlinks second-to-last, directories last. mydirs = [] ignored_unlink_errnos = ( errno.EBUSY, errno.ENOENT, errno.ENOTDIR, errno.EISDIR) ignored_rmdir_errnos = ( errno.EEXIST, errno.ENOTEMPTY, errno.EBUSY, errno.ENOENT, errno.ENOTDIR, errno.EISDIR) modprotect = os.path.join(self.vartree.root, "lib/modules/") def unlink(file_name, lstatobj): if bsd_chflags: if lstatobj.st_flags != 0: bsd_chflags.lchflags(file_name, 0) parent_name = os.path.dirname(file_name) # Use normal stat/chflags for the parent since we want to # follow any symlinks to the real parent directory. pflags = os.stat(parent_name).st_flags if pflags != 0: bsd_chflags.chflags(parent_name, 0) try: if not stat.S_ISLNK(lstatobj.st_mode): # Remove permissions to ensure that any hardlinks to # suid/sgid files are rendered harmless. os.chmod(file_name, 0) os.unlink(file_name) finally: if bsd_chflags and pflags != 0: # Restore the parent flags we saved before unlinking bsd_chflags.chflags(parent_name, pflags) def show_unmerge(zing, desc, file_type, file_name): showMessage("%s %s %s %s\n" % \ (zing, desc.ljust(8), file_type, file_name)) unmerge_desc = {} unmerge_desc["cfgpro"] = _("cfgpro") unmerge_desc["replaced"] = _("replaced") unmerge_desc["!dir"] = _("!dir") unmerge_desc["!empty"] = _("!empty") unmerge_desc["!fif"] = _("!fif") unmerge_desc["!found"] = _("!found") unmerge_desc["!md5"] = _("!md5") unmerge_desc["!mtime"] = _("!mtime") unmerge_desc["!obj"] = _("!obj") unmerge_desc["!sym"] = _("!sym") for i, objkey in enumerate(mykeys): if scheduler is not None and \ 0 == i % self._file_merge_yield_interval: scheduler.scheduleYield() obj = normalize_path(objkey) if os is _os_merge: try: _unicode_encode(obj, encoding=_encodings['merge'], errors='strict') except UnicodeEncodeError: # The package appears to have been merged with a # different value of sys.getfilesystemencoding(), # so fall back to utf_8 if appropriate. try: _unicode_encode(obj, encoding=_encodings['fs'], errors='strict') except UnicodeEncodeError: pass else: os = portage.os perf_md5 = portage.checksum.perform_md5 file_data = pkgfiles[objkey] file_type = file_data[0] statobj = None try: statobj = os.stat(obj) except OSError: pass lstatobj = None try: lstatobj = os.lstat(obj) except (OSError, AttributeError): pass islink = lstatobj is not None and stat.S_ISLNK(lstatobj.st_mode) if lstatobj is None: show_unmerge("---", unmerge_desc["!found"], file_type, obj) continue if obj.startswith(dest_root): relative_path = obj[dest_root_len:] is_owned = False for dblnk in others_in_slot: if dblnk.isowner(relative_path, dest_root): is_owned = True break if is_owned: # A new instance of this package claims the file, so # don't unmerge it. show_unmerge("---", unmerge_desc["replaced"], file_type, obj) continue elif relative_path in cfgfiledict: stale_confmem.append(relative_path) # next line includes a tweak to protect modules from being unmerged, # but we don't protect modules from being overwritten if they are # upgraded. We effectively only want one half of the config protection # functionality for /lib/modules. For portage-ng both capabilities # should be able to be independently specified. if obj.startswith(modprotect): show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj) continue # Don't unlink symlinks to directories here since that can # remove /lib and /usr/lib symlinks. if unmerge_orphans and \ lstatobj and not stat.S_ISDIR(lstatobj.st_mode) and \ not (islink and statobj and stat.S_ISDIR(statobj.st_mode)) and \ not self.isprotected(obj): try: unlink(obj, lstatobj) except EnvironmentError as e: if e.errno not in ignored_unlink_errnos: raise del e show_unmerge("<<<", "", file_type, obj) continue lmtime = str(lstatobj[stat.ST_MTIME]) if (pkgfiles[objkey][0] not in ("dir", "fif", "dev")) and (lmtime != pkgfiles[objkey][1]): show_unmerge("---", unmerge_desc["!mtime"], file_type, obj) continue if pkgfiles[objkey][0] == "dir": if lstatobj is None or not stat.S_ISDIR(lstatobj.st_mode): show_unmerge("---", unmerge_desc["!dir"], file_type, obj) continue mydirs.append(obj) elif pkgfiles[objkey][0] == "sym": if not islink: show_unmerge("---", unmerge_desc["!sym"], file_type, obj) continue # Go ahead and unlink symlinks to directories here when # they're actually recorded as symlinks in the contents. # Normally, symlinks such as /lib -> lib64 are not recorded # as symlinks in the contents of a package. If a package # installs something into ${D}/lib/, it is recorded in the # contents as a directory even if it happens to correspond # to a symlink when it's merged to the live filesystem. try: unlink(obj, lstatobj) show_unmerge("<<<", "", file_type, obj) except (OSError, IOError) as e: if e.errno not in ignored_unlink_errnos: raise del e show_unmerge("!!!", "", file_type, obj) elif pkgfiles[objkey][0] == "obj": if statobj is None or not stat.S_ISREG(statobj.st_mode): show_unmerge("---", unmerge_desc["!obj"], file_type, obj) continue mymd5 = None try: mymd5 = perf_md5(obj, calc_prelink=1) except FileNotFound as e: # the file has disappeared between now and our stat call show_unmerge("---", unmerge_desc["!obj"], file_type, obj) continue # string.lower is needed because db entries used to be in upper-case. The # string.lower allows for backwards compatibility. if mymd5 != pkgfiles[objkey][2].lower(): show_unmerge("---", unmerge_desc["!md5"], file_type, obj) continue try: unlink(obj, lstatobj) except (OSError, IOError) as e: if e.errno not in ignored_unlink_errnos: raise del e show_unmerge("<<<", "", file_type, obj) elif pkgfiles[objkey][0] == "fif": if not stat.S_ISFIFO(lstatobj[stat.ST_MODE]): show_unmerge("---", unmerge_desc["!fif"], file_type, obj) continue show_unmerge("---", "", file_type, obj) elif pkgfiles[objkey][0] == "dev": show_unmerge("---", "", file_type, obj) mydirs.sort() mydirs.reverse() for obj in mydirs: try: if bsd_chflags: lstatobj = os.lstat(obj) if lstatobj.st_flags != 0: bsd_chflags.lchflags(obj, 0) parent_name = os.path.dirname(obj) # Use normal stat/chflags for the parent since we want to # follow any symlinks to the real parent directory. pflags = os.stat(parent_name).st_flags if pflags != 0: bsd_chflags.chflags(parent_name, 0) try: os.rmdir(obj) finally: if bsd_chflags and pflags != 0: # Restore the parent flags we saved before unlinking bsd_chflags.chflags(parent_name, pflags) show_unmerge("<<<", "", "dir", obj) except EnvironmentError as e: if e.errno not in ignored_rmdir_errnos: raise if e.errno != errno.ENOENT: show_unmerge("---", unmerge_desc["!empty"], "dir", obj) del e # Remove stale entries from config memory. if stale_confmem: for filename in stale_confmem: del cfgfiledict[filename] writedict(cfgfiledict, conf_mem_file) #remove self from vartree database so that our own virtual gets zapped if we're the last node self.vartree.zap(self.mycpv) def isowner(self, filename, destroot): """ Check if a file belongs to this package. This may result in a stat call for the parent directory of every installed file, since the inode numbers are used to work around the problem of ambiguous paths caused by symlinked directories. The results of stat calls are cached to optimize multiple calls to this method. @param filename: @type filename: @param destroot: @type destroot: @rtype: Boolean @returns: 1. True if this package owns the file. 2. False if this package does not own the file. """ return bool(self._match_contents(filename, destroot)) def _match_contents(self, filename, destroot): """ The matching contents entry is returned, which is useful since the path may differ from the one given by the caller, due to symlinks. @rtype: String @return: the contents entry corresponding to the given path, or False if the file is not owned by this package. """ filename = _unicode_decode(filename, encoding=_encodings['content'], errors='strict') destroot = _unicode_decode(destroot, encoding=_encodings['content'], errors='strict') # The given filename argument might have a different encoding than the # the filenames contained in the contents, so use separate wrapped os # modules for each. The basename is more likely to contain non-ascii # characters than the directory path, so use os_filename_arg for all # operations involving the basename of the filename arg. os_filename_arg = _os_merge os = _os_merge try: _unicode_encode(filename, encoding=_encodings['merge'], errors='strict') except UnicodeEncodeError: # The package appears to have been merged with a # different value of sys.getfilesystemencoding(), # so fall back to utf_8 if appropriate. try: _unicode_encode(filename, encoding=_encodings['fs'], errors='strict') except UnicodeEncodeError: pass else: os_filename_arg = portage.os destfile = normalize_path( os_filename_arg.path.join(destroot, filename.lstrip(os_filename_arg.path.sep))) pkgfiles = self.getcontents() if pkgfiles and destfile in pkgfiles: return destfile if pkgfiles: basename = os_filename_arg.path.basename(destfile) if self._contents_basenames is None: try: for x in pkgfiles: _unicode_encode(x, encoding=_encodings['merge'], errors='strict') except UnicodeEncodeError: # The package appears to have been merged with a # different value of sys.getfilesystemencoding(), # so fall back to utf_8 if appropriate. try: for x in pkgfiles: _unicode_encode(x, encoding=_encodings['fs'], errors='strict') except UnicodeEncodeError: pass else: os = portage.os self._contents_basenames = set( os.path.basename(x) for x in pkgfiles) if basename not in self._contents_basenames: # This is a shortcut that, in most cases, allows us to # eliminate this package as an owner without the need # to examine inode numbers of parent directories. return False # Use stat rather than lstat since we want to follow # any symlinks to the real parent directory. parent_path = os_filename_arg.path.dirname(destfile) try: parent_stat = os_filename_arg.stat(parent_path) except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e return False if self._contents_inodes is None: if os is _os_merge: try: for x in pkgfiles: _unicode_encode(x, encoding=_encodings['merge'], errors='strict') except UnicodeEncodeError: # The package appears to have been merged with a # different value of sys.getfilesystemencoding(), # so fall back to utf_8 if appropriate. try: for x in pkgfiles: _unicode_encode(x, encoding=_encodings['fs'], errors='strict') except UnicodeEncodeError: pass else: os = portage.os self._contents_inodes = {} parent_paths = set() for x in pkgfiles: p_path = os.path.dirname(x) if p_path in parent_paths: continue parent_paths.add(p_path) try: s = os.stat(p_path) except OSError: pass else: inode_key = (s.st_dev, s.st_ino) # Use lists of paths in case multiple # paths reference the same inode. p_path_list = self._contents_inodes.get(inode_key) if p_path_list is None: p_path_list = [] self._contents_inodes[inode_key] = p_path_list if p_path not in p_path_list: p_path_list.append(p_path) p_path_list = self._contents_inodes.get( (parent_stat.st_dev, parent_stat.st_ino)) if p_path_list: for p_path in p_path_list: x = os_filename_arg.path.join(p_path, basename) if x in pkgfiles: return x return False def _collision_protect(self, srcroot, destroot, mypkglist, mycontents): os = _os_merge collision_ignore = set([normalize_path(myignore) for myignore in \ portage.util.shlex_split( self.settings.get("COLLISION_IGNORE", ""))]) showMessage = self._display_merge scheduler = self._scheduler stopmerge = False collisions = [] destroot = normalize_path(destroot).rstrip(os.path.sep) + \ os.path.sep showMessage(_(" %s checking %d files for package collisions\n") % \ (colorize("GOOD", "*"), len(mycontents))) for i, f in enumerate(mycontents): if i % 1000 == 0 and i != 0: showMessage(_("%d files checked ...\n") % i) if scheduler is not None and \ 0 == i % self._file_merge_yield_interval: scheduler.scheduleYield() dest_path = normalize_path( os.path.join(destroot, f.lstrip(os.path.sep))) try: dest_lstat = os.lstat(dest_path) except EnvironmentError as e: if e.errno == errno.ENOENT: del e continue elif e.errno == errno.ENOTDIR: del e # A non-directory is in a location where this package # expects to have a directory. dest_lstat = None parent_path = dest_path while len(parent_path) > len(destroot): parent_path = os.path.dirname(parent_path) try: dest_lstat = os.lstat(parent_path) break except EnvironmentError as e: if e.errno != errno.ENOTDIR: raise del e if not dest_lstat: raise AssertionError( "unable to find non-directory " + \ "parent for '%s'" % dest_path) dest_path = parent_path f = os.path.sep + dest_path[len(destroot):] if f in collisions: continue else: raise if f[0] != "/": f="/"+f isowned = False full_path = os.path.join(destroot, f.lstrip(os.path.sep)) for ver in mypkglist: if ver.isowner(f, destroot): isowned = True break if not isowned and self.isprotected(full_path): isowned = True if not isowned: stopmerge = True if collision_ignore: if f in collision_ignore: stopmerge = False else: for myignore in collision_ignore: if f.startswith(myignore + os.path.sep): stopmerge = False break if stopmerge: collisions.append(f) return collisions def _lstat_inode_map(self, path_iter): """ Use lstat to create a map of the form: {(st_dev, st_ino) : set([path1, path2, ...])} Multiple paths may reference the same inode due to hardlinks. All lstat() calls are relative to self.myroot. """ os = _os_merge root = self.myroot inode_map = {} for f in path_iter: path = os.path.join(root, f.lstrip(os.sep)) try: st = os.lstat(path) except OSError as e: if e.errno not in (errno.ENOENT, errno.ENOTDIR): raise del e continue key = (st.st_dev, st.st_ino) paths = inode_map.get(key) if paths is None: paths = set() inode_map[key] = paths paths.add(f) return inode_map def _security_check(self, installed_instances): if not installed_instances: return 0 os = _os_merge showMessage = self._display_merge scheduler = self._scheduler file_paths = set() for dblnk in installed_instances: file_paths.update(dblnk.getcontents()) inode_map = {} real_paths = set() for i, path in enumerate(file_paths): if scheduler is not None and \ 0 == i % self._file_merge_yield_interval: scheduler.scheduleYield() if os is _os_merge: try: _unicode_encode(path, encoding=_encodings['merge'], errors='strict') except UnicodeEncodeError: # The package appears to have been merged with a # different value of sys.getfilesystemencoding(), # so fall back to utf_8 if appropriate. try: _unicode_encode(path, encoding=_encodings['fs'], errors='strict') except UnicodeEncodeError: pass else: os = portage.os try: s = os.lstat(path) except OSError as e: if e.errno not in (errno.ENOENT, errno.ENOTDIR): raise del e continue if not stat.S_ISREG(s.st_mode): continue path = os.path.realpath(path) if path in real_paths: continue real_paths.add(path) if s.st_nlink > 1 and \ s.st_mode & (stat.S_ISUID | stat.S_ISGID): k = (s.st_dev, s.st_ino) inode_map.setdefault(k, []).append((path, s)) suspicious_hardlinks = [] for path_list in inode_map.values(): path, s = path_list[0] if len(path_list) == s.st_nlink: # All hardlinks seem to be owned by this package. continue suspicious_hardlinks.append(path_list) if not suspicious_hardlinks: return 0 msg = [] msg.append(_("suid/sgid file(s) " "with suspicious hardlink(s):")) msg.append("") for path_list in suspicious_hardlinks: for path, s in path_list: msg.append("\t%s" % path) msg.append("") msg.append(_("See the Gentoo Security Handbook " "guide for advice on how to proceed.")) self._eerror("preinst", msg) return 1 def _eqawarn(self, phase, lines): from portage.elog.messages import eqawarn as _eqawarn if self._scheduler is None: for l in lines: _eqawarn(l, phase=phase, key=self.settings.mycpv) else: self._scheduler.dblinkElog(self, phase, _eqawarn, lines) def _eerror(self, phase, lines): from portage.elog.messages import eerror as _eerror if self._scheduler is None: for l in lines: _eerror(l, phase=phase, key=self.settings.mycpv) else: self._scheduler.dblinkElog(self, phase, _eerror, lines) def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0, mydbapi=None, prev_mtimes=None): """ This function does the following: calls self._preserve_libs if FEATURES=preserve-libs calls self._collision_protect if FEATURES=collision-protect calls doebuild(mydo=pkg_preinst) Merges the package to the livefs unmerges old version (if required) calls doebuild(mydo=pkg_postinst) calls env_update calls elog_process @param srcroot: Typically this is ${D} @type srcroot: String (Path) @param destroot: Path to merge to (usually ${ROOT}) @type destroot: String (Path) @param inforoot: root of the vardb entry ? @type inforoot: String (Path) @param myebuild: path to the ebuild that we are processing @type myebuild: String (Path) @param mydbapi: dbapi which is handed to doebuild. @type mydbapi: portdbapi instance @param prev_mtimes: { Filename:mtime } mapping for env_update @type prev_mtimes: Dictionary @rtype: Boolean @returns: 1. 0 on success 2. 1 on failure secondhand is a list of symlinks that have been skipped due to their target not existing; we will merge these symlinks at a later time. """ os = _os_merge srcroot = _unicode_decode(srcroot, encoding=_encodings['content'], errors='strict') destroot = _unicode_decode(destroot, encoding=_encodings['content'], errors='strict') inforoot = _unicode_decode(inforoot, encoding=_encodings['content'], errors='strict') myebuild = _unicode_decode(myebuild, encoding=_encodings['content'], errors='strict') showMessage = self._display_merge scheduler = self._scheduler srcroot = normalize_path(srcroot).rstrip(os.path.sep) + os.path.sep destroot = normalize_path(destroot).rstrip(os.path.sep) + os.path.sep if not os.path.isdir(srcroot): showMessage(_("!!! Directory Not Found: D='%s'\n") % srcroot, level=logging.ERROR, noiselevel=-1) return 1 slot = '' for var_name in ('CHOST', 'SLOT'): try: val = codecs.open(_unicode_encode( os.path.join(inforoot, var_name), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace').readline().strip() except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e val = '' if var_name == 'SLOT': slot = val if not slot.strip(): slot = self.settings.get(var_name, '') if not slot.strip(): showMessage(_("!!! SLOT is undefined\n"), level=logging.ERROR, noiselevel=-1) return 1 write_atomic(os.path.join(inforoot, var_name), slot + '\n') if val != self.settings.get(var_name, ''): self._eqawarn('preinst', [_("QA Notice: Expected %(var_name)s='%(expected_value)s', got '%(actual_value)s'\n") % \ {"var_name":var_name, "expected_value":self.settings.get(var_name, ''), "actual_value":val}]) def eerror(lines): self._eerror("preinst", lines) if not os.path.exists(self.dbcatdir): os.makedirs(self.dbcatdir) otherversions = [] for v in self.vartree.dbapi.cp_list(self.mysplit[0]): otherversions.append(v.split("/")[1]) cp = self.mysplit[0] slot_atom = "%s:%s" % (cp, slot) # filter any old-style virtual matches slot_matches = [cpv for cpv in self.vartree.dbapi.match(slot_atom) \ if cpv_getkey(cpv) == cp] if self.mycpv not in slot_matches and \ self.vartree.dbapi.cpv_exists(self.mycpv): # handle multislot or unapplied slotmove slot_matches.append(self.mycpv) others_in_slot = [] from portage import config for cur_cpv in slot_matches: # Clone the config in case one of these has to be unmerged since # we need it to have private ${T} etc... for things like elog. others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], self.vartree.root, config(clone=self.settings), vartree=self.vartree, treetype="vartree", scheduler=self._scheduler)) retval = self._security_check(others_in_slot) if retval: return retval if slot_matches: # Used by self.isprotected(). max_dblnk = None max_counter = -1 for dblnk in others_in_slot: cur_counter = self.vartree.dbapi.cpv_counter(dblnk.mycpv) if cur_counter > max_counter: max_counter = cur_counter max_dblnk = dblnk self._installed_instance = max_dblnk # We check for unicode encoding issues after src_install. However, # the check must be repeated here for binary packages (it's # inexpensive since we call os.walk() here anyway). unicode_errors = [] while True: unicode_error = False myfilelist = [] mylinklist = [] paths_with_newlines = [] srcroot_len = len(srcroot) def onerror(e): raise for parent, dirs, files in os.walk(srcroot, onerror=onerror): try: parent = _unicode_decode(parent, encoding=_encodings['merge'], errors='strict') except UnicodeDecodeError: new_parent = _unicode_decode(parent, encoding=_encodings['merge'], errors='replace') new_parent = _unicode_encode(new_parent, encoding=_encodings['merge'], errors='backslashreplace') new_parent = _unicode_decode(new_parent, encoding=_encodings['merge'], errors='replace') os.rename(parent, new_parent) unicode_error = True unicode_errors.append(new_parent[srcroot_len:]) break for fname in files: try: fname = _unicode_decode(fname, encoding=_encodings['merge'], errors='strict') except UnicodeDecodeError: fpath = portage._os.path.join( parent.encode(_encodings['merge']), fname) new_fname = _unicode_decode(fname, encoding=_encodings['merge'], errors='replace') new_fname = _unicode_encode(new_fname, encoding=_encodings['merge'], errors='backslashreplace') new_fname = _unicode_decode(new_fname, encoding=_encodings['merge'], errors='replace') new_fpath = os.path.join(parent, new_fname) os.rename(fpath, new_fpath) unicode_error = True unicode_errors.append(new_fpath[srcroot_len:]) fname = new_fname fpath = new_fpath else: fpath = os.path.join(parent, fname) relative_path = fpath[srcroot_len:] if "\n" in relative_path: paths_with_newlines.append(relative_path) file_mode = os.lstat(fpath).st_mode if stat.S_ISREG(file_mode): myfilelist.append(relative_path) elif stat.S_ISLNK(file_mode): # Note: os.walk puts symlinks to directories in the "dirs" # list and it does not traverse them since that could lead # to an infinite recursion loop. mylinklist.append(relative_path) if unicode_error: break if not unicode_error: break if unicode_errors: eerror(portage._merge_unicode_error(unicode_errors)) if paths_with_newlines: msg = [] msg.append(_("This package installs one or more files containing a newline (\\n) character:")) msg.append("") paths_with_newlines.sort() for f in paths_with_newlines: msg.append("\t/%s" % (f.replace("\n", "\\n"))) msg.append("") msg.append(_("package %s NOT merged") % self.mycpv) msg.append("") eerror(msg) return 1 # If there are no files to merge, and an installed package in the same # slot has files, it probably means that something went wrong. if self.settings.get("PORTAGE_PACKAGE_EMPTY_ABORT") == "1" and \ not myfilelist and not mylinklist and others_in_slot: installed_files = None for other_dblink in others_in_slot: installed_files = other_dblink.getcontents() if not installed_files: continue from textwrap import wrap wrap_width = 72 msg = [] d = { "new_cpv":self.mycpv, "old_cpv":other_dblink.mycpv } msg.extend(wrap(_("The '%(new_cpv)s' package will not install " "any files, but the currently installed '%(old_cpv)s'" " package has the following files: ") % d, wrap_width)) msg.append("") msg.extend(sorted(installed_files)) msg.append("") msg.append(_("package %s NOT merged") % self.mycpv) msg.append("") msg.extend(wrap( _("Manually run `emerge --unmerge =%s` if you " "really want to remove the above files. Set " "PORTAGE_PACKAGE_EMPTY_ABORT=\"0\" in " "/etc/make.conf if you do not want to " "abort in cases like this.") % other_dblink.mycpv, wrap_width)) eerror(msg) if installed_files: return 1 # check for package collisions blockers = None if self._blockers is not None: # This is only supposed to be called when # the vdb is locked, like it is here. blockers = self._blockers() if blockers is None: blockers = [] collisions = \ self._collision_protect(srcroot, destroot, others_in_slot + blockers, myfilelist + mylinklist) # Make sure the ebuild environment is initialized and that ${T}/elog # exists for logging of collision-protect eerror messages. if myebuild is None: myebuild = os.path.join(inforoot, self.pkg + ".ebuild") doebuild_environment(myebuild, "preinst", destroot, self.settings, 0, 0, mydbapi) prepare_build_dirs(destroot, self.settings, cleanup) if collisions: collision_protect = "collision-protect" in self.settings.features protect_owned = "protect-owned" in self.settings.features msg = _("This package will overwrite one or more files that" " may belong to other packages (see list below).") if not (collision_protect or protect_owned): msg += _(" Add either \"collision-protect\" or" " \"protect-owned\" to FEATURES in" " make.conf if you would like the merge to abort" " in cases like this. See the make.conf man page for" " more information about these features.") if self.settings.get("PORTAGE_QUIET") != "1": msg += _(" You can use a command such as" " `portageq owners / ` to identify the" " installed package that owns a file. If portageq" " reports that only one package owns a file then do NOT" " file a bug report. A bug report is only useful if it" " identifies at least two or more packages that are known" " to install the same file(s)." " If a collision occurs and you" " can not explain where the file came from then you" " should simply ignore the collision since there is not" " enough information to determine if a real problem" " exists. Please do NOT file a bug report at" " http://bugs.gentoo.org unless you report exactly which" " two packages install the same file(s). Once again," " please do NOT file a bug report unless you have" " completely understood the above message.") self.settings["EBUILD_PHASE"] = "preinst" from textwrap import wrap msg = wrap(msg, 70) if collision_protect: msg.append("") msg.append(_("package %s NOT merged") % self.settings.mycpv) msg.append("") msg.append(_("Detected file collision(s):")) msg.append("") for f in collisions: msg.append("\t%s" % \ os.path.join(destroot, f.lstrip(os.path.sep))) eerror(msg) msg = [] msg.append("") msg.append(_("Searching all installed" " packages for file collisions...")) msg.append("") msg.append(_("Press Ctrl-C to Stop")) msg.append("") eerror(msg) owners = self.vartree.dbapi._owners.get_owners(collisions) self.vartree.dbapi.flush_cache() for pkg, owned_files in owners.items(): cpv = pkg.mycpv msg = [] msg.append("%s" % cpv) for f in sorted(owned_files): msg.append("\t%s" % os.path.join(destroot, f.lstrip(os.path.sep))) msg.append("") eerror(msg) if not owners: eerror([_("None of the installed" " packages claim the file(s)."), ""]) # The explanation about the collision and how to solve # it may not be visible via a scrollback buffer, especially # if the number of file collisions is large. Therefore, # show a summary at the end. if collision_protect: msg = _("Package '%s' NOT merged due to file collisions.") % \ self.settings.mycpv elif protect_owned and owners: msg = _("Package '%s' NOT merged due to file collisions.") % \ self.settings.mycpv else: msg = _("Package '%s' merged despite file collisions.") % \ self.settings.mycpv msg += _(" If necessary, refer to your elog " "messages for the whole content of the above message.") eerror(wrap(msg, 70)) if collision_protect or (protect_owned and owners): return 1 # The merge process may move files out of the image directory, # which causes invalidation of the .installed flag. try: os.unlink(os.path.join( os.path.dirname(normalize_path(srcroot)), ".installed")) except OSError as e: if e.errno != errno.ENOENT: raise del e self.dbdir = self.dbtmpdir self.delete() ensure_dirs(self.dbtmpdir) # run preinst script if scheduler is None: showMessage(_(">>> Merging %(cpv)s to %(destroot)s\n") % {"cpv":self.mycpv, "destroot":destroot}) a = doebuild(myebuild, "preinst", destroot, self.settings, use_cache=0, tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) else: a = scheduler.dblinkEbuildPhase( self, mydbapi, myebuild, "preinst") # XXX: Decide how to handle failures here. if a != os.EX_OK: showMessage(_("!!! FAILED preinst: ")+str(a)+"\n", level=logging.ERROR, noiselevel=-1) return a # copy "info" files (like SLOT, CFLAGS, etc.) into the database for x in os.listdir(inforoot): self.copyfile(inforoot+"/"+x) # write local package counter for recording counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv) codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), encoding=_encodings['fs'], errors='strict'), 'w', encoding=_encodings['repo.content'], errors='backslashreplace' ).write(str(counter)) # open CONTENTS file (possibly overwriting old one) for recording outfile = codecs.open(_unicode_encode( os.path.join(self.dbtmpdir, 'CONTENTS'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='backslashreplace') self.updateprotect() #if we have a file containing previously-merged config file md5sums, grab it. conf_mem_file = os.path.join(destroot, CONFIG_MEMORY_FILE) cfgfiledict = grabdict(conf_mem_file) cfgfiledict_orig = cfgfiledict.copy() if "NOCONFMEM" in self.settings: cfgfiledict["IGNORE"]=1 else: cfgfiledict["IGNORE"]=0 # Always behave like --noconfmem is enabled for downgrades # so that people who don't know about this option are less # likely to get confused when doing upgrade/downgrade cycles. pv_split = catpkgsplit(self.mycpv)[1:] for other in others_in_slot: if pkgcmp(pv_split, catpkgsplit(other.mycpv)[1:]) < 0: cfgfiledict["IGNORE"] = 1 break # Don't bump mtimes on merge since some application require # preservation of timestamps. This means that the unmerge phase must # check to see if file belongs to an installed instance in the same # slot. mymtime = None # set umask to 0 for merging; back up umask, save old one in prevmask (since this is a global change) prevmask = os.umask(0) secondhand = [] # we do a first merge; this will recurse through all files in our srcroot but also build up a # "second hand" of symlinks to merge later if self.mergeme(srcroot, destroot, outfile, secondhand, "", cfgfiledict, mymtime): return 1 # now, it's time for dealing our second hand; we'll loop until we can't merge anymore. The rest are # broken symlinks. We'll merge them too. lastlen = 0 while len(secondhand) and len(secondhand)!=lastlen: # clear the thirdhand. Anything from our second hand that # couldn't get merged will be added to thirdhand. thirdhand = [] if self.mergeme(srcroot, destroot, outfile, thirdhand, secondhand, cfgfiledict, mymtime): return 1 #swap hands lastlen = len(secondhand) # our thirdhand now becomes our secondhand. It's ok to throw # away secondhand since thirdhand contains all the stuff that # couldn't be merged. secondhand = thirdhand if len(secondhand): # force merge of remaining symlinks (broken or circular; oh well) if self.mergeme(srcroot, destroot, outfile, None, secondhand, cfgfiledict, mymtime): return 1 self._md5_merge_map.clear() #restore umask os.umask(prevmask) #if we opened it, close it outfile.flush() outfile.close() # write out our collection of md5sums cfgfiledict.pop("IGNORE", None) if cfgfiledict != cfgfiledict_orig: ensure_dirs(os.path.dirname(conf_mem_file), gid=portage_gid, mode=0o2750, mask=0o2) writedict(cfgfiledict, conf_mem_file) # These caches are populated during collision-protect and the data # they contain is now invalid. It's very important to invalidate # the contents_inodes cache so that FEATURES=unmerge-orphans # doesn't unmerge anything that belongs to this package that has # just been merged. for dblnk in others_in_slot: dblnk._clear_contents_cache() self._clear_contents_cache() # If portage is reinstalling itself, remove the old # version now since we want to use the temporary # PORTAGE_BIN_PATH that will be removed when we return. reinstall_self = False if self.myroot == "/" and \ match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]): reinstall_self = True if scheduler is None: def emerge_log(msg): pass else: emerge_log = scheduler.dblinkEmergeLog autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes" if autoclean: emerge_log(_(" >>> AUTOCLEAN: %s") % (slot_atom,)) others_in_slot.append(self) # self has just been merged for dblnk in list(others_in_slot): if dblnk is self: continue if not (autoclean or dblnk.mycpv == self.mycpv or reinstall_self): continue showMessage(_(">>> Safely unmerging already-installed instance...\n")) emerge_log(_(" === Unmerging... (%s)") % (dblnk.mycpv,)) others_in_slot.remove(dblnk) # dblnk will unmerge itself now unmerge_rval = dblnk.unmerge(trimworld=0, ldpath_mtimes=prev_mtimes, others_in_slot=others_in_slot) if unmerge_rval == os.EX_OK: emerge_log(_(" >>> unmerge success: %s") % (dblnk.mycpv,)) else: emerge_log(_(" !!! unmerge FAILURE: %s") % (dblnk.mycpv,)) # TODO: Check status and abort if necessary. dblnk.delete() showMessage(_(">>> Original instance of package unmerged safely.\n")) if len(others_in_slot) > 1: showMessage(colorize("WARN", _("WARNING:")) + _(" AUTOCLEAN is disabled. This can cause serious" " problems due to overlapping packages.\n"), level=logging.WARN, noiselevel=-1) # We hold both directory locks. self.dbdir = self.dbpkgdir self.delete() _movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings) # Check for file collisions with blocking packages # and remove any colliding files from their CONTENTS # since they now belong to this package. self._clear_contents_cache() contents = self.getcontents() destroot_len = len(destroot) - 1 for blocker in blockers: self.vartree.dbapi.removeFromContents(blocker, iter(contents), relative_paths=False) self.vartree.dbapi._add(self) contents = self.getcontents() #do postinst script self.settings["PORTAGE_UPDATE_ENV"] = \ os.path.join(self.dbpkgdir, "environment.bz2") self.settings.backup_changes("PORTAGE_UPDATE_ENV") try: if scheduler is None: a = doebuild(myebuild, "postinst", destroot, self.settings, use_cache=0, tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) if a == os.EX_OK: showMessage(_(">>> %s merged.\n") % self.mycpv) else: a = scheduler.dblinkEbuildPhase( self, mydbapi, myebuild, "postinst") finally: self.settings.pop("PORTAGE_UPDATE_ENV", None) # XXX: Decide how to handle failures here. if a != os.EX_OK: showMessage(_("!!! FAILED postinst: ")+str(a)+"\n", level=logging.ERROR, noiselevel=-1) return a downgrade = False for v in otherversions: if pkgcmp(catpkgsplit(self.pkg)[1:], catpkgsplit(v)[1:]) < 0: downgrade = True #update environment settings, library paths. DO NOT change symlinks. env_update(makelinks=(not downgrade), target_root=self.settings["ROOT"], prev_mtimes=prev_mtimes, contents=contents, env=self.settings.environ(), writemsg_level=self._display_merge) return os.EX_OK def _new_backup_path(self, p): """ The works for any type path, such as a regular file, symlink, or directory. The parent directory is assumed to exist. The returned filename is of the form p + '.backup.' + x, where x guarantees that the returned path does not exist yet. """ os = _os_merge x = -1 while True: x += 1 backup_p = p + '.backup.' + str(x).rjust(4, '0') try: os.lstat(backup_p) except OSError: break return backup_p def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime): """ This function handles actual merging of the package contents to the livefs. It also handles config protection. @param srcroot: Where are we copying files from (usually ${D}) @type srcroot: String (Path) @param destroot: Typically ${ROOT} @type destroot: String (Path) @param outfile: File to log operations to @type outfile: File Object @param secondhand: A set of items to merge in pass two (usually or symlinks that point to non-existing files that may get merged later) @type secondhand: List @param stufftomerge: Either a diretory to merge, or a list of items. @type stufftomerge: String or List @param cfgfiledict: { File:mtime } mapping for config_protected files @type cfgfiledict: Dictionary @param thismtime: The current time (typically long(time.time()) @type thismtime: Long @rtype: None or Boolean @returns: 1. True on failure 2. None otherwise """ showMessage = self._display_merge writemsg = self._display_merge scheduler = self._scheduler os = _os_merge sep = os.sep join = os.path.join srcroot = normalize_path(srcroot).rstrip(sep) + sep destroot = normalize_path(destroot).rstrip(sep) + sep # this is supposed to merge a list of files. There will be 2 forms of argument passing. if isinstance(stufftomerge, basestring): #A directory is specified. Figure out protection paths, listdir() it and process it. mergelist = os.listdir(join(srcroot, stufftomerge)) offset = stufftomerge else: mergelist = stufftomerge offset = "" for i, x in enumerate(mergelist): if scheduler is not None and \ 0 == i % self._file_merge_yield_interval: scheduler.scheduleYield() mysrc = join(srcroot, offset, x) mydest = join(destroot, offset, x) # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/") myrealdest = join(sep, offset, x) # stat file once, test using S_* macros many times (faster that way) mystat = os.lstat(mysrc) mymode = mystat[stat.ST_MODE] # handy variables; mydest is the target object on the live filesystems; # mysrc is the source object in the temporary install dir try: mydstat = os.lstat(mydest) mydmode = mydstat.st_mode except OSError as e: if e.errno != errno.ENOENT: raise del e #dest file doesn't exist mydstat = None mydmode = None if stat.S_ISLNK(mymode): # we are merging a symbolic link myabsto = abssymlink(mysrc) if myabsto.startswith(srcroot): myabsto = myabsto[len(srcroot):] myabsto = myabsto.lstrip(sep) myto = os.readlink(mysrc) if self.settings and self.settings["D"]: if myto.startswith(self.settings["D"]): myto = myto[len(self.settings["D"]):] # myrealto contains the path of the real file to which this symlink points. # we can simply test for existence of this file to see if the target has been merged yet myrealto = normalize_path(os.path.join(destroot, myabsto)) if mydmode!=None: #destination exists if not stat.S_ISLNK(mydmode): if stat.S_ISDIR(mydmode): # directory in the way: we can't merge a symlink over a directory # we won't merge this, continue with next file... continue if os.path.exists(mysrc) and stat.S_ISDIR(os.stat(mysrc)[stat.ST_MODE]): # Kill file blocking installation of symlink to dir #71787 pass elif self.isprotected(mydest): # Use md5 of the target in ${D} if it exists... try: newmd5 = perform_md5(join(srcroot, myabsto)) except FileNotFound: # Maybe the target is merged already. try: newmd5 = perform_md5(myrealto) except FileNotFound: newmd5 = None mydest = new_protect_filename(mydest, newmd5=newmd5) # if secondhand is None it means we're operating in "force" mode and should not create a second hand. if (secondhand != None) and (not os.path.exists(myrealto)): # either the target directory doesn't exist yet or the target file doesn't exist -- or # the target is a broken symlink. We will add this file to our "second hand" and merge # it later. secondhand.append(mysrc[len(srcroot):]) continue # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, encoding=_encodings['merge']) if mymtime != None: showMessage(">>> %s -> %s\n" % (mydest, myto)) outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") else: showMessage(_("!!! Failed to move file.\n"), level=logging.ERROR, noiselevel=-1) showMessage("!!! %s -> %s\n" % (mydest, myto), level=logging.ERROR, noiselevel=-1) return 1 elif stat.S_ISDIR(mymode): # we are merging a directory if mydmode != None: # destination exists if bsd_chflags: # Save then clear flags on dest. dflags = mydstat.st_flags if dflags != 0: bsd_chflags.lchflags(mydest, 0) if not os.access(mydest, os.W_OK): pkgstuff = pkgsplit(self.pkg) writemsg(_("\n!!! Cannot write to '%s'.\n") % mydest, noiselevel=-1) writemsg(_("!!! Please check permissions and directories for broken symlinks.\n")) writemsg(_("!!! You may start the merge process again by using ebuild:\n")) writemsg("!!! ebuild "+self.settings["PORTDIR"]+"/"+self.cat+"/"+pkgstuff[0]+"/"+self.pkg+".ebuild merge\n") writemsg(_("!!! And finish by running this: env-update\n\n")) return 1 if stat.S_ISDIR(mydmode) or \ (stat.S_ISLNK(mydmode) and os.path.isdir(mydest)): # a symlink to an existing directory will work for us; keep it: showMessage("--- %s/\n" % mydest) if bsd_chflags: bsd_chflags.lchflags(mydest, dflags) else: # a non-directory and non-symlink-to-directory. Won't work for us. Move out of the way. backup_dest = self._new_backup_path(mydest) msg = [] msg.append("") msg.append("Installation of a directory is blocked by a file:") msg.append(" '%s'" % mydest) msg.append("This file will be renamed to a different name:") msg.append(" '%s'" % backup_dest) msg.append("") self._eerror("preinst", msg) if movefile(mydest, backup_dest, mysettings=self.settings, encoding=_encodings['merge']) is None: return 1 showMessage(_("bak %s %s.backup\n") % (mydest, mydest), level=logging.ERROR, noiselevel=-1) #now create our directory if self.settings.selinux_enabled(): _selinux_merge.mkdir(mydest, mysrc) else: os.mkdir(mydest) if bsd_chflags: bsd_chflags.lchflags(mydest, dflags) os.chmod(mydest, mystat[0]) os.chown(mydest, mystat[4], mystat[5]) showMessage(">>> %s/\n" % mydest) else: #destination doesn't exist if self.settings.selinux_enabled(): _selinux_merge.mkdir(mydest, mysrc) else: os.mkdir(mydest) os.chmod(mydest, mystat[0]) os.chown(mydest, mystat[4], mystat[5]) showMessage(">>> %s/\n" % mydest) outfile.write("dir "+myrealdest+"\n") # recurse and merge this directory if self.mergeme(srcroot, destroot, outfile, secondhand, join(offset, x), cfgfiledict, thismtime): return 1 elif stat.S_ISREG(mymode): # we are merging a regular file mymd5 = perform_md5(mysrc, calc_prelink=1) # calculate config file protection stuff mydestdir = os.path.dirname(mydest) moveme = 1 zing = "!!!" mymtime = None protected = self.isprotected(mydest) if mydmode != None: # destination file exists if stat.S_ISDIR(mydmode): # install of destination is blocked by an existing directory with the same name newdest = self._new_backup_path(mydest) msg = [] msg.append("") msg.append("Installation of a regular file is blocked by a directory:") msg.append(" '%s'" % mydest) msg.append("This file will be merged with a different name:") msg.append(" '%s'" % newdest) msg.append("") self._eerror("preinst", msg) mydest = newdest elif stat.S_ISREG(mydmode) or (stat.S_ISLNK(mydmode) and os.path.exists(mydest) and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])): # install of destination is blocked by an existing regular file, # or by a symlink to an existing regular file; # now, config file management may come into play. # we only need to tweak mydest if cfg file management is in play. if protected: # we have a protection path; enable config file management. cfgprot = 0 destmd5 = perform_md5(mydest, calc_prelink=1) if mymd5 == destmd5: #file already in place; simply update mtimes of destination moveme = 1 else: if mymd5 == cfgfiledict.get(myrealdest, [None])[0]: """ An identical update has previously been merged. Skip it unless the user has chosen --noconfmem.""" moveme = cfgfiledict["IGNORE"] cfgprot = cfgfiledict["IGNORE"] if not moveme: zing = "---" mymtime = long(mystat.st_mtime) else: moveme = 1 cfgprot = 1 if moveme: # Merging a new file, so update confmem. cfgfiledict[myrealdest] = [mymd5] elif destmd5 == cfgfiledict.get(myrealdest, [None])[0]: """A previously remembered update has been accepted, so it is removed from confmem.""" del cfgfiledict[myrealdest] if cfgprot: mydest = new_protect_filename(mydest, newmd5=mymd5) # whether config protection or not, we merge the new file the # same way. Unless moveme=0 (blocking directory) if moveme: # Do not hardlink files unless they are in the same # directory, since otherwise tar may not be able to # extract a tarball of the resulting hardlinks due to # 'Invalid cross-device link' errors (depends on layout of # mount points). Also, don't hardlink zero-byte files since # it doesn't save any space, and don't hardlink # CONFIG_PROTECTed files since config files shouldn't be # hardlinked to eachother (for example, shadow installs # several identical config files inside /etc/pam.d/). parent_dir = os.path.dirname(myrealdest) hardlink_key = (parent_dir, mymd5, mystat.st_size, mystat.st_mode, mystat.st_uid, mystat.st_gid) hardlink_candidates = None if not protected and mystat.st_size != 0: hardlink_candidates = self._md5_merge_map.get(hardlink_key) if hardlink_candidates is None: hardlink_candidates = [] self._md5_merge_map[hardlink_key] = hardlink_candidates mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, hardlink_candidates=hardlink_candidates, encoding=_encodings['merge']) if mymtime is None: return 1 if hardlink_candidates is not None: hardlink_candidates.append(mydest) zing = ">>>" if mymtime != None: outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") showMessage("%s %s\n" % (zing,mydest)) else: # we are merging a fifo or device node zing = "!!!" if mydmode is None: # destination doesn't exist if movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, encoding=_encodings['merge']) is not None: zing = ">>>" else: return 1 if stat.S_ISFIFO(mymode): outfile.write("fif %s\n" % myrealdest) else: outfile.write("dev %s\n" % myrealdest) showMessage(zing + " " + mydest + "\n") def merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0, mydbapi=None, prev_mtimes=None): """ If portage is reinstalling itself, create temporary copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH in order to avoid relying on the new versions which may be incompatible. Register an atexit hook to clean up the temporary directories. Pre-load elog modules here since we won't be able to later if they get unmerged (happens when namespace changes). """ if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None if self.myroot == "/" and \ match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]) and \ not self.vartree.dbapi.cpv_exists(self.mycpv): settings = self.settings base_path_orig = os.path.dirname(settings["PORTAGE_BIN_PATH"]) from tempfile import mkdtemp # Make the temp directory inside PORTAGE_TMPDIR since, unlike # /tmp, it can't be mounted with the "noexec" option. base_path_tmp = mkdtemp("", "._portage_reinstall_.", settings["PORTAGE_TMPDIR"]) from portage.process import atexit_register atexit_register(shutil.rmtree, base_path_tmp) dir_perms = 0o755 for subdir in "bin", "pym": var_name = "PORTAGE_%s_PATH" % subdir.upper() var_orig = settings[var_name] var_new = os.path.join(base_path_tmp, subdir) settings[var_name] = var_new settings.backup_changes(var_name) shutil.copytree(var_orig, var_new, symlinks=True) os.chmod(var_new, dir_perms) os.chmod(base_path_tmp, dir_perms) # This serves so pre-load the modules. elog_process(self.mycpv, self.settings, phasefilter=filter_mergephases) return self._merge(mergeroot, inforoot, myroot, myebuild=myebuild, cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes) def _merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0, mydbapi=None, prev_mtimes=None): retval = -1 self.lockdb() try: retval = self.treewalk(mergeroot, myroot, inforoot, myebuild, cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes) # Process ebuild logfiles elog_process(self.mycpv, self.settings, phasefilter=filter_mergephases) if 'noclean' not in self.settings.features and \ (retval == os.EX_OK or \ ('fail-clean' in self.settings.features and \ os.path.isdir(self.settings['PORTAGE_BUILDDIR']))): if myebuild is None: myebuild = os.path.join(inforoot, self.pkg + ".ebuild") if self._scheduler is None: doebuild(myebuild, "clean", myroot, self.settings, tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) else: self._scheduler.dblinkEbuildPhase( self, mydbapi, myebuild, "clean") finally: self.unlockdb() return retval def getstring(self,name): "returns contents of a file with whitespace converted to spaces" if not os.path.exists(self.dbdir+"/"+name): return "" mydata = codecs.open( _unicode_encode(os.path.join(self.dbdir, name), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' ).read().split() return " ".join(mydata) def copyfile(self,fname): shutil.copyfile(fname,self.dbdir+"/"+os.path.basename(fname)) def getfile(self,fname): if not os.path.exists(self.dbdir+"/"+fname): return "" return codecs.open(_unicode_encode(os.path.join(self.dbdir, fname), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' ).read() def setfile(self,fname,data): kwargs = {} if fname == 'environment.bz2' or not isinstance(data, basestring): kwargs['mode'] = 'wb' else: kwargs['mode'] = 'w' kwargs['encoding'] = _encodings['repo.content'] write_atomic(os.path.join(self.dbdir, fname), data, **kwargs) def getelements(self,ename): if not os.path.exists(self.dbdir+"/"+ename): return [] mylines = codecs.open(_unicode_encode( os.path.join(self.dbdir, ename), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace' ).readlines() myreturn = [] for x in mylines: for y in x[:-1].split(): myreturn.append(y) return myreturn def setelements(self,mylist,ename): myelement = codecs.open(_unicode_encode( os.path.join(self.dbdir, ename), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='backslashreplace') for x in mylist: myelement.write(x+"\n") myelement.close() def isregular(self): "Is this a regular package (does it have a CATEGORY file? A dblink can be virtual *and* regular)" return os.path.exists(os.path.join(self.dbdir, "CATEGORY")) def write_contents(contents, root, f): """ Write contents to any file like object. The file will be left open. """ root_len = len(root) - 1 for filename in sorted(contents): entry_data = contents[filename] entry_type = entry_data[0] relative_filename = filename[root_len:] if entry_type == "obj": entry_type, mtime, md5sum = entry_data line = "%s %s %s %s\n" % \ (entry_type, relative_filename, md5sum, mtime) elif entry_type == "sym": entry_type, mtime, link = entry_data line = "%s %s -> %s %s\n" % \ (entry_type, relative_filename, link, mtime) else: # dir, dev, fif line = "%s %s\n" % (entry_type, relative_filename) f.write(line) def tar_contents(contents, root, tar, protect=None, onProgress=None): os = _os_merge try: for x in contents: _unicode_encode(x, encoding=_encodings['merge'], errors='strict') except UnicodeEncodeError: # The package appears to have been merged with a # different value of sys.getfilesystemencoding(), # so fall back to utf_8 if appropriate. try: for x in contents: _unicode_encode(x, encoding=_encodings['fs'], errors='strict') except UnicodeEncodeError: pass else: os = portage.os from portage.util import normalize_path import tarfile root = normalize_path(root).rstrip(os.path.sep) + os.path.sep id_strings = {} maxval = len(contents) curval = 0 if onProgress: onProgress(maxval, 0) paths = list(contents) paths.sort() for path in paths: curval += 1 try: lst = os.lstat(path) except OSError as e: if e.errno != errno.ENOENT: raise del e if onProgress: onProgress(maxval, curval) continue contents_type = contents[path][0] if path.startswith(root): arcname = path[len(root):] else: raise ValueError("invalid root argument: '%s'" % root) live_path = path if 'dir' == contents_type and \ not stat.S_ISDIR(lst.st_mode) and \ os.path.isdir(live_path): # Even though this was a directory in the original ${D}, it exists # as a symlink to a directory in the live filesystem. It must be # recorded as a real directory in the tar file to ensure that tar # can properly extract it's children. live_path = os.path.realpath(live_path) tarinfo = tar.gettarinfo(live_path, arcname) if stat.S_ISREG(lst.st_mode): # break hardlinks due to bug #185305 tarinfo.type = tarfile.REGTYPE if protect and protect(path): # Create an empty file as a place holder in order to avoid # potential collision-protect issues. tarinfo.size = 0 tar.addfile(tarinfo) else: f = open(_unicode_encode(path, encoding=object.__getattribute__(os, '_encoding'), errors='strict'), 'rb') try: tar.addfile(tarinfo, f) finally: f.close() else: tar.addfile(tarinfo) if onProgress: onProgress(maxval, curval)