From fbceadf2a286ddc7ac2bf2ae35f491cfc7eb71e8 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Fri, 21 Aug 2009 02:42:45 +0000 Subject: Use _encodings where appropriate. svn path=/main/trunk/; revision=14110 --- pym/portage/dbapi/vartree.py | 151 ++++++++++++++++++++++++++----------------- pym/portage/util.py | 23 ++++--- 2 files changed, 104 insertions(+), 70 deletions(-) (limited to 'pym') diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 9421da20c..d73311bc0 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -37,9 +37,7 @@ from portage import listdir, dep_expand, digraph, flatten, key_expand, \ # This is a special version of the os module, wrapped for unicode support. from portage import os -from portage import _content_encoding -from portage import _fs_encoding -from portage import _merge_encoding +from portage import _encodings from portage import _os_merge from portage import _selinux_merge from portage import _unicode_decode @@ -80,7 +78,8 @@ class PreservedLibsRegistry(object): self._data = None try: self._data = pickle.load( - open(_unicode_encode(self._filename), 'rb')) + open(_unicode_encode(self._filename, + encoding=_encodings['fs'], errors='strict'), 'rb')) except (ValueError, pickle.UnpicklingError), e: writemsg_level(_("!!! Error loading '%s': %s\n") % \ (self._filename, e), level=logging.ERROR, noiselevel=-1) @@ -327,7 +326,15 @@ class LinkageMap(object): raise CommandNotFound(args[0]) else: for l in proc.stdout: - l = portage._unicode_decode(l) + try: + l = _unicode_decode(l, + encoding=_encodings['content'], errors='strict') + except UnicodeDecodeError: + l = _unicode_decode(l, + encoding=_encodings['content'], errors='replace') + writemsg_level(_("\nError decoding characters " \ + "returned from scanelf: %s\n\n") % (l,), + level=logging.ERROR, noiselevel=-1) l = l[3:].rstrip("\n") if not l: continue @@ -765,7 +772,8 @@ class vardbapi(dbapi): now has a categories property that is generated from the available packages. """ - self.root = portage._unicode_decode(root) + self.root = _unicode_decode(root, + encoding=_encodings['content'], errors='strict') #cache for category directory mtimes self.mtdircache = {} @@ -841,7 +849,8 @@ class vardbapi(dbapi): except KeyError: continue h.update(_unicode_encode(counter, - encoding=_content_encoding, errors='replace')) + encoding=_encodings['repo.content'], + errors='backslashreplace')) return h.hexdigest() def cpv_inject(self, mycpv): @@ -1106,7 +1115,8 @@ class vardbapi(dbapi): # python-2.x, but buffering makes it much worse. open_kwargs["buffering"] = 0 try: - f = open(_unicode_encode(self._aux_cache_filename), + f = open(_unicode_encode(self._aux_cache_filename, + encoding=_encodings['fs'], errors='strict'), mode='rb', **open_kwargs) mypickle = pickle.Unpickler(f) try: @@ -1208,7 +1218,8 @@ class vardbapi(dbapi): if cache_valid: # Migrate old metadata to unicode. for k, v in metadata.iteritems(): - metadata[k] = portage._unicode_decode(v) + metadata[k] = _unicode_decode(v, + encoding=_encodings['repo.content'], errors='replace') mydata.update(metadata) pull_me.difference_update(mydata) @@ -1255,8 +1266,9 @@ class vardbapi(dbapi): try: myf = codecs.open( _unicode_encode(os.path.join(mydir, x), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') try: myd = myf.read() finally: @@ -1326,8 +1338,9 @@ class vardbapi(dbapi): try: cfile = codecs.open( _unicode_encode(self._counter_path, - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') except EnvironmentError, e: new_vdb = not bool(self.cpv_all()) if not new_vdb: @@ -1394,7 +1407,8 @@ class vardbapi(dbapi): removed = 0 for filename in paths: - filename = portage._unicode_decode(filename) + filename = _unicode_decode(filename, + encoding=_encodings['content'], errors='strict') filename = normalize_path(filename) if relative_paths: relative_filename = filename @@ -1463,7 +1477,8 @@ class vardbapi(dbapi): # Always use a constant utf_8 encoding here, since # the "default" encoding can change. h.update(_unicode_encode(s, - encoding=_content_encoding, errors='replace')) + encoding=_encodings['repo.content'], + errors='backslashreplace')) h = h.hexdigest() h = h[-self._hex_chars:] h = int(h, 16) @@ -1912,8 +1927,9 @@ class dblink(object): pkgfiles = {} try: myc = codecs.open(_unicode_encode(contents_file, - encoding=_content_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') except EnvironmentError, e: if e.errno != errno.ENOENT: raise @@ -2389,13 +2405,15 @@ class dblink(object): obj = normalize_path(objkey) if os is _os_merge: try: - _unicode_encode(obj, encoding=_merge_encoding, errors='strict') + _unicode_encode(obj, + encoding=_encodings['merge'], errors='strict') except UnicodeEncodeError: # The package appears to have been merged with a # different value of sys.getfilesystemencoding(), # so fall back to utf_8 if appropriate. try: - _unicode_encode(obj, encoding=_fs_encoding, errors='strict') + _unicode_encode(obj, + encoding=_encodings['fs'], errors='strict') except UnicodeEncodeError: pass else: @@ -2589,9 +2607,11 @@ class dblink(object): os = _os_merge - filename = portage._unicode_decode(filename) + filename = _unicode_decode(filename, + encoding=_encodings['content'], errors='strict') - destroot = portage._unicode_decode(destroot) + destroot = _unicode_decode(destroot, + encoding=_encodings['content'], errors='strict') destfile = normalize_path( os.path.join(destroot, filename.lstrip(os.path.sep))) @@ -2777,7 +2797,8 @@ class dblink(object): new_contents = self.getcontents().copy() old_contents = self._installed_instance.getcontents() for f in sorted(preserve_paths): - f = portage._unicode_decode(f, encoding=_merge_encoding) + f = _unicode_decode(f, + encoding=_encodings['content'], errors='strict') f_abs = os.path.join(root, f.lstrip(os.sep)) contents_entry = old_contents.get(f_abs) if contents_entry is None: @@ -3213,10 +3234,14 @@ class dblink(object): os = _os_merge - srcroot = portage._unicode_decode(srcroot) - destroot = portage._unicode_decode(destroot) - inforoot = portage._unicode_decode(inforoot) - myebuild = portage._unicode_decode(myebuild) + srcroot = _unicode_decode(srcroot, + encoding=_encodings['content'], errors='strict') + destroot = _unicode_decode(destroot, + encoding=_encodings['content'], errors='strict') + inforoot = _unicode_decode(inforoot, + encoding=_encodings['content'], errors='strict') + myebuild = _unicode_decode(myebuild, + encoding=_encodings['content'], errors='strict') showMessage = self._display_merge scheduler = self._scheduler @@ -3234,9 +3259,9 @@ class dblink(object): try: val = codecs.open(_unicode_encode( os.path.join(inforoot, var_name), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace' - ).readline().strip() + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace').readline().strip() except EnvironmentError, e: if e.errno != errno.ENOENT: raise @@ -3324,14 +3349,14 @@ class dblink(object): for parent, dirs, files in os.walk(srcroot, onerror=onerror): try: parent = _unicode_decode(parent, - encoding=_merge_encoding, errors='strict') + encoding=_encodings['merge'], errors='strict') except UnicodeDecodeError: new_parent = _unicode_decode(parent, - encoding=_merge_encoding, errors='replace') + encoding=_encodings['merge'], errors='replace') new_parent = _unicode_encode(new_parent, - encoding=_merge_encoding, errors='backslashreplace') + encoding=_encodings['merge'], errors='backslashreplace') new_parent = _unicode_decode(new_parent, - encoding=_merge_encoding, errors='replace') + encoding=_encodings['merge'], errors='replace') os.rename(parent, new_parent) unicode_error = True unicode_errors.append(new_parent[srcroot_len:]) @@ -3340,16 +3365,16 @@ class dblink(object): for fname in files: try: fname = _unicode_decode(fname, - encoding=_merge_encoding, errors='strict') + encoding=_encodings['merge'], errors='strict') except UnicodeDecodeError: fpath = portage._os.path.join( - parent.encode(_merge_encoding), fname) + parent.encode(_encodings['merge']), fname) new_fname = _unicode_decode(fname, - encoding=_merge_encoding, errors='replace') + encoding=_encodings['merge'], errors='replace') new_fname = _unicode_encode(new_fname, - encoding=_merge_encoding, errors='backslashreplace') + encoding=_encodings['merge'], errors='backslashreplace') new_fname = _unicode_decode(new_fname, - encoding=_merge_encoding, errors='replace') + encoding=_encodings['merge'], errors='replace') new_fpath = os.path.join(parent, new_fname) os.rename(fpath, new_fpath) unicode_error = True @@ -3577,14 +3602,17 @@ class dblink(object): # write local package counter for recording counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv) - open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER')), - 'w').write(str(counter)) + codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), + encoding=_encodings['fs'], errors='strict'), + 'w', encoding=_encodings['repo.content'], errors='backslashreplace' + ).write(str(counter)) # open CONTENTS file (possibly overwriting old one) for recording outfile = codecs.open(_unicode_encode( os.path.join(self.dbtmpdir, 'CONTENTS'), - encoding=_fs_encoding, errors='strict'), - mode='w', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='backslashreplace') self.updateprotect() @@ -3934,7 +3962,7 @@ class dblink(object): # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, - encoding=_merge_encoding) + encoding=_encodings['merge']) if mymtime != None: showMessage(">>> %s -> %s\n" % (mydest, myto)) outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") @@ -3974,7 +4002,7 @@ class dblink(object): # a non-directory and non-symlink-to-directory. Won't work for us. Move out of the way. if movefile(mydest, mydest+".backup", mysettings=self.settings, - encoding=_merge_encoding) is None: + encoding=_encodings['merge']) is None: return 1 showMessage(_("bak %s %s.backup\n") % (mydest, mydest), level=logging.ERROR, noiselevel=-1) @@ -4079,7 +4107,7 @@ class dblink(object): mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, hardlink_candidates=hardlink_candidates, - encoding=_merge_encoding) + encoding=_encodings['merge']) if mymtime is None: return 1 if hardlink_candidates is not None: @@ -4096,7 +4124,7 @@ class dblink(object): # destination doesn't exist if movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, - encoding=_merge_encoding) is not None: + encoding=_encodings['merge']) is not None: zing = ">>>" else: return 1 @@ -4184,8 +4212,8 @@ class dblink(object): return "" mydata = codecs.open( _unicode_encode(os.path.join(self.dbdir, name), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace' + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace' ).read().split() return " ".join(mydata) @@ -4196,22 +4224,27 @@ class dblink(object): if not os.path.exists(self.dbdir+"/"+fname): return "" return codecs.open(_unicode_encode(os.path.join(self.dbdir, fname), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace').read() + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace' + ).read() def setfile(self,fname,data): - mode = 'w' + kwargs = {} if fname == 'environment.bz2' or not isinstance(data, basestring): - mode = 'wb' - write_atomic(os.path.join(self.dbdir, fname), data, mode=mode) + kwargs['mode'] = 'wb' + else: + kwargs['mode'] = 'w' + kwargs['encoding'] = _encodings['repo.content'] + write_atomic(os.path.join(self.dbdir, fname), data, **kwargs) def getelements(self,ename): if not os.path.exists(self.dbdir+"/"+ename): return [] mylines = codecs.open(_unicode_encode( os.path.join(self.dbdir, ename), - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace').readlines() + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace' + ).readlines() myreturn = [] for x in mylines: for y in x[:-1].split(): @@ -4221,8 +4254,9 @@ class dblink(object): def setelements(self,mylist,ename): myelement = codecs.open(_unicode_encode( os.path.join(self.dbdir, ename), - encoding=_fs_encoding, errors='strict'), mode='w', - encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='backslashreplace') for x in mylist: myelement.write(x+"\n") myelement.close() @@ -4300,7 +4334,8 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): tarinfo.size = 0 tar.addfile(tarinfo) else: - f = open(_unicode_encode(path, encoding=_merge_encoding), 'rb') + f = open(_unicode_encode(path, + encoding=_encodings['merge'], errors='strict'), 'rb') try: tar.addfile(tarinfo, f) finally: diff --git a/pym/portage/util.py b/pym/portage/util.py index 521429aad..036191040 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -23,8 +23,7 @@ import sys import portage from portage import os -from portage import _content_encoding -from portage import _fs_encoding +from portage import _encodings from portage import _os_merge from portage import _unicode_encode from portage import _unicode_decode @@ -328,8 +327,8 @@ def grablines(myfilename,recursive=0): else: try: myfile = codecs.open(_unicode_encode(myfilename, - encoding=_fs_encoding, errors='strict'), - mode='r', encoding=_content_encoding, errors='replace') + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace') mylines = myfile.readlines() myfile.close() except IOError, e: @@ -396,11 +395,11 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): # (produces spurious \0 characters with python-2.6.2) if sys.hexversion < 0x3000000: content = open(_unicode_encode(mycfg, - encoding=_fs_encoding, errors='strict'), 'rb').read() + encoding=_encodings['fs'], errors='strict'), 'rb').read() else: content = open(_unicode_encode(mycfg, - encoding=_fs_encoding, errors='strict'), mode='r', - encoding=_content_encoding, errors='replace').read() + encoding=_encodings['fs'], errors='strict'), mode='r', + encoding=_encodings['content'], errors='replace').read() if content and content[-1] != '\n': content += '\n' except IOError, e: @@ -593,7 +592,7 @@ def pickle_read(filename,default=None,debug=0): data = None try: myf = open(_unicode_encode(filename, - encoding=_fs_encoding, errors='strict'), 'rb') + encoding=_encodings['fs'], errors='strict'), 'rb') mypickle = pickle.Unpickler(myf) data = mypickle.load() myf.close() @@ -908,8 +907,8 @@ class atomic_ofstream(ObjectProxy): open_func = open else: open_func = codecs.open - kargs.setdefault('encoding', _content_encoding) - kargs.setdefault('errors', 'replace') + kargs.setdefault('encoding', _encodings['content']) + kargs.setdefault('errors', 'backslashreplace') if follow_links: canonical_path = os.path.realpath(filename) @@ -918,7 +917,7 @@ class atomic_ofstream(ObjectProxy): try: object.__setattr__(self, '_file', open_func(_unicode_encode(tmp_name, - encoding=_fs_encoding, errors='strict'), + encoding=_encodings['fs'], errors='strict'), mode=mode, **kargs)) return except IOError, e: @@ -932,7 +931,7 @@ class atomic_ofstream(ObjectProxy): tmp_name = "%s.%i" % (filename, os.getpid()) object.__setattr__(self, '_file', open_func(_unicode_encode(tmp_name, - encoding=_fs_encoding, errors='strict'), + encoding=_encodings['fs'], errors='strict'), mode=mode, **kargs)) def _get_target(self): -- cgit v1.2.3-1-g7c22