Use _encodings where appropriate.

svn path=/main/trunk/; revision=14110

2 files changed, 104 insertions, 70 deletions

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 9421da20c..d73311bc0 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -37,9 +37,7 @@ from portage import listdir, dep_expand, digraph, flatten, key_expand, \
 
 # This is a special version of the os module, wrapped for unicode support.
 from portage import os
-from portage import _content_encoding
-from portage import _fs_encoding
-from portage import _merge_encoding
+from portage import _encodings
 from portage import _os_merge
 from portage import _selinux_merge
 from portage import _unicode_decode
@@ -80,7 +78,8 @@ class PreservedLibsRegistry(object):
 		self._data = None
 		try:
 			self._data = pickle.load(
-				open(_unicode_encode(self._filename), 'rb'))
+				open(_unicode_encode(self._filename,
+					encoding=_encodings['fs'], errors='strict'), 'rb'))
 		except (ValueError, pickle.UnpicklingError), e:
 			writemsg_level(_("!!! Error loading '%s': %s\n") % \
 				(self._filename, e), level=logging.ERROR, noiselevel=-1)
@@ -327,7 +326,15 @@ class LinkageMap(object):
 				raise CommandNotFound(args[0])
 			else:
 				for l in proc.stdout:
-					l = portage._unicode_decode(l)
+					try:
+						l = _unicode_decode(l,
+							encoding=_encodings['content'], errors='strict')
+					except UnicodeDecodeError:
+						l = _unicode_decode(l,
+							encoding=_encodings['content'], errors='replace')
+						writemsg_level(_("\nError decoding characters " \
+							"returned from scanelf: %s\n\n") % (l,),
+							level=logging.ERROR, noiselevel=-1)
 					l = l[3:].rstrip("\n")
 					if not l:
 						continue
@@ -765,7 +772,8 @@ class vardbapi(dbapi):
 		now has a categories property that is generated from the
 		available packages.
 		"""
-		self.root = portage._unicode_decode(root)
+		self.root = _unicode_decode(root,
+			encoding=_encodings['content'], errors='strict')
 
 		#cache for category directory mtimes
 		self.mtdircache = {}
@@ -841,7 +849,8 @@ class vardbapi(dbapi):
 			except KeyError:
 				continue
 			h.update(_unicode_encode(counter,
-				encoding=_content_encoding, errors='replace'))
+				encoding=_encodings['repo.content'],
+				errors='backslashreplace'))
 		return h.hexdigest()
 
 	def cpv_inject(self, mycpv):
@@ -1106,7 +1115,8 @@ class vardbapi(dbapi):
 			# python-2.x, but buffering makes it much worse.
 			open_kwargs["buffering"] = 0
 		try:
-			f = open(_unicode_encode(self._aux_cache_filename),
+			f = open(_unicode_encode(self._aux_cache_filename,
+				encoding=_encodings['fs'], errors='strict'),
 				mode='rb', **open_kwargs)
 			mypickle = pickle.Unpickler(f)
 			try:
@@ -1208,7 +1218,8 @@ class vardbapi(dbapi):
 		if cache_valid:
 			# Migrate old metadata to unicode.
 			for k, v in metadata.iteritems():
-				metadata[k] = portage._unicode_decode(v)
+				metadata[k] = _unicode_decode(v,
+					encoding=_encodings['repo.content'], errors='replace')
 
 			mydata.update(metadata)
 			pull_me.difference_update(mydata)
@@ -1255,8 +1266,9 @@ class vardbapi(dbapi):
 			try:
 				myf = codecs.open(
 					_unicode_encode(os.path.join(mydir, x),
-					encoding=_fs_encoding, errors='strict'),
-					mode='r', encoding=_content_encoding, errors='replace')
+					encoding=_encodings['fs'], errors='strict'),
+					mode='r', encoding=_encodings['repo.content'],
+					errors='replace')
 				try:
 					myd = myf.read()
 				finally:
@@ -1326,8 +1338,9 @@ class vardbapi(dbapi):
 		try:
 			cfile = codecs.open(
 				_unicode_encode(self._counter_path,
-				encoding=_fs_encoding, errors='strict'),
-				mode='r', encoding=_content_encoding, errors='replace')
+				encoding=_encodings['fs'], errors='strict'),
+				mode='r', encoding=_encodings['repo.content'],
+				errors='replace')
 		except EnvironmentError, e:
 			new_vdb = not bool(self.cpv_all())
 			if not new_vdb:
@@ -1394,7 +1407,8 @@ class vardbapi(dbapi):
 		removed = 0
 
 		for filename in paths:
-			filename = portage._unicode_decode(filename)
+			filename = _unicode_decode(filename,
+				encoding=_encodings['content'], errors='strict')
 			filename = normalize_path(filename)
 			if relative_paths:
 				relative_filename = filename
@@ -1463,7 +1477,8 @@ class vardbapi(dbapi):
 			# Always use a constant utf_8 encoding here, since
 			# the "default" encoding can change.
 			h.update(_unicode_encode(s,
-				encoding=_content_encoding, errors='replace'))
+				encoding=_encodings['repo.content'],
+				errors='backslashreplace'))
 			h = h.hexdigest()
 			h = h[-self._hex_chars:]
 			h = int(h, 16)
@@ -1912,8 +1927,9 @@ class dblink(object):
 		pkgfiles = {}
 		try:
 			myc = codecs.open(_unicode_encode(contents_file,
-				encoding=_content_encoding, errors='strict'),
-				mode='r', encoding=_content_encoding, errors='replace')
+				encoding=_encodings['fs'], errors='strict'),
+				mode='r', encoding=_encodings['repo.content'],
+				errors='replace')
 		except EnvironmentError, e:
 			if e.errno != errno.ENOENT:
 				raise
@@ -2389,13 +2405,15 @@ class dblink(object):
 				obj = normalize_path(objkey)
 				if os is _os_merge:
 					try:
-						_unicode_encode(obj, encoding=_merge_encoding, errors='strict')
+						_unicode_encode(obj,
+							encoding=_encodings['merge'], errors='strict')
 					except UnicodeEncodeError:
 						# The package appears to have been merged with a 
 						# different value of sys.getfilesystemencoding(),
 						# so fall back to utf_8 if appropriate.
 						try:
-							_unicode_encode(obj, encoding=_fs_encoding, errors='strict')
+							_unicode_encode(obj,
+								encoding=_encodings['fs'], errors='strict')
 						except UnicodeEncodeError:
 							pass
 						else:
@@ -2589,9 +2607,11 @@ class dblink(object):
 
 		os = _os_merge
 
-		filename = portage._unicode_decode(filename)
+		filename = _unicode_decode(filename,
+			encoding=_encodings['content'], errors='strict')
 
-		destroot = portage._unicode_decode(destroot)
+		destroot = _unicode_decode(destroot,
+			encoding=_encodings['content'], errors='strict')
 
 		destfile = normalize_path(
 			os.path.join(destroot, filename.lstrip(os.path.sep)))
@@ -2777,7 +2797,8 @@ class dblink(object):
 		new_contents = self.getcontents().copy()
 		old_contents = self._installed_instance.getcontents()
 		for f in sorted(preserve_paths):
-			f = portage._unicode_decode(f, encoding=_merge_encoding)
+			f = _unicode_decode(f,
+				encoding=_encodings['content'], errors='strict')
 			f_abs = os.path.join(root, f.lstrip(os.sep))
 			contents_entry = old_contents.get(f_abs)
 			if contents_entry is None:
@@ -3213,10 +3234,14 @@ class dblink(object):
 
 		os = _os_merge
 
-		srcroot = portage._unicode_decode(srcroot)
-		destroot = portage._unicode_decode(destroot)
-		inforoot = portage._unicode_decode(inforoot)
-		myebuild = portage._unicode_decode(myebuild)
+		srcroot = _unicode_decode(srcroot,
+			encoding=_encodings['content'], errors='strict')
+		destroot = _unicode_decode(destroot,
+			encoding=_encodings['content'], errors='strict')
+		inforoot = _unicode_decode(inforoot,
+			encoding=_encodings['content'], errors='strict')
+		myebuild = _unicode_decode(myebuild,
+			encoding=_encodings['content'], errors='strict')
 
 		showMessage = self._display_merge
 		scheduler = self._scheduler
@@ -3234,9 +3259,9 @@ class dblink(object):
 			try:
 				val = codecs.open(_unicode_encode(
 					os.path.join(inforoot, var_name),
-					encoding=_fs_encoding, errors='strict'),
-					mode='r', encoding=_content_encoding, errors='replace'
-					).readline().strip()
+					encoding=_encodings['fs'], errors='strict'),
+					mode='r', encoding=_encodings['repo.content'],
+					errors='replace').readline().strip()
 			except EnvironmentError, e:
 				if e.errno != errno.ENOENT:
 					raise
@@ -3324,14 +3349,14 @@ class dblink(object):
 			for parent, dirs, files in os.walk(srcroot, onerror=onerror):
 				try:
 					parent = _unicode_decode(parent,
-						encoding=_merge_encoding, errors='strict')
+						encoding=_encodings['merge'], errors='strict')
 				except UnicodeDecodeError:
 					new_parent = _unicode_decode(parent,
-						encoding=_merge_encoding, errors='replace')
+						encoding=_encodings['merge'], errors='replace')
 					new_parent = _unicode_encode(new_parent,
-						encoding=_merge_encoding, errors='backslashreplace')
+						encoding=_encodings['merge'], errors='backslashreplace')
 					new_parent = _unicode_decode(new_parent,
-						encoding=_merge_encoding, errors='replace')
+						encoding=_encodings['merge'], errors='replace')
 					os.rename(parent, new_parent)
 					unicode_error = True
 					unicode_errors.append(new_parent[srcroot_len:])
@@ -3340,16 +3365,16 @@ class dblink(object):
 				for fname in files:
 					try:
 						fname = _unicode_decode(fname,
-							encoding=_merge_encoding, errors='strict')
+							encoding=_encodings['merge'], errors='strict')
 					except UnicodeDecodeError:
 						fpath = portage._os.path.join(
-							parent.encode(_merge_encoding), fname)
+							parent.encode(_encodings['merge']), fname)
 						new_fname = _unicode_decode(fname,
-							encoding=_merge_encoding, errors='replace')
+							encoding=_encodings['merge'], errors='replace')
 						new_fname = _unicode_encode(new_fname,
-							encoding=_merge_encoding, errors='backslashreplace')
+							encoding=_encodings['merge'], errors='backslashreplace')
 						new_fname = _unicode_decode(new_fname,
-							encoding=_merge_encoding, errors='replace')
+							encoding=_encodings['merge'], errors='replace')
 						new_fpath = os.path.join(parent, new_fname)
 						os.rename(fpath, new_fpath)
 						unicode_error = True
@@ -3577,14 +3602,17 @@ class dblink(object):
 
 		# write local package counter for recording
 		counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv)
-		open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER')),
-			'w').write(str(counter))
+		codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
+			encoding=_encodings['fs'], errors='strict'),
+			'w', encoding=_encodings['repo.content'], errors='backslashreplace'
+			).write(str(counter))
 
 		# open CONTENTS file (possibly overwriting old one) for recording
 		outfile = codecs.open(_unicode_encode(
 			os.path.join(self.dbtmpdir, 'CONTENTS'),
-			encoding=_fs_encoding, errors='strict'),
-			mode='w', encoding=_content_encoding, errors='replace')
+			encoding=_encodings['fs'], errors='strict'),
+			mode='w', encoding=_encodings['repo.content'],
+			errors='backslashreplace')
 
 		self.updateprotect()
 
@@ -3934,7 +3962,7 @@ class dblink(object):
 				# unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly
 				mymtime = movefile(mysrc, mydest, newmtime=thismtime,
 					sstat=mystat, mysettings=self.settings,
-					encoding=_merge_encoding)
+					encoding=_encodings['merge'])
 				if mymtime != None:
 					showMessage(">>> %s -> %s\n" % (mydest, myto))
 					outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n")
@@ -3974,7 +4002,7 @@ class dblink(object):
 						# a non-directory and non-symlink-to-directory.  Won't work for us.  Move out of the way.
 						if movefile(mydest, mydest+".backup",
 							mysettings=self.settings,
-							encoding=_merge_encoding) is None:
+							encoding=_encodings['merge']) is None:
 							return 1
 						showMessage(_("bak %s %s.backup\n") % (mydest, mydest),
 							level=logging.ERROR, noiselevel=-1)
@@ -4079,7 +4107,7 @@ class dblink(object):
 					mymtime = movefile(mysrc, mydest, newmtime=thismtime,
 						sstat=mystat, mysettings=self.settings,
 						hardlink_candidates=hardlink_candidates,
-						encoding=_merge_encoding)
+						encoding=_encodings['merge'])
 					if mymtime is None:
 						return 1
 					if hardlink_candidates is not None:
@@ -4096,7 +4124,7 @@ class dblink(object):
 					# destination doesn't exist
 					if movefile(mysrc, mydest, newmtime=thismtime,
 						sstat=mystat, mysettings=self.settings,
-						encoding=_merge_encoding) is not None:
+						encoding=_encodings['merge']) is not None:
 						zing = ">>>"
 					else:
 						return 1
@@ -4184,8 +4212,8 @@ class dblink(object):
 			return ""
 		mydata = codecs.open(
 			_unicode_encode(os.path.join(self.dbdir, name),
-			encoding=_fs_encoding, errors='strict'),
-			mode='r', encoding=_content_encoding, errors='replace'
+			encoding=_encodings['fs'], errors='strict'),
+			mode='r', encoding=_encodings['repo.content'], errors='replace'
 			).read().split()
 		return " ".join(mydata)
 
@@ -4196,22 +4224,27 @@ class dblink(object):
 		if not os.path.exists(self.dbdir+"/"+fname):
 			return ""
 		return codecs.open(_unicode_encode(os.path.join(self.dbdir, fname),
-			encoding=_fs_encoding, errors='strict'), 
-			mode='r', encoding=_content_encoding, errors='replace').read()
+			encoding=_encodings['fs'], errors='strict'), 
+			mode='r', encoding=_encodings['repo.content'], errors='replace'
+			).read()
 
 	def setfile(self,fname,data):
-		mode = 'w'
+		kwargs = {}
 		if fname == 'environment.bz2' or not isinstance(data, basestring):
-			mode = 'wb'
-		write_atomic(os.path.join(self.dbdir, fname), data, mode=mode)
+			kwargs['mode'] = 'wb'
+		else:
+			kwargs['mode'] = 'w'
+			kwargs['encoding'] = _encodings['repo.content']
+		write_atomic(os.path.join(self.dbdir, fname), data, **kwargs)
 
 	def getelements(self,ename):
 		if not os.path.exists(self.dbdir+"/"+ename):
 			return []
 		mylines = codecs.open(_unicode_encode(
 			os.path.join(self.dbdir, ename),
-			encoding=_fs_encoding, errors='strict'),
-			mode='r', encoding=_content_encoding, errors='replace').readlines()
+			encoding=_encodings['fs'], errors='strict'),
+			mode='r', encoding=_encodings['repo.content'], errors='replace'
+			).readlines()
 		myreturn = []
 		for x in mylines:
 			for y in x[:-1].split():
@@ -4221,8 +4254,9 @@ class dblink(object):
 	def setelements(self,mylist,ename):
 		myelement = codecs.open(_unicode_encode(
 			os.path.join(self.dbdir, ename),
-			encoding=_fs_encoding, errors='strict'), mode='w',
-			encoding=_content_encoding, errors='replace')
+			encoding=_encodings['fs'], errors='strict'),
+			mode='w', encoding=_encodings['repo.content'],
+			errors='backslashreplace')
 		for x in mylist:
 			myelement.write(x+"\n")
 		myelement.close()
@@ -4300,7 +4334,8 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None):
 				tarinfo.size = 0
 				tar.addfile(tarinfo)
 			else:
-				f = open(_unicode_encode(path, encoding=_merge_encoding), 'rb')
+				f = open(_unicode_encode(path,
+					encoding=_encodings['merge'], errors='strict'), 'rb')
 				try:
 					tar.addfile(tarinfo, f)
 				finally:
diff --git a/pym/portage/util.py b/pym/portage/util.py
index 521429aad..036191040 100644
--- a/pym/portage/util.py
+++ b/pym/portage/util.py
@@ -23,8 +23,7 @@ import sys
 
 import portage
 from portage import os
-from portage import _content_encoding
-from portage import _fs_encoding
+from portage import _encodings
 from portage import _os_merge
 from portage import _unicode_encode
 from portage import _unicode_decode
@@ -328,8 +327,8 @@ def grablines(myfilename,recursive=0):
 	else:
 		try:
 			myfile = codecs.open(_unicode_encode(myfilename,
-				encoding=_fs_encoding, errors='strict'),
-				mode='r', encoding=_content_encoding, errors='replace')
+				encoding=_encodings['fs'], errors='strict'),
+				mode='r', encoding=_encodings['content'], errors='replace')
 			mylines = myfile.readlines()
 			myfile.close()
 		except IOError, e:
@@ -396,11 +395,11 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True):
 		# (produces spurious \0 characters with python-2.6.2)
 		if sys.hexversion < 0x3000000:
 			content = open(_unicode_encode(mycfg,
-				encoding=_fs_encoding, errors='strict'), 'rb').read()
+				encoding=_encodings['fs'], errors='strict'), 'rb').read()
 		else:
 			content = open(_unicode_encode(mycfg,
-				encoding=_fs_encoding, errors='strict'), mode='r',
-				encoding=_content_encoding, errors='replace').read()
+				encoding=_encodings['fs'], errors='strict'), mode='r',
+				encoding=_encodings['content'], errors='replace').read()
 		if content and content[-1] != '\n':
 			content += '\n'
 	except IOError, e:
@@ -593,7 +592,7 @@ def pickle_read(filename,default=None,debug=0):
 	data = None
 	try:
 		myf = open(_unicode_encode(filename,
-			encoding=_fs_encoding, errors='strict'), 'rb')
+			encoding=_encodings['fs'], errors='strict'), 'rb')
 		mypickle = pickle.Unpickler(myf)
 		data = mypickle.load()
 		myf.close()
@@ -908,8 +907,8 @@ class atomic_ofstream(ObjectProxy):
 			open_func = open
 		else:
 			open_func = codecs.open
-			kargs.setdefault('encoding', _content_encoding)
-			kargs.setdefault('errors', 'replace')
+			kargs.setdefault('encoding', _encodings['content'])
+			kargs.setdefault('errors', 'backslashreplace')
 
 		if follow_links:
 			canonical_path = os.path.realpath(filename)
@@ -918,7 +917,7 @@ class atomic_ofstream(ObjectProxy):
 			try:
 				object.__setattr__(self, '_file',
 					open_func(_unicode_encode(tmp_name,
-						encoding=_fs_encoding, errors='strict'),
+						encoding=_encodings['fs'], errors='strict'),
 						mode=mode, **kargs))
 				return
 			except IOError, e:
@@ -932,7 +931,7 @@ class atomic_ofstream(ObjectProxy):
 		tmp_name = "%s.%i" % (filename, os.getpid())
 		object.__setattr__(self, '_file',
 			open_func(_unicode_encode(tmp_name,
-				encoding=_fs_encoding, errors='strict'),
+				encoding=_encodings['fs'], errors='strict'),
 				mode=mode, **kargs))
 
 	def _get_target(self):