Add a new portage.util.shlex_split() function to wrap all shlex.split() calls,

since shlex.split() doesn't behave well with unicode strings.

svn path=/main/trunk/; revision=13939

11 files changed, 42 insertions, 32 deletions

diff --git a/bin/portageq b/bin/portageq
index 825152911..084c20cbb 100755
--- a/bin/portageq
+++ b/bin/portageq
@@ -251,12 +251,12 @@ def is_protected(argv):
 		err.flush()
 		return 2
 
-	import shlex
 	from portage.util import ConfigProtect
 
 	settings = portage.settings
-	protect = shlex.split(settings.get("CONFIG_PROTECT", ""))
-	protect_mask = shlex.split(settings.get("CONFIG_PROTECT_MASK", ""))
+	protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))
+	protect_mask = portage.util.shlex_split(
+		settings.get("CONFIG_PROTECT_MASK", ""))
 	protect_obj = ConfigProtect(root, protect, protect_mask)
 
 	if protect_obj.isprotected(f):
@@ -284,12 +284,12 @@ def filter_protected(argv):
 	except OSError:
 		pass
 
-	import shlex
 	from portage.util import ConfigProtect
 
 	settings = portage.settings
-	protect = shlex.split(settings.get("CONFIG_PROTECT", ""))
-	protect_mask = shlex.split(settings.get("CONFIG_PROTECT_MASK", ""))
+	protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))
+	protect_mask = portage.util.shlex_split(
+		settings.get("CONFIG_PROTECT_MASK", ""))
 	protect_obj = ConfigProtect(root, protect, protect_mask)
 
 	protected = 0
diff --git a/pym/_emerge/BinpkgFetcher.py b/pym/_emerge/BinpkgFetcher.py
index 977ccb4d8..f3b11bb6c 100644
--- a/pym/_emerge/BinpkgFetcher.py
+++ b/pym/_emerge/BinpkgFetcher.py
@@ -5,7 +5,7 @@
 from _emerge.SpawnProcess import SpawnProcess
 import urlparse
 import sys
-import shlex
+
 try:
 	import portage
 except ImportError:
@@ -83,7 +83,7 @@ class BinpkgFetcher(SpawnProcess):
 
 		fetch_env = dict(settings.iteritems())
 		fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \
-			for x in shlex.split(fcmd)]
+			for x in portage.util.shlex_split(fcmd)]
 
 		if self.fd_pipes is None:
 			self.fd_pipes = {}
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 94ec37e8a..4632c10f2 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -9,7 +9,6 @@ import os
 import platform
 import pwd
 import re
-import shlex
 import signal
 import socket
 import stat
@@ -1833,8 +1832,8 @@ def action_sync(settings, trees, mtimedb, myopts, myaction):
 			# defaults.
 
 			portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
-			rsync_opts.extend(
-				shlex.split(settings.get("PORTAGE_RSYNC_OPTS","")))
+			rsync_opts.extend(portage.util.shlex_split(
+				settings.get("PORTAGE_RSYNC_OPTS", "")))
 			for opt in ("--recursive", "--times"):
 				if opt not in rsync_opts:
 					portage.writemsg(yellow("WARNING:") + " adding required option " + \
@@ -1912,7 +1911,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction):
 			user_name=""
 		updatecache_flg=True
 		all_rsync_opts = set(rsync_opts)
-		extra_rsync_opts = shlex.split(
+		extra_rsync_opts = portage.util.shlex_split(
 			settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
 		all_rsync_opts.update(extra_rsync_opts)
 		family = socket.AF_INET
diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py
index e0e66d8ac..78ac776f5 100644
--- a/pym/_emerge/main.py
+++ b/pym/_emerge/main.py
@@ -3,7 +3,6 @@
 # $Id$
 
 import logging
-import shlex
 import signal
 import sys
 import textwrap
@@ -779,7 +778,7 @@ def ionice(settings):
 
 	ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND")
 	if ionice_cmd:
-		ionice_cmd = shlex.split(ionice_cmd)
+		ionice_cmd = portage.util.shlex_split(ionice_cmd)
 	if not ionice_cmd:
 		return
 
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 5b18fd056..df7961b14 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -3848,9 +3848,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 		# no digests because fetch was not called for a specific package
 		mydigests = {}
 
-	import shlex
 	ro_distdirs = [x for x in \
-		shlex.split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
+		util.shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
 		if os.path.isdir(x)]
 
 	fsmirrors = []
@@ -4408,8 +4407,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 						"URI":     loc,
 						"FILE":    myfile
 					}
-					import shlex
-					myfetch = shlex.split(locfetch)
+
+					myfetch = util.shlex_split(locfetch)
 					myfetch = [varexpand(x, mydict=variables) for x in myfetch]
 					myret = -1
 					try:
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 52d48a70f..dc1180c54 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -40,7 +40,6 @@ from portage.cache.mappings import slot_dict_class
 import codecs
 import os, re, shutil, stat, errno, copy, subprocess
 import logging
-import shlex
 import sys
 from itertools import izip
 
@@ -1809,8 +1808,9 @@ class dblink(object):
 
 		self.myroot=myroot
 		protect_obj = ConfigProtect(myroot,
-			shlex.split(mysettings.get("CONFIG_PROTECT", "")),
-			shlex.split(mysettings.get("CONFIG_PROTECT_MASK", "")))
+			portage.util.shlex_split(mysettings.get("CONFIG_PROTECT", "")),
+			portage.util.shlex_split(
+			mysettings.get("CONFIG_PROTECT_MASK", "")))
 		self.updateprotect = protect_obj.updateprotect
 		self.isprotected = protect_obj.isprotected
 		self._installed_instance = None
@@ -2907,7 +2907,8 @@ class dblink(object):
 
 	def _collision_protect(self, srcroot, destroot, mypkglist, mycontents):
 			collision_ignore = set([normalize_path(myignore) for myignore in \
-				shlex.split(self.settings.get("COLLISION_IGNORE", ""))])
+				portage.util.shlex_split(
+				self.settings.get("COLLISION_IGNORE", ""))])
 
 			# For collisions with preserved libraries, the current package
 			# will assume ownership and the libraries will be unregistered.
diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py
index 29e2ff4f8..75d997e54 100644
--- a/pym/portage/getbinpkg.py
+++ b/pym/portage/getbinpkg.py
@@ -5,7 +5,7 @@
 
 from portage.output import colorize
 from portage.cache.mappings import slot_dict_class
-import portage.xpak
+import portage
 import HTMLParser
 import sys
 import os
@@ -400,10 +400,10 @@ def file_get(baseurl,dest,conn=None,fcmd=None):
 		"URI":     baseurl,
 		"FILE":    os.path.basename(baseurl)
 	}
-	import shlex
+
 	from portage.util import varexpand
 	from portage.process import spawn
-	myfetch = shlex.split(fcmd)
+	myfetch = portage.util.shlex_split(fcmd)
 	myfetch = [varexpand(x, mydict=variables) for x in myfetch]
 	fd_pipes= {
 		0:sys.stdin.fileno(),
diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py
index d6e78bfb6..0f82bd1c9 100644
--- a/pym/portage/sets/dbapi.py
+++ b/pym/portage/sets/dbapi.py
@@ -7,6 +7,7 @@ from portage.dep import Atom
 from portage.localization import _
 from portage.sets.base import PackageSet
 from portage.sets import SetConfigError, get_boolean
+import portage
 
 __all__ = ["CategorySet", "DowngradeSet",
 	"EverythingSet", "OwnerSet", "VariableSet"]
@@ -84,9 +85,8 @@ class OwnerSet(PackageSet):
 		if not "files" in options:
 			raise SetConfigError(_("no files given"))
 
-		import shlex
 		return cls(vardb=trees["vartree"].dbapi,
-			files=frozenset(shlex.split(options["files"])))
+			files=frozenset(portage.util.shlex_split(options["files"])))
 
 	singleBuilder = classmethod(singleBuilder)
 
diff --git a/pym/portage/sets/libs.py b/pym/portage/sets/libs.py
index 806cb63f6..cbaf57c27 100644
--- a/pym/portage/sets/libs.py
+++ b/pym/portage/sets/libs.py
@@ -6,6 +6,7 @@ from portage.localization import _
 from portage.sets.base import PackageSet
 from portage.sets import get_boolean
 from portage.versions import catpkgsplit
+import portage
 
 class LibraryConsumerSet(PackageSet):
 	_operations = ["merge", "unmerge"]
@@ -47,8 +48,7 @@ class LibraryFileConsumerSet(LibraryConsumerSet):
 		self._setAtoms(self.mapPathsToAtoms(consumers))
 
 	def singleBuilder(cls, options, settings, trees):
-		import shlex
-		files = tuple(shlex.split(options.get("files", "")))
+		files = tuple(portage.util.shlex_split(options.get("files", "")))
 		if not files:
 			raise SetConfigError(_("no files given"))
 		debug = get_boolean(options, "debug", False)
diff --git a/pym/portage/util.py b/pym/portage/util.py
index 5b860218d..0a8035de6 100644
--- a/pym/portage/util.py
+++ b/pym/portage/util.py
@@ -350,6 +350,19 @@ def writedict(mydict,myfilename,writekey=True):
 		return 0
 	return 1
 
+def shlex_split(s):
+	"""
+	This is equivalent to shlex.split but it temporarily encodes unicode
+	strings to bytes since shlex.split() doesn't handle unicode strings.
+	"""
+	is_unicode = isinstance(s, unicode)
+	if is_unicode:
+		s = s.encode('utf_8', 'replace')
+	rval = shlex.split(s)
+	if is_unicode:
+		rval = [unicode(x, encoding='utf_8', errors='replace') for x in rval]
+	return rval
+
 class _tolerant_shlex(shlex.shlex):
 	def sourcehook(self, newfile):
 		try:
diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py
index 6003deeac..9ea5ff467 100644
--- a/pym/repoman/utilities.py
+++ b/pym/repoman/utilities.py
@@ -238,7 +238,7 @@ def format_qa_output(formatter, stats, fails, dofull, dofail, options, qawarning
 def editor_is_executable(editor):
 	"""
 	Given an EDITOR string, validate that it refers to
-	an executable. This uses shlex.split() to split the
+	an executable. This uses shlex_split() to split the
 	first component and do a PATH lookup if necessary.
 
 	@param editor: An EDITOR value from the environment.
@@ -246,8 +246,7 @@ def editor_is_executable(editor):
 	@rtype: bool
 	@returns: True if an executable is found, False otherwise.
 	"""
-	import shlex
-	editor_split = shlex.split(editor)
+	editor_split = util.shlex_split(editor)
 	if not editor_split:
 		return False
 	filename = editor_split[0]