From 78e6f6b713ce7941980143c1e2b63aa845659a21 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Thu, 6 Aug 2009 21:47:56 +0000 Subject: Add a new portage.util.shlex_split() function to wrap all shlex.split() calls, since shlex.split() doesn't behave well with unicode strings. svn path=/main/trunk/; revision=13939 --- bin/portageq | 12 ++++++------ pym/_emerge/BinpkgFetcher.py | 4 ++-- pym/_emerge/actions.py | 7 +++---- pym/_emerge/main.py | 3 +-- pym/portage/__init__.py | 7 +++---- pym/portage/dbapi/vartree.py | 9 +++++---- pym/portage/getbinpkg.py | 6 +++--- pym/portage/sets/dbapi.py | 4 ++-- pym/portage/sets/libs.py | 4 ++-- pym/portage/util.py | 13 +++++++++++++ pym/repoman/utilities.py | 5 ++--- 11 files changed, 42 insertions(+), 32 deletions(-) diff --git a/bin/portageq b/bin/portageq index 825152911..084c20cbb 100755 --- a/bin/portageq +++ b/bin/portageq @@ -251,12 +251,12 @@ def is_protected(argv): err.flush() return 2 - import shlex from portage.util import ConfigProtect settings = portage.settings - protect = shlex.split(settings.get("CONFIG_PROTECT", "")) - protect_mask = shlex.split(settings.get("CONFIG_PROTECT_MASK", "")) + protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", "")) + protect_mask = portage.util.shlex_split( + settings.get("CONFIG_PROTECT_MASK", "")) protect_obj = ConfigProtect(root, protect, protect_mask) if protect_obj.isprotected(f): @@ -284,12 +284,12 @@ def filter_protected(argv): except OSError: pass - import shlex from portage.util import ConfigProtect settings = portage.settings - protect = shlex.split(settings.get("CONFIG_PROTECT", "")) - protect_mask = shlex.split(settings.get("CONFIG_PROTECT_MASK", "")) + protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", "")) + protect_mask = portage.util.shlex_split( + settings.get("CONFIG_PROTECT_MASK", "")) protect_obj = ConfigProtect(root, protect, protect_mask) protected = 0 diff --git a/pym/_emerge/BinpkgFetcher.py b/pym/_emerge/BinpkgFetcher.py index 977ccb4d8..f3b11bb6c 100644 --- a/pym/_emerge/BinpkgFetcher.py +++ b/pym/_emerge/BinpkgFetcher.py @@ -5,7 +5,7 @@ from _emerge.SpawnProcess import SpawnProcess import urlparse import sys -import shlex + try: import portage except ImportError: @@ -83,7 +83,7 @@ class BinpkgFetcher(SpawnProcess): fetch_env = dict(settings.iteritems()) fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \ - for x in shlex.split(fcmd)] + for x in portage.util.shlex_split(fcmd)] if self.fd_pipes is None: self.fd_pipes = {} diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 94ec37e8a..4632c10f2 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -9,7 +9,6 @@ import os import platform import pwd import re -import shlex import signal import socket import stat @@ -1833,8 +1832,8 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): # defaults. portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1) - rsync_opts.extend( - shlex.split(settings.get("PORTAGE_RSYNC_OPTS",""))) + rsync_opts.extend(portage.util.shlex_split( + settings.get("PORTAGE_RSYNC_OPTS", ""))) for opt in ("--recursive", "--times"): if opt not in rsync_opts: portage.writemsg(yellow("WARNING:") + " adding required option " + \ @@ -1912,7 +1911,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): user_name="" updatecache_flg=True all_rsync_opts = set(rsync_opts) - extra_rsync_opts = shlex.split( + extra_rsync_opts = portage.util.shlex_split( settings.get("PORTAGE_RSYNC_EXTRA_OPTS","")) all_rsync_opts.update(extra_rsync_opts) family = socket.AF_INET diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index e0e66d8ac..78ac776f5 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -3,7 +3,6 @@ # $Id$ import logging -import shlex import signal import sys import textwrap @@ -779,7 +778,7 @@ def ionice(settings): ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND") if ionice_cmd: - ionice_cmd = shlex.split(ionice_cmd) + ionice_cmd = portage.util.shlex_split(ionice_cmd) if not ionice_cmd: return diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 5b18fd056..df7961b14 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -3848,9 +3848,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", # no digests because fetch was not called for a specific package mydigests = {} - import shlex ro_distdirs = [x for x in \ - shlex.split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \ + util.shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \ if os.path.isdir(x)] fsmirrors = [] @@ -4408,8 +4407,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", "URI": loc, "FILE": myfile } - import shlex - myfetch = shlex.split(locfetch) + + myfetch = util.shlex_split(locfetch) myfetch = [varexpand(x, mydict=variables) for x in myfetch] myret = -1 try: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 52d48a70f..dc1180c54 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -40,7 +40,6 @@ from portage.cache.mappings import slot_dict_class import codecs import os, re, shutil, stat, errno, copy, subprocess import logging -import shlex import sys from itertools import izip @@ -1809,8 +1808,9 @@ class dblink(object): self.myroot=myroot protect_obj = ConfigProtect(myroot, - shlex.split(mysettings.get("CONFIG_PROTECT", "")), - shlex.split(mysettings.get("CONFIG_PROTECT_MASK", ""))) + portage.util.shlex_split(mysettings.get("CONFIG_PROTECT", "")), + portage.util.shlex_split( + mysettings.get("CONFIG_PROTECT_MASK", ""))) self.updateprotect = protect_obj.updateprotect self.isprotected = protect_obj.isprotected self._installed_instance = None @@ -2907,7 +2907,8 @@ class dblink(object): def _collision_protect(self, srcroot, destroot, mypkglist, mycontents): collision_ignore = set([normalize_path(myignore) for myignore in \ - shlex.split(self.settings.get("COLLISION_IGNORE", ""))]) + portage.util.shlex_split( + self.settings.get("COLLISION_IGNORE", ""))]) # For collisions with preserved libraries, the current package # will assume ownership and the libraries will be unregistered. diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index 29e2ff4f8..75d997e54 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -5,7 +5,7 @@ from portage.output import colorize from portage.cache.mappings import slot_dict_class -import portage.xpak +import portage import HTMLParser import sys import os @@ -400,10 +400,10 @@ def file_get(baseurl,dest,conn=None,fcmd=None): "URI": baseurl, "FILE": os.path.basename(baseurl) } - import shlex + from portage.util import varexpand from portage.process import spawn - myfetch = shlex.split(fcmd) + myfetch = portage.util.shlex_split(fcmd) myfetch = [varexpand(x, mydict=variables) for x in myfetch] fd_pipes= { 0:sys.stdin.fileno(), diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py index d6e78bfb6..0f82bd1c9 100644 --- a/pym/portage/sets/dbapi.py +++ b/pym/portage/sets/dbapi.py @@ -7,6 +7,7 @@ from portage.dep import Atom from portage.localization import _ from portage.sets.base import PackageSet from portage.sets import SetConfigError, get_boolean +import portage __all__ = ["CategorySet", "DowngradeSet", "EverythingSet", "OwnerSet", "VariableSet"] @@ -84,9 +85,8 @@ class OwnerSet(PackageSet): if not "files" in options: raise SetConfigError(_("no files given")) - import shlex return cls(vardb=trees["vartree"].dbapi, - files=frozenset(shlex.split(options["files"]))) + files=frozenset(portage.util.shlex_split(options["files"]))) singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/sets/libs.py b/pym/portage/sets/libs.py index 806cb63f6..cbaf57c27 100644 --- a/pym/portage/sets/libs.py +++ b/pym/portage/sets/libs.py @@ -6,6 +6,7 @@ from portage.localization import _ from portage.sets.base import PackageSet from portage.sets import get_boolean from portage.versions import catpkgsplit +import portage class LibraryConsumerSet(PackageSet): _operations = ["merge", "unmerge"] @@ -47,8 +48,7 @@ class LibraryFileConsumerSet(LibraryConsumerSet): self._setAtoms(self.mapPathsToAtoms(consumers)) def singleBuilder(cls, options, settings, trees): - import shlex - files = tuple(shlex.split(options.get("files", ""))) + files = tuple(portage.util.shlex_split(options.get("files", ""))) if not files: raise SetConfigError(_("no files given")) debug = get_boolean(options, "debug", False) diff --git a/pym/portage/util.py b/pym/portage/util.py index 5b860218d..0a8035de6 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -350,6 +350,19 @@ def writedict(mydict,myfilename,writekey=True): return 0 return 1 +def shlex_split(s): + """ + This is equivalent to shlex.split but it temporarily encodes unicode + strings to bytes since shlex.split() doesn't handle unicode strings. + """ + is_unicode = isinstance(s, unicode) + if is_unicode: + s = s.encode('utf_8', 'replace') + rval = shlex.split(s) + if is_unicode: + rval = [unicode(x, encoding='utf_8', errors='replace') for x in rval] + return rval + class _tolerant_shlex(shlex.shlex): def sourcehook(self, newfile): try: diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py index 6003deeac..9ea5ff467 100644 --- a/pym/repoman/utilities.py +++ b/pym/repoman/utilities.py @@ -238,7 +238,7 @@ def format_qa_output(formatter, stats, fails, dofull, dofail, options, qawarning def editor_is_executable(editor): """ Given an EDITOR string, validate that it refers to - an executable. This uses shlex.split() to split the + an executable. This uses shlex_split() to split the first component and do a PATH lookup if necessary. @param editor: An EDITOR value from the environment. @@ -246,8 +246,7 @@ def editor_is_executable(editor): @rtype: bool @returns: True if an executable is found, False otherwise. """ - import shlex - editor_split = shlex.split(editor) + editor_split = util.shlex_split(editor) if not editor_split: return False filename = editor_split[0] -- cgit v1.2.3-1-g7c22