diff --git a/doc/ChangeLog b/doc/ChangeLog index 340b142..3c2a91f 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,12 @@ +2007-11-09 Jörgen Stenarson + * More fixes to ensure unicode works when completing on filepaths with non ascii symbols. + This fix assumes sys.stdout.encoding is valid for the filesystem. A more correct fix + must be applied to the completer code which is outside pyreadline. + Any str characters fed into pyreadline should be converted to unicode using + unicode_helper.ensure_unicode. + * The encoding assumed of strings is sys.stdout.encoding this can be changed by: + e.g. pyreadline.unicode_helper.pyreadline_codepage="utf8" + 2007-10-30 Jörgen Stenarson * Fixing console.title to work with wide characters * Fixing wide character bug for log_sock diff --git a/pyreadline/__init__.py b/pyreadline/__init__.py index dd31888..cacef00 100644 --- a/pyreadline/__init__.py +++ b/pyreadline/__init__.py @@ -6,7 +6,7 @@ # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. #***************************************************************************** -import logger,clipboard,lineeditor,modes +import unicode_helper,logger,clipboard,lineeditor,modes from rlmain import * __all__ = [ 'parse_and_bind', 'get_line_buffer', diff --git a/pyreadline/configuration/startup.py b/pyreadline/configuration/startup.py index 685a1c7..c425398 100644 --- a/pyreadline/configuration/startup.py +++ b/pyreadline/configuration/startup.py @@ -4,6 +4,12 @@ try: import pyreadline.rlmain #pyreadline.rlmain.config_path=r"c:\xxx\pyreadlineconfig.ini" import readline,atexit + import pyreadline.unicode_helper + # + # + #Normally the codepage for pyreadline is set to be sys.stdout.encoding + #if you need to change this uncomment the following line + #pyreadline.unicode_helper.pyreadline_codepage="utf8" except ImportError: print "Module readline not available." else: diff --git a/pyreadline/console/console.py b/pyreadline/console/console.py index 7133776..577e4f6 100644 --- a/pyreadline/console/console.py +++ b/pyreadline/console/console.py @@ -17,7 +17,8 @@ import sys import traceback import re from pyreadline.logger import log,log_sock - +from pyreadline.unicode_helper import ensure_unicode +import pyreadline.unicode_helper as unicode_helper try: from ctypes import * from _ctypes import call_function @@ -116,18 +117,6 @@ class CONSOLE_CURSOR_INFO(Structure): ("bVisible", c_byte)] -try: - consolecodepage=sys.stdout.encoding -except AttributeError: #This error occurs when pdb imports readline and doctest has replaced - #stdout with stdout collector - consolecodepage="ascii" #assume ascii codepage - -def ensure_text(text): - """helper to ensure that text passed to WriteConsoleA is ascii""" - if isinstance(text, str): - return text.decode(consolecodepage, "replace") - return text - # I didn't want to have to individually import these so I made a list, they are # added to the Console class later in this file. @@ -363,7 +352,7 @@ class Console(object): return n def write_color(self, text, attr=None): - text = ensure_text(text) + text = ensure_unicode(text) n,res= self.ansiwriter.write_color(text,attr) junk = c_int(0) for attr,chunk in res: @@ -388,7 +377,7 @@ class Console(object): attr = self.attr n = c_int(0) self.SetConsoleTextAttribute(self.hout, attr) - self.WriteConsoleW(self.hout, ensure_text(chunk), len(chunk), byref(junk), None) + self.WriteConsoleW(self.hout, ensure_unicode(chunk), len(chunk), byref(junk), None) return len(text) # make this class look like a file object @@ -688,7 +677,7 @@ def hook_wrapper_23(stdin, stdout, prompt): '''Wrap a Python readline so it behaves like GNU readline.''' try: # call the Python hook - res = readline_hook(prompt).encode(consolecodepage) + res = readline_hook(prompt).encode(unicode_helper.pyreadline_codepage) # make sure it returned the right sort of thing if res and not isinstance(res, str): raise TypeError, 'readline must return a string.' @@ -712,7 +701,7 @@ def hook_wrapper(prompt): '''Wrap a Python readline so it behaves like GNU readline.''' try: # call the Python hook - res = readline_hook(prompt).encode(consolecodepage) + res = readline_hook(prompt).encode(unicode_helper.pyreadline_codepage) # make sure it returned the right sort of thing if res and not isinstance(res, str): raise TypeError, 'readline must return a string.' diff --git a/pyreadline/lineeditor/lineobj.py b/pyreadline/lineeditor/lineobj.py index d581e39..79364ce 100644 --- a/pyreadline/lineeditor/lineobj.py +++ b/pyreadline/lineeditor/lineobj.py @@ -5,11 +5,12 @@ # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. #***************************************************************************** -import re,operator +import re,operator,sys import wordmatcher import pyreadline.clipboard as clipboard from pyreadline.logger import log,log_sock +from pyreadline.unicode_helper import ensure_unicode class NotAWordError(IndexError): pass @@ -241,11 +242,13 @@ class TextLine(object): def quoted_text(self): quoted = [ quote_char(c) for c in self.line_buffer ] self.line_char_width = [ len(c) for c in quoted ] - return ''.join(quoted) + return u''.join(map(ensure_unicode,quoted)) def get_line_text(self): - return ''.join(self.line_buffer) - + buf=self.line_buffer + buf=map(ensure_unicode,buf) + return u''.join(buf) + def set_line(self, text, cursor=None): self.line_buffer = [ c for c in str(text) ] if cursor is None: diff --git a/pyreadline/unicode_helper.py b/pyreadline/unicode_helper.py new file mode 100644 index 0000000..ab03e09 --- /dev/null +++ b/pyreadline/unicode_helper.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +#***************************************************************************** +# Copyright (C) 2007 Jorgen Stenarson. +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#***************************************************************************** +import sys + +try: + pyreadline_codepage=sys.stdout.encoding +except AttributeError: #This error occurs when pdb imports readline and doctest has replaced + #stdout with stdout collector + pyreadline_codepage="ascii" #assume ascii codepage + + +def ensure_unicode(text): + """helper to ensure that text passed to WriteConsoleW is unicode""" + if isinstance(text, str): + return text.decode(pyreadline_codepage, "replace") + return text