pyreadline: fixes to make completion work on file paths containing non-ascii characters.

encoding is assumed to be compatible to sys.stdout.encoding. A more correct solution must be applied to the completer function which is outside pyreadline. If another encoding is desired set pyreadline.unicode_helper.pyreadline_codepage

doc/ChangeLog

@@ -1,3 +1,12 @@
+2007-11-09 Jörgen Stenarson  <jorgen.stenarson -at- bostream.nu>
+	* More fixes to ensure unicode works when completing on filepaths with non ascii symbols. 
+	  This fix assumes sys.stdout.encoding is valid for the filesystem. A more correct fix
+	  must be applied to the completer code which is outside pyreadline.
+	  Any str characters fed into pyreadline should be converted to unicode using 
+	  unicode_helper.ensure_unicode.
+	* The encoding assumed of strings is sys.stdout.encoding this can be changed by:
+	  e.g.  pyreadline.unicode_helper.pyreadline_codepage="utf8"
+
 2007-10-30 Jörgen Stenarson  <jorgen.stenarson -at- bostream.nu>
 	* Fixing console.title to work with wide characters
 	* Fixing wide character bug for log_sock

pyreadline/__init__.py

@@ -6,7 +6,7 @@
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
-import logger,clipboard,lineeditor,modes
+import unicode_helper,logger,clipboard,lineeditor,modes
 from rlmain import *
 __all__ = [ 'parse_and_bind',
             'get_line_buffer',

pyreadline/configuration/startup.py

@@ -4,6 +4,12 @@ try:
     import pyreadline.rlmain
     #pyreadline.rlmain.config_path=r"c:\xxx\pyreadlineconfig.ini"
     import readline,atexit
+    import pyreadline.unicode_helper
+    #
+    #
+    #Normally the codepage for pyreadline is set to be sys.stdout.encoding
+    #if you need to change this uncomment the following line
+    #pyreadline.unicode_helper.pyreadline_codepage="utf8"
 except ImportError:
     print "Module readline not available."
 else:

pyreadline/console/console.py

@@ -17,7 +17,8 @@ import sys
 import traceback
 import re
 from pyreadline.logger import log,log_sock
-
+from pyreadline.unicode_helper import ensure_unicode
+import pyreadline.unicode_helper as unicode_helper
 try:
     from ctypes import *
     from _ctypes import call_function
@@ -116,18 +117,6 @@ class CONSOLE_CURSOR_INFO(Structure):
                 ("bVisible", c_byte)]
 
 
-try:
-    consolecodepage=sys.stdout.encoding
-except AttributeError:        #This error occurs when pdb imports readline and doctest has replaced 
-                              #stdout with stdout collector
-    consolecodepage="ascii"   #assume ascii codepage
-    
-def ensure_text(text):
-    """helper to ensure that text passed to WriteConsoleA is ascii"""
-    if isinstance(text, str):
-        return text.decode(consolecodepage, "replace")
-    return text
-
 # I didn't want to have to individually import these so I made a list, they are
 # added to the Console class later in this file.
 
@@ -363,7 +352,7 @@ class Console(object):
         return n
 
     def write_color(self, text, attr=None):
-        text = ensure_text(text)
+        text = ensure_unicode(text)
         n,res= self.ansiwriter.write_color(text,attr)
         junk = c_int(0)
         for attr,chunk in res:
@@ -388,7 +377,7 @@ class Console(object):
             attr = self.attr
         n = c_int(0)
         self.SetConsoleTextAttribute(self.hout, attr)
-        self.WriteConsoleW(self.hout, ensure_text(chunk), len(chunk), byref(junk), None)
+        self.WriteConsoleW(self.hout, ensure_unicode(chunk), len(chunk), byref(junk), None)
         return len(text)
 
     # make this class look like a file object
@@ -688,7 +677,7 @@ def hook_wrapper_23(stdin, stdout, prompt):
     '''Wrap a Python readline so it behaves like GNU readline.'''
     try:
         # call the Python hook
-        res = readline_hook(prompt).encode(consolecodepage)
+        res = readline_hook(prompt).encode(unicode_helper.pyreadline_codepage)
         # make sure it returned the right sort of thing
         if res and not isinstance(res, str):
             raise TypeError, 'readline must return a string.'
@@ -712,7 +701,7 @@ def hook_wrapper(prompt):
     '''Wrap a Python readline so it behaves like GNU readline.'''
     try:
         # call the Python hook
-        res = readline_hook(prompt).encode(consolecodepage)
+        res = readline_hook(prompt).encode(unicode_helper.pyreadline_codepage)
         # make sure it returned the right sort of thing
         if res and not isinstance(res, str):
             raise TypeError, 'readline must return a string.'

pyreadline/lineeditor/lineobj.py

@@ -5,11 +5,12 @@
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
-import re,operator
+import re,operator,sys
 
 import wordmatcher
 import pyreadline.clipboard as clipboard
 from pyreadline.logger import  log,log_sock
+from pyreadline.unicode_helper import ensure_unicode
 class NotAWordError(IndexError):
     pass
 
@@ -241,11 +242,13 @@ class TextLine(object):
     def quoted_text(self):
         quoted = [ quote_char(c) for c in self.line_buffer ]
         self.line_char_width = [ len(c) for c in quoted ]
-        return ''.join(quoted)
+        return u''.join(map(ensure_unicode,quoted))
 
     def get_line_text(self):
-        return ''.join(self.line_buffer)
-
+        buf=self.line_buffer
+        buf=map(ensure_unicode,buf)
+        return u''.join(buf)
+            
     def set_line(self, text, cursor=None):
         self.line_buffer = [ c for c in str(text) ]
         if cursor is None:

pyreadline/unicode_helper.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#*****************************************************************************
+#       Copyright (C) 2007  Jorgen Stenarson. <jorgen.stenarson@bostream.nu>
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#*****************************************************************************
+import sys
+
+try:
+    pyreadline_codepage=sys.stdout.encoding
+except AttributeError:        #This error occurs when pdb imports readline and doctest has replaced 
+                              #stdout with stdout collector
+    pyreadline_codepage="ascii"   #assume ascii codepage
+    
+
+def ensure_unicode(text):
+    """helper to ensure that text passed to WriteConsoleW is unicode"""
+    if isinstance(text, str):
+        return text.decode(pyreadline_codepage, "replace")
+    return text