Fix encoding issues, esp. with CLI tool

compleat/__init__.py

@@ -1,6 +1,6 @@
 from query import Query
 
-VERSION = (0, 0, 1)
+VERSION = (0, 0, 2)
 __version__ = ".".join(map(str,VERSION))
 
 def suggest(query_string, lang="en"):

compleat/cli.py

@@ -5,6 +5,10 @@ import itertools
 import time
 import sys
 
+encoding = sys.stdin.encoding
+def cli_decode(string):
+    return string.decode(encoding)
+
 parser = argparse.ArgumentParser(
     description="Retreive autocomplete suggetions, and output results as CSV [default] or JSON, or store in a database.")
 
@@ -12,16 +16,19 @@ parser.add_argument("--queries",
     "-q",
     nargs="+",
     required=True,
+    type=cli_decode,
     help="Queries to execute. Space-delimited. If --template is provided, query is first substituted into the template.")
 
 parser.add_argument("--template",
     "-t",
     default="{}",
+    type=cli_decode,
     help="String describing the shape of the query. {}s will be replaced by the values supplied to --queries. Default: '{}'")
 
 parser.add_argument("--languages",
     "-l",
     nargs="+",
+    type=cli_decode,
     default=["en"],
     help="Languages (as two-letter codes) to search with. Default: 'en'")
 
@@ -46,9 +53,9 @@ args = parser.parse_args()
 
 def log_query(query_string, lang):
     if args.silent: return
-    sys.stderr.write("{lang}: {query}\n".format(
-        lang=lang,
-        query=query_string))
+    template = u"{lang}: {query}\n"
+    msg = template.format(lang=lang, query=query_string)
+    sys.stderr.write(msg)
     
 def exec_query(query_string, lang):
     time.sleep(args.wait)

compleat/query.py

@@ -13,12 +13,12 @@ class Query(object):
         self.timestamp = datetime.datetime.now()
         self.rand = str(random.random())
         req = requests.get(self.url)
-        utf8 = req.content.decode("latin-1").encode("utf-8")
-        self.response = json.loads(utf8)
+        self.response = req.json()
 
     @property
     def url(self):
-        escaped = urllib.quote(self.query)
+        encoded = self.query.encode("utf-8")
+        escaped = urllib.quote(encoded)
         return self.URL_TEMPLATE.format(
             query=escaped,
             lang=self.lang)
@@ -46,7 +46,7 @@ class Query(object):
             self.lang,
             self.timestamp.ctime(),
             self.rand
-        ])
+        ]).encode("utf-8")
         return hashlib.md5(_).hexdigest()
 
     @property

setup.py

@@ -7,7 +7,7 @@ if sys.version_info <= (2, 6):
 
 setup(
     name='compleat',
-    version='0.0.1',
+    version='0.0.2',
     description="Fetch autocomplete suggestions from Google Search. Use responsibly. Not affiliated with Google.",
     long_description="",
     classifiers=[