From 6d8d28a59a62472afc684926cbb0427139d61a01 Mon Sep 17 00:00:00 2001 From: Jeremy Singer-Vine Date: Fri, 31 Jan 2014 02:20:55 -0500 Subject: [PATCH] Extract output functions from cli.py, reorganize. --- compleat/__init__.py | 2 +- compleat/{output => }/cli.py | 36 ++---------------------------------- compleat/output.py | 31 +++++++++++++++++++++++++++++++ compleat/output/__init__.py | 0 setup.py | 4 ++-- 5 files changed, 36 insertions(+), 37 deletions(-) rename compleat/{output => }/cli.py (64%) create mode 100644 compleat/output.py delete mode 100644 compleat/output/__init__.py diff --git a/compleat/__init__.py b/compleat/__init__.py index 246606e..3e25bdb 100644 --- a/compleat/__init__.py +++ b/compleat/__init__.py @@ -1,6 +1,6 @@ from query import Query -VERSION = (0, 0, 0) +VERSION = (0, 0, 1) __version__ = ".".join(map(str,VERSION)) def suggest(query_string, lang="en"): diff --git a/compleat/output/cli.py b/compleat/cli.py similarity index 64% rename from compleat/output/cli.py rename to compleat/cli.py index 18ad9e7..92c4dd2 100644 --- a/compleat/output/cli.py +++ b/compleat/cli.py @@ -1,9 +1,9 @@ import compleat +from compleat.output import to_db, to_csv, to_json import argparse import itertools import time import sys -import codecs parser = argparse.ArgumentParser( description="Retreive autocomplete suggetions, and output results as CSV [default] or JSON, or store in a database.") @@ -44,38 +44,6 @@ parser.add_argument("--wait", args = parser.parse_args() -def to_db(db_str, queries): - import dataset - db = dataset.connect(args.db) - query_table = db.get_table("queries") - suggestion_table = db.get_table("suggestions") - for q in queries: - query_table.insert(q.meta) - suggestion_table.insert_many([ - dict(s.items() + { - "query_uid": q.uid - }.items()) for s in q.suggestions - ]) - -def to_csv(queries): - import unicodecsv - fieldnames = queries[0].meta.keys() + \ - queries[0].suggestions[0].keys() - writer = unicodecsv.DictWriter(sys.stdout, fieldnames) - writer.writeheader() - for q in queries: - for s in q.suggestions: - row_dict = dict(q.meta.items() + s.items()) - writer.writerow(row_dict) - -def to_json(queries): - import json - def convert(query): - sugg_tuple = ("suggestions", query.suggestions) - return dict(query.meta.items() + [ sugg_tuple ]) - obj = map(convert, queries) - json.dump(obj, sys.stdout) - def log_query(query_string, lang): if args.silent: return sys.stderr.write("{lang}: {query}\n".format( @@ -92,7 +60,7 @@ def main(): templated = map(args.template.format, args.queries) combos = itertools.product(templated, args.languages) queries = [ exec_query(*c) for c in combos ] - if args.db: to_db(args.db, queries) + if args.db: to_db(queries, args.db) elif args.json: to_json(queries) else: to_csv(queries) diff --git a/compleat/output.py b/compleat/output.py new file mode 100644 index 0000000..eac5681 --- /dev/null +++ b/compleat/output.py @@ -0,0 +1,31 @@ +def to_db(queries, db_str): + import dataset + db = dataset.connect(db_str) + query_table = db.get_table("queries") + suggestion_table = db.get_table("suggestions") + for q in queries: + query_table.insert(q.meta) + suggestion_table.insert_many([ + dict(s.items() + { + "query_uid": q.uid + }.items()) for s in q.suggestions + ]) + +def to_csv(queries): + import unicodecsv, sys + fieldnames = queries[0].meta.keys() + \ + queries[0].suggestions[0].keys() + writer = unicodecsv.DictWriter(sys.stdout, fieldnames) + writer.writeheader() + for q in queries: + for s in q.suggestions: + row_dict = dict(q.meta.items() + s.items()) + writer.writerow(row_dict) + +def to_json(queries): + import json + def convert(query): + sugg_tuple = ("suggestions", query.suggestions) + return dict(query.meta.items() + [ sugg_tuple ]) + obj = map(convert, queries) + json.dump(obj, sys.stdout) diff --git a/compleat/output/__init__.py b/compleat/output/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index 8054d45..c5de011 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ if sys.version_info <= (2, 6): setup( name='compleat', - version='0.0.0', + version='0.0.1', description="Fetch autocomplete suggestions from Google Search. Use responsibly. Not affiliated with Google.", long_description="", classifiers=[ @@ -39,7 +39,7 @@ setup( test_suite='test', entry_points={ 'console_scripts': [ - 'compleat = compleat.output.cli:main', + 'compleat = compleat.cli:main', ] } )