ci: Redo format.sh --all script & backfill lint fixes (#9956)

This commit is contained in:
Barak Michener
2020-08-07 16:49:49 -07:00
committed by GitHub
parent 1d01c668f0
commit 8e76796fd0
147 changed files with 702 additions and 636 deletions
+25 -18
View File
@@ -9,7 +9,6 @@ import ray
@ray.remote
class NewsServer(object):
def __init__(self):
self.conn = sqlite3.connect("newsreader.db")
c = self.conn.cursor()
@@ -25,29 +24,36 @@ class NewsServer(object):
items = []
c = self.conn.cursor()
for item in feed.items:
items.append({"title": item.title,
"link": item.link,
"description": item.description,
"description_text": item.description,
"pubDate": str(item.pub_date)})
c.execute("""INSERT INTO news (title, link, description,
items.append({
"title": item.title,
"link": item.link,
"description": item.description,
"description_text": item.description,
"pubDate": str(item.pub_date)
})
c.execute(
"""INSERT INTO news (title, link, description,
published, feed, liked) values
(?, ?, ?, ?, ?, ?)""", (
item.title, item.link, item.description,
item.pub_date, feed.link, False))
(?, ?, ?, ?, ?, ?)""",
(item.title, item.link, item.description, item.pub_date,
feed.link, False))
self.conn.commit()
return {"channel": {"title": feed.title,
"link": feed.link,
"url": feed.link},
"items": items}
return {
"channel": {
"title": feed.title,
"link": feed.link,
"url": feed.link
},
"items": items
}
def like_item(self, url, is_faved):
c = self.conn.cursor()
if is_faved:
c.execute("UPDATE news SET liked = 1 WHERE link = ?", (url,))
c.execute("UPDATE news SET liked = 1 WHERE link = ?", (url, ))
else:
c.execute("UPDATE news SET liked = 0 WHERE link = ?", (url,))
c.execute("UPDATE news SET liked = 0 WHERE link = ?", (url, ))
self.conn.commit()
@@ -71,8 +77,9 @@ def dispatcher():
result = ray.get(method.remote(*method_args))
return jsonify(result)
else:
return jsonify(
{"error": "method_name '" + method_name + "' not found"})
return jsonify({
"error": "method_name '" + method_name + "' not found"
})
if __name__ == "__main__":
+20 -12
View File
@@ -7,10 +7,13 @@ import ray
import wikipedia
parser = argparse.ArgumentParser()
parser.add_argument("--num-mappers",
help="number of mapper actors used", default=3, type=int)
parser.add_argument("--num-reducers",
help="number of reducer actors used", default=4, type=int)
parser.add_argument(
"--num-mappers", help="number of mapper actors used", default=3, type=int)
parser.add_argument(
"--num-reducers",
help="number of reducer actors used",
default=4,
type=int)
@ray.remote
@@ -47,8 +50,10 @@ class Reducer(object):
word_count_sum = defaultdict(lambda: 0)
# Get the word counts for this Reducer's keys from all of the Mappers
# and aggregate the results.
count_ids = [mapper.get_range.remote(article_index, self.keys)
for mapper in self.mappers]
count_ids = [
mapper.get_range.remote(article_index, self.keys)
for mapper in self.mappers
]
# TODO(rkn): We should process these out of order using ray.wait.
for count_id in count_ids:
for k, v in ray.get(count_id):
@@ -78,8 +83,9 @@ if __name__ == "__main__":
streams.append(Stream([line.strip() for line in f.readlines()]))
# Partition the keys among the reducers.
chunks = np.array_split([chr(i) for i in range(ord("a"), ord("z") + 1)],
args.num_reducers)
chunks = np.array_split([chr(i)
for i in range(ord("a"),
ord("z") + 1)], args.num_reducers)
keys = [[chunk[0], chunk[-1]] for chunk in chunks]
# Create a number of mappers.
@@ -93,12 +99,14 @@ if __name__ == "__main__":
while True:
print("article index = {}".format(article_index))
wordcounts = {}
counts = ray.get([reducer.next_reduce_result.remote(article_index)
for reducer in reducers])
counts = ray.get([
reducer.next_reduce_result.remote(article_index)
for reducer in reducers
])
for count in counts:
wordcounts.update(count)
most_frequent_words = heapq.nlargest(10, wordcounts,
key=wordcounts.get)
most_frequent_words = heapq.nlargest(
10, wordcounts, key=wordcounts.get)
for word in most_frequent_words:
print(" ", word, wordcounts[word])
article_index += 1