From a07ec4af4a787457edb663889f4b989aeff16ac9 Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 13:47:30 -0700 Subject: [PATCH 01/18] added telemetry --- play.py | 17 +++++++++++++++++ story/story_manager.py | 20 +++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/play.py b/play.py index ef3f419..f1cd89d 100644 --- a/play.py +++ b/play.py @@ -41,6 +41,12 @@ def play_aidungeon_2(): starter = file.read() print(starter) + save_story = input("Help improve AIDungeon by enabling story saving? (Y/n)") + if save_story.lower() in ["no", "No", "n"]: + upload_story = True + else: + upload_story = True + while True: print("\n\n") @@ -55,6 +61,13 @@ def play_aidungeon_2(): tcflush(sys.stdin, TCIFLUSH) action = input("> ") if action == "restart": + if upload_story: + rating = input("Please rate the story quality from 1-10: ") + try: + rating_float = float(rating) + story_manager.story.rating = rating_float + except: + pass break if action != "" and action.lower() != "continue": @@ -73,6 +86,10 @@ def play_aidungeon_2(): #action = first_to_second_person(action) result = "\n" + story_manager.act(action) + + if upload_story: + story_manager.story.save_to_storage() + if player_died(result): console_print(result + "\nGAME OVER") break diff --git a/story/story_manager.py b/story/story_manager.py index a0c3b25..f473a5b 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -1,12 +1,14 @@ from story.utils import * import json - +import uuid +from google.cloud import storage class Story(): def __init__(self, story_start, context ="", seed=None, game_state=None): self.story_start = story_start self.context = context + self.rating = -1 # list of actions. First action is the prompt length should always equal that of story blocks self.actions = [] @@ -18,6 +20,7 @@ class Story(): self.seed = seed self.choices = [] self.possible_action_results = None + self.uuid = uuid.uuid1() if game_state is None: game_state = dict() @@ -35,6 +38,12 @@ class Story(): self.possible_action_results = story_dict["possible_action_results"] self.game_state = story_dict["game_state"] self.context = story_dict["context"] + self.uuid = story_dict["uuid"] + + if "rating" in story_dict.keys(): + self.rating = story_dict["rating"] + else: + self.rating = -1 def add_to_story(self, action, story_block): self.actions.append(action) @@ -75,9 +84,18 @@ class Story(): story_dict["possible_action_results"] = self.possible_action_results story_dict["game_state"] = self.game_state story_dict["context"] = self.context + story_dict["uuid"] = self.uuid + story_dict["rating"] = self.rating return json.dumps(story_dict) + def save_to_storage(self): + client = storage.Client() + # https://console.cloud.google.com/storage/browser/[bucket-id]/ + bucket = client.get_bucket('aidungeon2stories') + blob = bucket.blob("story" + str(self.uuid) + ".json") + blob.upload_from_string(self.to_json()) + class StoryManager(): From da56d389d2b44067622395d3668a75c7ead3a517 Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 13:50:25 -0700 Subject: [PATCH 02/18] added telemetry --- play.py | 2 +- story/story_data.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/play.py b/play.py index f1cd89d..9fa59ba 100644 --- a/play.py +++ b/play.py @@ -41,7 +41,7 @@ def play_aidungeon_2(): starter = file.read() print(starter) - save_story = input("Help improve AIDungeon by enabling story saving? (Y/n)") + save_story = input("Help improve AIDungeon by enabling story saving? (Y/n) ") if save_story.lower() in ["no", "No", "n"]: upload_story = True else: diff --git a/story/story_data.yaml b/story/story_data.yaml index 0416584..e7ecbb3 100644 --- a/story/story_data.yaml +++ b/story/story_data.yaml @@ -78,19 +78,19 @@ custom: item2: "spellbook" ranger: - prompt: ["You spot the deer and are ready to finish your hunt when suddenly"] + prompts: ["You spot the deer and are ready to finish your hunt when suddenly"] item1: "hunting bow" item2: "quiver of arrows" peasant: - prompt: ["You wake up and begin working in the fields. You see"] + prompts: ["You wake up and begin working in the fields. You see"] item1: "pitchfork" item2: "nothing else" rogue: - prompt: ["You successfully sneak into the merchant's house. You look around and see"] + prompts: ["You successfully sneak into the merchant's house. You look around and see"] item1: "long steel dagger" item2: "length of rope" From d919450161713ac95433303c32d5dc66a602504e Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 13:57:02 -0700 Subject: [PATCH 03/18] added telemetry --- story/story_manager.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/story/story_manager.py b/story/story_manager.py index f473a5b..aec8be1 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -1,7 +1,7 @@ from story.utils import * import json import uuid -from google.cloud import storage +from subprocess import Popen class Story(): @@ -90,11 +90,12 @@ class Story(): return json.dumps(story_dict) def save_to_storage(self): - client = storage.Client() - # https://console.cloud.google.com/storage/browser/[bucket-id]/ - bucket = client.get_bucket('aidungeon2stories') - blob = bucket.blob("story" + str(self.uuid) + ".json") - blob.upload_from_string(self.to_json()) + story_json = self.to_json() + file_name = "story" + str(self.uuid) + ".json" + f = open(file_name, "w") + f.write(story_json) + f.close() + p = Popen(['gsutil', 'cp', 'file_name', 'aidungeon2stories']) class StoryManager(): From 378829cd4c8f2bb3aff1a1131373d9218c07a4ed Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 14:00:52 -0700 Subject: [PATCH 04/18] added telemetry --- play.py | 8 ++++---- story/story_manager.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/play.py b/play.py index 9fa59ba..7c88040 100644 --- a/play.py +++ b/play.py @@ -37,16 +37,16 @@ def play_aidungeon_2(): story_manager = UnconstrainedStoryManager(generator) print("\n\n\n\n") - with open('opening.txt', 'r') as file: - starter = file.read() - print(starter) - save_story = input("Help improve AIDungeon by enabling story saving? (Y/n) ") if save_story.lower() in ["no", "No", "n"]: upload_story = True else: upload_story = True + with open('opening.txt', 'r') as file: + starter = file.read() + print(starter) + while True: print("\n\n") diff --git a/story/story_manager.py b/story/story_manager.py index aec8be1..e52612f 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -20,7 +20,7 @@ class Story(): self.seed = seed self.choices = [] self.possible_action_results = None - self.uuid = uuid.uuid1() + self.uuid = str(uuid.uuid1()) if game_state is None: game_state = dict() From b2d8e13ebd0d99108871f62efe76ebf1898b7758 Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 14:04:58 -0700 Subject: [PATCH 05/18] added telemetry --- play.py | 10 +++++----- story/story_manager.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/play.py b/play.py index 7c88040..903ecb4 100644 --- a/play.py +++ b/play.py @@ -32,17 +32,17 @@ def instructions(): def play_aidungeon_2(): - print("Initializing AI Dungeon! (This might take a few minutes)") - generator = GPT2Generator() - story_manager = UnconstrainedStoryManager(generator) - print("\n\n\n\n") - save_story = input("Help improve AIDungeon by enabling story saving? (Y/n) ") if save_story.lower() in ["no", "No", "n"]: upload_story = True else: upload_story = True + print("Initializing AI Dungeon! (This might take a few minutes)") + generator = GPT2Generator() + story_manager = UnconstrainedStoryManager(generator) + print("\n\n\n\n") + with open('opening.txt', 'r') as file: starter = file.read() print(starter) diff --git a/story/story_manager.py b/story/story_manager.py index e52612f..d4ac1d3 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -95,7 +95,7 @@ class Story(): f = open(file_name, "w") f.write(story_json) f.close() - p = Popen(['gsutil', 'cp', 'file_name', 'aidungeon2stories']) + p = Popen(['gsutil', 'cp', file_name, 'aidungeon2stories']) class StoryManager(): From 6b819d4b1f0afccbe2dd59ffe0d7808f6eb89a49 Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 14:07:31 -0700 Subject: [PATCH 06/18] added telemetry --- play.py | 2 +- story/story_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/play.py b/play.py index 903ecb4..bd777fd 100644 --- a/play.py +++ b/play.py @@ -38,7 +38,7 @@ def play_aidungeon_2(): else: upload_story = True - print("Initializing AI Dungeon! (This might take a few minutes)") + print("\nInitializing AI Dungeon! (This might take a few minutes)\n") generator = GPT2Generator() story_manager = UnconstrainedStoryManager(generator) print("\n\n\n\n") diff --git a/story/story_manager.py b/story/story_manager.py index d4ac1d3..dc9ca9d 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -95,7 +95,7 @@ class Story(): f = open(file_name, "w") f.write(story_json) f.close() - p = Popen(['gsutil', 'cp', file_name, 'aidungeon2stories']) + p = Popen(['gsutil', 'cp', file_name, 'aidungeon2stories', ">", "/dev/null"]) class StoryManager(): From f6bbe88aab28e36ee7c3132720627ed5da5b5d2c Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 14:11:35 -0700 Subject: [PATCH 07/18] added telemetry --- story/story_manager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/story/story_manager.py b/story/story_manager.py index dc9ca9d..2118101 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -2,6 +2,8 @@ from story.utils import * import json import uuid from subprocess import Popen +import subprocess +import os class Story(): @@ -95,7 +97,9 @@ class Story(): f = open(file_name, "w") f.write(story_json) f.close() - p = Popen(['gsutil', 'cp', file_name, 'aidungeon2stories', ">", "/dev/null"]) + + FNULL = open(os.devnull, 'w') + p = Popen(['gsutil', 'cp', file_name, 'aidungeon2stories'], stdout=FNULL, stderr=subprocess.STDOUT) class StoryManager(): From 41f4836e269eeea8717c960bbffae20f41c85acd Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 14:17:55 -0700 Subject: [PATCH 08/18] added telemetry --- story/story_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/story/story_manager.py b/story/story_manager.py index 2118101..7297e5c 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -99,7 +99,7 @@ class Story(): f.close() FNULL = open(os.devnull, 'w') - p = Popen(['gsutil', 'cp', file_name, 'aidungeon2stories'], stdout=FNULL, stderr=subprocess.STDOUT) + p = Popen(['gsutil', 'cp', file_name, 'aidungeonstories'], stdout=FNULL, stderr=subprocess.STDOUT) class StoryManager(): From f7496fae3d5bcdd033c2cb0e2a88d3785ab5917e Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 14:24:54 -0700 Subject: [PATCH 09/18] added telemetry --- story/story_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/story/story_manager.py b/story/story_manager.py index 7297e5c..1ab306e 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -99,7 +99,7 @@ class Story(): f.close() FNULL = open(os.devnull, 'w') - p = Popen(['gsutil', 'cp', file_name, 'aidungeonstories'], stdout=FNULL, stderr=subprocess.STDOUT) + p = Popen(['gsutil', 'cp', file_name, 'gs://aidungeonstories'], stdout=FNULL, stderr=subprocess.STDOUT) class StoryManager(): From 8a200b62b2cedd396b4ddde93d3054f47eadc30d Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 14:30:14 -0700 Subject: [PATCH 10/18] added telemetry --- play.py | 1 + 1 file changed, 1 insertion(+) diff --git a/play.py b/play.py index bd777fd..529b169 100644 --- a/play.py +++ b/play.py @@ -66,6 +66,7 @@ def play_aidungeon_2(): try: rating_float = float(rating) story_manager.story.rating = rating_float + story_manager.story.save_to_storage() except: pass break From 2ce06037f8a794c454ca9f45c68156d709511d09 Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 14:40:58 -0700 Subject: [PATCH 11/18] added telemetry --- generator/gpt2/gpt2_generator.py | 2 +- generator/gpt2/src/sample.py | 2 +- play.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/generator/gpt2/gpt2_generator.py b/generator/gpt2/gpt2_generator.py index 66a0813..f64deef 100644 --- a/generator/gpt2/gpt2_generator.py +++ b/generator/gpt2/gpt2_generator.py @@ -10,7 +10,7 @@ import numpy as np class GPT2Generator: - def __init__(self, generate_num=60, temperature=0.3, top_k=40, top_p=0.9): + def __init__(self, generate_num=60, temperature=0.4, top_k=40, top_p=0.9): self.generate_num=generate_num self.temp = temperature self.top_k = top_k diff --git a/generator/gpt2/src/sample.py b/generator/gpt2/src/sample.py index 5879d11..f50eaf8 100644 --- a/generator/gpt2/src/sample.py +++ b/generator/gpt2/src/sample.py @@ -17,7 +17,7 @@ def penalize_used(logits, output): return tf.compat.v1.where( bool_tensor, - logits / 1.2, + logits * 0.85, logits) diff --git a/play.py b/play.py index 529b169..20dd1a1 100644 --- a/play.py +++ b/play.py @@ -34,7 +34,7 @@ def play_aidungeon_2(): save_story = input("Help improve AIDungeon by enabling story saving? (Y/n) ") if save_story.lower() in ["no", "No", "n"]: - upload_story = True + upload_story = False else: upload_story = True From 47869ed9ab0a2ac7f29bebd6ba678f93494af415 Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 14:58:21 -0700 Subject: [PATCH 12/18] update --- play.py | 32 +++++++++++++++++++++----------- story/story_manager.py | 11 ++++++++++- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/play.py b/play.py index 20dd1a1..b5039f1 100644 --- a/play.py +++ b/play.py @@ -29,6 +29,15 @@ def instructions(): text += '\n* Finally if you want to end your game and start a new one just enter "restart" for any action. ' return text +def upload_story_to_cloud(story): + rating = input("Please rate the story quality from 1-10: ") + try: + rating_float = float(rating) + story.rating = rating_float + story.save_to_storage() + except: + pass + def play_aidungeon_2(): @@ -52,8 +61,9 @@ def play_aidungeon_2(): print("\n\n") context, prompt = select_game() console_print(instructions()) + print("\nGenerating story...") - story_manager.start_new_story(prompt, context=context) + story_manager.start_new_story(prompt, context=context, upload_story=upload_story) print("\n") console_print(context + str(story_manager.story)) @@ -62,14 +72,12 @@ def play_aidungeon_2(): action = input("> ") if action == "restart": if upload_story: - rating = input("Please rate the story quality from 1-10: ") - try: - rating_float = float(rating) - story_manager.story.rating = rating_float - story_manager.story.save_to_storage() - except: - pass + upload_story_to_cloud(story_manager.story) break + elif action == "quit": + if upload_story: + upload_story_to_cloud(story_manager.story) + exit() if action != "" and action.lower() != "continue": action = action.strip() @@ -88,14 +96,16 @@ def play_aidungeon_2(): result = "\n" + story_manager.act(action) - if upload_story: - story_manager.story.save_to_storage() - if player_died(result): console_print(result + "\nGAME OVER") + if upload_story: + upload_story_to_cloud(story_manager.story) break elif player_won(result): console_print(result + "\n CONGRATS YOU WIN") + if upload_story: + upload_story_to_cloud(story_manager.story) + break else: console_print(result) diff --git a/story/story_manager.py b/story/story_manager.py index 1ab306e..3f5c41e 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -7,10 +7,11 @@ import os class Story(): - def __init__(self, story_start, context ="", seed=None, game_state=None): + def __init__(self, story_start, context ="", seed=None, game_state=None, upload_story=False): self.story_start = story_start self.context = context self.rating = -1 + self.upload_story = upload_story # list of actions. First action is the prompt length should always equal that of story blocks self.actions = [] @@ -29,6 +30,14 @@ class Story(): self.game_state = game_state self.memory = 10 + def __del__(self): + rating = input("Please rate the story quality from 1-10: ") + try: + rating_float = float(rating) + self.rating = rating_float + self.save_to_storage() + except: + pass def initialize_from_json(self, json_string): story_dict = json.loads(json_string) From d7eea8c21083e4dcea0092a827085ced5acdb1f3 Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 15:00:02 -0700 Subject: [PATCH 13/18] update --- story/story_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/story/story_manager.py b/story/story_manager.py index 3f5c41e..6fc6528 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -116,10 +116,10 @@ class StoryManager(): def __init__(self, generator): self.generator = generator - def start_new_story(self, story_prompt, context="", game_state=None): + def start_new_story(self, story_prompt, context="", game_state=None, upload_story=False): block = self.generator.generate(context + story_prompt) block = cut_trailing_sentence(block) - self.story = Story(story_prompt + block, context=context, game_state=game_state) + self.story = Story(story_prompt + block, context=context, game_state=game_state, upload_story=upload_story) return self.story def load_story(self, story, from_json=False): From eceea2254467af2ab6e2fc2d0efd2464cd6abb4e Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 17:35:18 -0700 Subject: [PATCH 14/18] fixed bug --- play.py | 21 +++------------------ story/story_manager.py | 2 ++ 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/play.py b/play.py index b5039f1..98b5676 100644 --- a/play.py +++ b/play.py @@ -29,16 +29,6 @@ def instructions(): text += '\n* Finally if you want to end your game and start a new one just enter "restart" for any action. ' return text -def upload_story_to_cloud(story): - rating = input("Please rate the story quality from 1-10: ") - try: - rating_float = float(rating) - story.rating = rating_float - story.save_to_storage() - except: - pass - - def play_aidungeon_2(): save_story = input("Help improve AIDungeon by enabling story saving? (Y/n) ") @@ -58,6 +48,9 @@ def play_aidungeon_2(): while True: + if story_manager.story != None: + del story_manager.story + print("\n\n") context, prompt = select_game() console_print(instructions()) @@ -71,12 +64,8 @@ def play_aidungeon_2(): tcflush(sys.stdin, TCIFLUSH) action = input("> ") if action == "restart": - if upload_story: - upload_story_to_cloud(story_manager.story) break elif action == "quit": - if upload_story: - upload_story_to_cloud(story_manager.story) exit() if action != "" and action.lower() != "continue": @@ -98,13 +87,9 @@ def play_aidungeon_2(): if player_died(result): console_print(result + "\nGAME OVER") - if upload_story: - upload_story_to_cloud(story_manager.story) break elif player_won(result): console_print(result + "\n CONGRATS YOU WIN") - if upload_story: - upload_story_to_cloud(story_manager.story) break else: console_print(result) diff --git a/story/story_manager.py b/story/story_manager.py index 6fc6528..9c30916 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -31,6 +31,7 @@ class Story(): self.memory = 10 def __del__(self): + print("Before you go...") rating = input("Please rate the story quality from 1-10: ") try: rating_float = float(rating) @@ -115,6 +116,7 @@ class StoryManager(): def __init__(self, generator): self.generator = generator + self.story = None def start_new_story(self, story_prompt, context="", game_state=None, upload_story=False): block = self.generator.generate(context + story_prompt) From d50ac99a9b6f2f5bb64a01a15f75c98b5a6c1572 Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 19 Nov 2019 17:49:18 -0700 Subject: [PATCH 15/18] update --- story/story_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/story/story_manager.py b/story/story_manager.py index 9c30916..2a1c963 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -31,7 +31,6 @@ class Story(): self.memory = 10 def __del__(self): - print("Before you go...") rating = input("Please rate the story quality from 1-10: ") try: rating_float = float(rating) From 80ff1f907d34b78822b749609418888e9b098fee Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Thu, 21 Nov 2019 09:39:14 -0700 Subject: [PATCH 16/18] Develop (#7) * generatesettings * no_settings * no_settings * moresettings * moresettings * moresettings * moresettings * custom * update * new install script * update * update * update * fixed zombie data * Updated message * updated to get out tensorflow logs * changed msg --- data/.gitignore | 2 + data/build_training_data.py | 12 ++- data/make_reddit_data.py | 56 ++++++++++++ generator/simple/finetune.py | 7 +- install.sh | 24 +++++- play.py | 59 +++++++++---- story/custom_story.py | 53 ------------ story/story_data.yaml | 159 +++++++++++++++++++++-------------- story/story_manager.py | 17 ++-- story/utils.py | 73 +++++++--------- 10 files changed, 272 insertions(+), 190 deletions(-) create mode 100644 data/.gitignore create mode 100644 data/make_reddit_data.py delete mode 100644 story/custom_story.py diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 0000000..2a963a5 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1,2 @@ +writingprompts +*.txt \ No newline at end of file diff --git a/data/build_training_data.py b/data/build_training_data.py index 97b7990..18fefff 100644 --- a/data/build_training_data.py +++ b/data/build_training_data.py @@ -1,5 +1,6 @@ import csv import json +from story.utils import * def load_tree(filename): @@ -7,9 +8,18 @@ def load_tree(filename): tree = json.load(fp) return tree +def remove_phrase(text): + phrases = ["Years pass...", "Years pass"] + for phrase in phrases: + text = text.replace(phrase, "") + return text + def make_stories(current_story, tree): stories = [] - current_story += ("\n> " + tree["action"] + "\n" + tree["result"]) + action = first_to_second_person(tree["action"]) + action = remove_phrase(action) + result = remove_phrase(tree["result"]) + current_story += ("\n> " + action + "\n" + result) action_results = tree["action_results"] if len(action_results) == 0 or action_results[0] is None: diff --git a/data/make_reddit_data.py b/data/make_reddit_data.py new file mode 100644 index 0000000..218ac01 --- /dev/null +++ b/data/make_reddit_data.py @@ -0,0 +1,56 @@ +import json +from story.utils import * +import os + +def load_stories(file): + + + try: + with open(file) as fp: + stories = json.load(fp) + return stories + except: + with open(file) as fp: + stories = [] + for line in fp: + if len(line) > 10: + story = json.loads(line) + stories.append(story) + return stories + + +def modify_story(story): + + text = story["body"] + if len(text) < 100: + return None + + first_person = is_first_person(text) + second_person = is_second_person(text) + if first_person or second_person: + return first_to_second_person(text) + else: + return None + +current = os.getcwd() +files = os.listdir(current + "/writingprompts") +output_file_path = "writing_prompts.txt" +with open(output_file_path, 'w') as output_file: + filenames = ["writingprompts/" + file for file in files] + cleaned_stories = [] + for filename in filenames: + print("Processing file ", filename) + stories = load_stories(filename) + for story in stories: + cleaned_story = modify_story(story) + if cleaned_story is not None: + cleaned_stories.append(cleaned_story) + + raw_text = "" + start_token = "<|startoftext|>" + end_token = "<|endoftext|>" + for story in cleaned_stories: + raw_text += start_token + story + end_token + "\n" + print(len(raw_text)) + + output_file.write(raw_text) diff --git a/generator/simple/finetune.py b/generator/simple/finetune.py index d161b12..47334c4 100644 --- a/generator/simple/finetune.py +++ b/generator/simple/finetune.py @@ -8,7 +8,7 @@ if not os.path.isdir(os.path.join("models", model_name)): print("Downloading ", model_name, " model...") gpt2.download_gpt2(model_name=model_name) # model is saved into current directory under /models/124M/ -file_name = "text_adventures.txt" +file_name = "writing_prompts.txt" sess = gpt2.start_tf_sess() gpt2.finetune(sess, @@ -17,6 +17,9 @@ gpt2.finetune(sess, batch_size=8, learning_rate=0.0001, model_name=model_name, - steps=100) + sample_every=1000, + max_checkpoints=1, + save_every=200, + steps=600) gpt2.generate(sess) diff --git a/install.sh b/install.sh index d8546db..00573f7 100755 --- a/install.sh +++ b/install.sh @@ -1,10 +1,26 @@ -MODEL_DIRECTORY=aidungeon/generator/gpt2/models/model_v4 +MODELS_DIRECTORY=generator/gpt2/models +MODEL_VERSION=model_v4 +MODEL_NAME=model-200 +DOWNLOAD_URL=https://students.cs.byu.edu/~nickwalt -if [ -d "$MODEL_DIRECTORY" ]; then +if [ -d "${MODELS_DIRECTORY}/${MODEL_VERSION}" ]; then echo "AIDungeon2 is already installed" else - echo "Downloading AIDungeon2 Model" - gsutil -m cp -r gs://aidungeon2model/model_v4 ./generator/gpt2/models + echo "Downloading AIDungeon2 Model... (this may take a few minutes)" + cd ${MODELS_DIRECTORY} + mkdir ${MODEL_VERSION} + cd ${MODEL_VERSION} + apt-get install aria2 > /dev/null + aria2c -x 16 -s 32 "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.data-00000-of-00001" + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/checkpoint" > /dev/null + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/encoder.json" > /dev/null + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/hparams.json" > /dev/null + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.index" > /dev/null + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.meta" > /dev/null + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/vocab.bpe" > /dev/null + echo "Download Complete!" + cd ../../../.. + pip install -r requirements.txt > /dev/null fi diff --git a/play.py b/play.py index 98b5676..f35eab1 100644 --- a/play.py +++ b/play.py @@ -1,24 +1,50 @@ from story.story_manager import * from generator.gpt2.gpt2_generator import * from story.utils import * -from story.custom_story import * from termios import tcflush, TCIFLUSH -import time,sys +import time, sys, os +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" def select_game(): - print("Which game would you like to play?") - options = ["zombies", "hospital", "apocalypse", "classic", "knight", "necromancer", "custom"] - for i, option in enumerate(options): - console_print(str(i) + ") " + option + "\n") + with open(YAML_FILE, 'r') as stream: + data = yaml.safe_load(stream) - choice = get_num_options(len(options)) - if options[choice] == "custom": - context, prompt = make_custom_story() + print("Pick a setting.") + settings = data["settings"].keys() + for i, setting in enumerate(settings): + print_str = str(i) + ") " + setting + if setting == "fantasy": + print_str += " (recommended for new players)" + console_print(print_str) + console_print(str(len(settings)) + ") custom (for advanced players)") + choice = get_num_options(len(settings)+1) - else: - game = options[choice] - prompt = get_story_start(game) - context = get_context(game) + if choice == len(settings): + + console_print("Enter a sentence or two that describes the context of who your character is. Ex. ' " + + "You are a knight living in the king of Larion. You have a sword and shield. '") + context = input("Context: ") + console_print("Enter the first couple sentences to start your adventure off. Ex. " + + "'You enter the forest searching for the dragon and see' ") + prompt = input("Starting Prompt: ") + return context, prompt + + setting_key = list(settings)[choice] + + print("Pick a character") + characters = data["settings"][setting_key]["characters"] + for i, character in enumerate(characters): + console_print(str(i) + ") " + character) + character_key = list(characters)[get_num_options(len(characters))] + + name = input("What is your name? ") + setting_description = data["settings"][setting_key]["description"] + character = data["settings"][setting_key]["characters"][character_key] + + context = "You are " + name + ", a " + character_key + " " + setting_description + \ + "You have a " + character["item1"] + " and a " + character["item2"] + ". " + prompt_num = np.random.randint(0, len(character["prompts"])) + prompt = character["prompts"][prompt_num] return context, prompt @@ -31,7 +57,7 @@ def instructions(): def play_aidungeon_2(): - save_story = input("Help improve AIDungeon by enabling story saving? (Y/n) ") + save_story = input("Help AIDungeon by letting us store your adventure to improve the model? (Y/n) ") if save_story.lower() in ["no", "No", "n"]: upload_story = False else: @@ -40,14 +66,13 @@ def play_aidungeon_2(): print("\nInitializing AI Dungeon! (This might take a few minutes)\n") generator = GPT2Generator() story_manager = UnconstrainedStoryManager(generator) - print("\n\n\n\n") + print("\n") with open('opening.txt', 'r') as file: starter = file.read() print(starter) while True: - if story_manager.story != None: del story_manager.story @@ -80,8 +105,6 @@ def play_aidungeon_2(): action = action + "." action = "\n> " + action + "\n" - # action = remove_profanity(action) - #action = first_to_second_person(action) result = "\n" + story_manager.act(action) diff --git a/story/custom_story.py b/story/custom_story.py deleted file mode 100644 index 5ef25fc..0000000 --- a/story/custom_story.py +++ /dev/null @@ -1,53 +0,0 @@ -from story.utils import * -import numpy as np - -YAML_FILE = "story/story_data.yaml" - - -def make_custom_story(): - - with open(YAML_FILE, 'r') as stream: - data = yaml.safe_load(stream)["custom"] - - print("Pick a setting.") - settings = data["settings"].keys() - for i, setting in enumerate(settings): - console_print(str(i) + ") " + setting) - setting_key = list(settings)[get_num_options(len(settings))] - - print("Pick a character") - characters = data["settings"][setting_key]["characters"] - for i, character in enumerate(characters): - console_print(str(i) + ") " + character) - character_key = list(characters)[get_num_options(len(characters))] - - name = input("What is your name? ") - setting_description = data["settings"][setting_key]["description"] - character = data["settings"][setting_key]["characters"][character_key] - - context = "You are " + name + ", a " + character_key + " " + setting_description + \ - "You have a " + character["item1"] + " and a " + character["item2"] + ". " - prompt_num = np.random.randint(0,len(character["prompts"])) - prompt = character["prompts"][prompt_num] - - return context, prompt - -if __name__=='__main__': - c, p = make_custom_story() - print(c) - print(p) - - # print("Pick a setting.") - # for i, setting in enumerate(settings): - # console_print(str(i) + ") " + setting) - # setting_choice = get_num_options(len(settings)) - # - # - # - # print("") - # options = ["zombies", "hospital", "peasant", "apocalypse", "classic", "knight", "necromancer"] - # for i, option in enumerate(options): - # console_print(str(i) + ") " + option + ": " + get_context(option) + "\n") - # - # choice = get_num_options(len(options)) - # return options[choice] \ No newline at end of file diff --git a/story/story_data.yaml b/story/story_data.yaml index e7ecbb3..8a4c9b7 100644 --- a/story/story_data.yaml +++ b/story/story_data.yaml @@ -1,99 +1,134 @@ -prompts: - classic: "You enter a dungeon with your trusty sword and shield. You are searching for the evil necromancer who killed your family. You've heard that he resides at the bottom of the dungeon, guarded by legions of the undead. You enter the first door and see" - hospital: "You wake up in an old rundown hospital with no memory of how you got there. You take a look around the room and see" +settings: - peasant: "You were kicked out of your home and have set out to see if you can find work in the next town. You step onto the road and see" + fantasy: + description: "living in the kingdom of Larion. " + characters: - apocalypse: "You walk for two hours and take a break. You've left the town you were in and are now in a more rural area. You look around and see " + noble: + prompts: ["You are awakened by one of your servants who tells you that your keep is under attack. You look out the window and see"] - zombies: "You're on top of a building. You look over the city and see" + item1: "pouch of gold" + item2: "small dagger" - knight: "You wake up and continue your journey towards Larion Kingdom. You enter a dark forest that you must pass through. " + knight: + prompts: ["You enter the forest where you believe the ogre that has been terrorizing your home has been hiding. You step inside and"] - necromancer: "It’s been five years since the Great Lich Lord was defeated and now it’s time to take over again.\nA messenger runs towards you and tells y that your new necromancer lord has already begun laying waste to the lands under your control. You look around at your necromancer captains and ask their advice. " + item1: "steel longsword" + item2: "wooden shield" - scifi: "You have crashlanded on an some planet. You have a laser pistol and a pack of supplies. You leave your escape pod and see" + squire: + prompts: ["You follow Sir Theo as he enters the forest, he turns to you and says"] - vague: "You are a " + item1: "spear" + item2: "pack of supplies" -contexts: - zombies: "A few months ago a zombie infection broke out. You now are trying to survive on the ruins of what's left in the midst of zombie hordes. " + wizard: + prompts: ["You finish your long journey and finally arrive at the ruin you've been looking for. You look around and see"] - hospital: "You forgot everything before the hospital and are now trying to figure out what to do next. " + item1: "staff" + item2: "spellbook" - peasant: "You are a poor medieval peasant. " + ranger: + prompts: ["You spot the deer and are ready to finish your hunt when suddenly"] - vague: "" + item1: "hunting bow" + item2: "quiver of arrows" - apocalypse: "Long ago the bombs fell and the world ended. You are one of the few who is still alive. You are trying to survive by scavenging among the ruins of what is left behind. " + peasant: + prompts: ["You wake up and begin working in the fields. You see"] - classic: "You are a knight in a deep dungeon. " + item1: "pitchfork" + item2: "nothing else" - knight: "You are a knight on a quest to defeat the great dragon of Larion. You are armed with your sword and shield. " + rogue: + prompts: ["You successfully sneak into the merchant's house. You look around and see"] - necromancer: "You are a necromancer in a world where undead spell casters rule and fight one another for power and control. " + item1: "long steel dagger" + item2: "length of rope" - scifi: "You are a UN space marine with information crucial to the war effort that must get back to Earth. " + mystery: + description: "living in Chicago. " + characters: -custom: - settings: -# apocalyptic: -# description: "You live in a post apocalyptic world trying to survive by scavenging among the ruins of what is left behind. " -# -# zombies: -# description: "A few months ago a zombie infection broke out. You now are trying to survive on the ruins of what's left in the midst of zombie hordes. " -# -# modern: -# description: "" + patient: + prompts: ["You wake up in an old rundown hospital with no memory of how you got there. You take a look around the room and see"] -# context is "you are [name] a [character] [description]. You have a [item1] and [item2]. + item1: "hospital bracelet" + item2: "pack of bandages" - fantasy: - description: "living in the kingdom of Larion. " - characters: + detective: + prompts: ["You enter the forest where you believe the criminal you're searching for fled to. Suddenly"] - noble: - prompts: ["You are awakened by one of your servants who tells you that your keep is under attack. You look out the window and see"] + item1: "pistol" + item2: "police badge" - item1: "pouch of gold" - item2: "small dagger" + spy: + prompts: ["You listen to the russian diplomats and hear them discussing"] - knight: - prompts: ["You enter the forest where you believe the ogre that has been terrorizing your home has been hiding. You step inside and"] + item1: "concealed pistol" + item2: "syringe of poison" - item1: "steel longsword" - item2: "wooden shield" + manager: + prompt: ["It's late at night when you decide to head home after a long day of work. You step into the parking lot and suddenly"] - squire: - prompts: ["You follow Sir Theo as he enters the forest, he turns to you and says"] + item1: "wallet full of cash" + item2: "bowler hat" - item1: "spear" - item2: "pack of supplies" + apocalyptic: + description: " trying to survive in a post apocalyptic world by scavenging among the ruins of what is left. " - wizard: - prompts: ["You finish your long journey and finally arrive at the ruin you've been looking for. You look around and see"] + characters: - item1: "staff" - item2: "spellbook" + soldier: + prompts: ["You have lived in this military bunker since the bombs went off. You wake up and report for duty to your commanding officer and"] - ranger: - prompts: ["You spot the deer and are ready to finish your hunt when suddenly"] + item1: "automatic rifle" + item2: "grenade" - item1: "hunting bow" - item2: "quiver of arrows" + scavenger: + prompts: ["You walk for two hours and take a break. You've left your town in search of food. You look around and see "] - peasant: - prompts: ["You wake up and begin working in the fields. You see"] + item1: "rusty knife" + item2: "canteen" - item1: "pitchfork" - item2: "nothing else" + survivor: + prompts: ["You live in a small community of survivors who have managed to build a piece of civilization after + the bombs fell. You are talking to one of the leaders of the community when"] - rogue: - prompts: ["You successfully sneak into the merchant's house. You look around and see"] + item1: "hatchet" + item2: "hunting rifle" - item1: "long steel dagger" - item2: "length of rope" + courier: + prompts: ["It's a long and dangerous road from Boston to Charleston, but you're one of the only people who + knows the roads well enough to get your parcel of letters there. You set out in the morning and"] + + item1: "parcel of letters" + item2: "small pistol" + + zombies: + description: " trying to survive in a world filled with infected zombies everywhere. " + + characters: + soldier: + prompts: ["Your unit lost a lot of men when the infection broke, but you've managed to keep the small town + you're stationed near safe for now. You look over the town and think"] + + item1: "automatic rifle" + item2: "grenade" + + survivor: + prompts: ["You have managed to survive several months avoiding zombies and scavenging food. + You cautiously enter a rundown store and hear"] + + item1: "pistol" + item2: "backpack" + + scientist: + prompts: ["You pound your fist on the table angry that you still haven't found the cure to the infection. You turn to your assistant and"] + + item1: "backpack" + item2: "solar powered tablet" diff --git a/story/story_manager.py b/story/story_manager.py index 2a1c963..11a1dc4 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -28,16 +28,17 @@ class Story(): if game_state is None: game_state = dict() self.game_state = game_state - self.memory = 10 + self.memory = 8 def __del__(self): - rating = input("Please rate the story quality from 1-10: ") - try: - rating_float = float(rating) - self.rating = rating_float - self.save_to_storage() - except: - pass + if self.upload_story: + rating = input("Please rate the story quality from 1-10: ") + try: + rating_float = float(rating) + self.rating = rating_float + self.save_to_storage() + except: + pass def initialize_from_json(self, json_string): story_dict = json.loads(json_string) diff --git a/story/utils.py b/story/utils.py index cf5cb07..ba5a46e 100644 --- a/story/utils.py +++ b/story/utils.py @@ -21,7 +21,6 @@ def console_print(text, width=75): i += 1 print(text) - def get_num_options(num): while True: @@ -35,31 +34,6 @@ def get_num_options(num): except ValueError: print("Error invalid choice. ") -def get_context(key): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["contexts"][key] - -def get_allowed_ctrl_verbs(): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["ctrl_verbs"]["movement"] + data_loaded["ctrl_verbs"]["non_movement"] - -def get_story_start(key): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["prompts"][key] - - -def get_action_verbs(key): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["action_verbs"][key] - def player_died(text): @@ -78,21 +52,6 @@ def player_won(text): return True return False - -def get_ctrl_verbs(key): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["ctrl_verbs"][key] - - -def get_rooms(key): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["rooms"][key] - - def remove_profanity(text): return pf.censor(text) @@ -146,7 +105,37 @@ def replace_outside_quotes(text, current_word, repl_word): output = reg_expr.sub(repl_word, text) return output - + +def is_first_person(text): + + count = 0 + for pair in first_to_second_mappings: + variations = mapping_variation_pairs(pair) + for variation in variations: + reg_expr = re.compile(variation[0] + '(?=([^"]*"[^"]*")*[^"]*$)') + matches = re.findall(reg_expr, text) + count += len(matches) + + if count > 3: + return True + else: + return False + + +def is_second_person(text): + count = 0 + for pair in second_to_first_mappings: + variations = mapping_variation_pairs(pair) + for variation in variations: + reg_expr = re.compile(variation[0] + '(?=([^"]*"[^"]*")*[^"]*$)') + matches = re.findall(reg_expr, text) + count += len(matches) + + if count > 3: + return True + else: + return False + def capitalize(word): return word[0].upper() + word[1:] From 810a6768c9847bb87fffba0bf2f301a01e55d5a3 Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Thu, 21 Nov 2019 11:23:42 -0700 Subject: [PATCH 17/18] Update play.py --- play.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/play.py b/play.py index f35eab1..7329fcd 100644 --- a/play.py +++ b/play.py @@ -31,13 +31,13 @@ def select_game(): setting_key = list(settings)[choice] - print("Pick a character") + print("\nPick a character") characters = data["settings"][setting_key]["characters"] for i, character in enumerate(characters): console_print(str(i) + ") " + character) character_key = list(characters)[get_num_options(len(characters))] - name = input("What is your name? ") + name = input("\nWhat is your name? ") setting_description = data["settings"][setting_key]["description"] character = data["settings"][setting_key]["characters"][character_key] From 6b6a1b0d358711daef5b2df211db2797e7d0d70b Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Thu, 21 Nov 2019 16:58:55 -0700 Subject: [PATCH 18/18] Update play.py --- play.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/play.py b/play.py index 7329fcd..4a9a052 100644 --- a/play.py +++ b/play.py @@ -108,10 +108,7 @@ def play_aidungeon_2(): result = "\n" + story_manager.act(action) - if player_died(result): - console_print(result + "\nGAME OVER") - break - elif player_won(result): + if player_won(result): console_print(result + "\n CONGRATS YOU WIN") break else: