diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 0000000..2a963a5 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1,2 @@ +writingprompts +*.txt \ No newline at end of file diff --git a/data/build_training_data.py b/data/build_training_data.py index 97b7990..18fefff 100644 --- a/data/build_training_data.py +++ b/data/build_training_data.py @@ -1,5 +1,6 @@ import csv import json +from story.utils import * def load_tree(filename): @@ -7,9 +8,18 @@ def load_tree(filename): tree = json.load(fp) return tree +def remove_phrase(text): + phrases = ["Years pass...", "Years pass"] + for phrase in phrases: + text = text.replace(phrase, "") + return text + def make_stories(current_story, tree): stories = [] - current_story += ("\n> " + tree["action"] + "\n" + tree["result"]) + action = first_to_second_person(tree["action"]) + action = remove_phrase(action) + result = remove_phrase(tree["result"]) + current_story += ("\n> " + action + "\n" + result) action_results = tree["action_results"] if len(action_results) == 0 or action_results[0] is None: diff --git a/data/make_reddit_data.py b/data/make_reddit_data.py new file mode 100644 index 0000000..218ac01 --- /dev/null +++ b/data/make_reddit_data.py @@ -0,0 +1,56 @@ +import json +from story.utils import * +import os + +def load_stories(file): + + + try: + with open(file) as fp: + stories = json.load(fp) + return stories + except: + with open(file) as fp: + stories = [] + for line in fp: + if len(line) > 10: + story = json.loads(line) + stories.append(story) + return stories + + +def modify_story(story): + + text = story["body"] + if len(text) < 100: + return None + + first_person = is_first_person(text) + second_person = is_second_person(text) + if first_person or second_person: + return first_to_second_person(text) + else: + return None + +current = os.getcwd() +files = os.listdir(current + "/writingprompts") +output_file_path = "writing_prompts.txt" +with open(output_file_path, 'w') as output_file: + filenames = ["writingprompts/" + file for file in files] + cleaned_stories = [] + for filename in filenames: + print("Processing file ", filename) + stories = load_stories(filename) + for story in stories: + cleaned_story = modify_story(story) + if cleaned_story is not None: + cleaned_stories.append(cleaned_story) + + raw_text = "" + start_token = "<|startoftext|>" + end_token = "<|endoftext|>" + for story in cleaned_stories: + raw_text += start_token + story + end_token + "\n" + print(len(raw_text)) + + output_file.write(raw_text) diff --git a/generator/gpt2/gpt2_generator.py b/generator/gpt2/gpt2_generator.py index 66a0813..f64deef 100644 --- a/generator/gpt2/gpt2_generator.py +++ b/generator/gpt2/gpt2_generator.py @@ -10,7 +10,7 @@ import numpy as np class GPT2Generator: - def __init__(self, generate_num=60, temperature=0.3, top_k=40, top_p=0.9): + def __init__(self, generate_num=60, temperature=0.4, top_k=40, top_p=0.9): self.generate_num=generate_num self.temp = temperature self.top_k = top_k diff --git a/generator/gpt2/src/sample.py b/generator/gpt2/src/sample.py index 5879d11..f50eaf8 100644 --- a/generator/gpt2/src/sample.py +++ b/generator/gpt2/src/sample.py @@ -17,7 +17,7 @@ def penalize_used(logits, output): return tf.compat.v1.where( bool_tensor, - logits / 1.2, + logits * 0.85, logits) diff --git a/generator/simple/finetune.py b/generator/simple/finetune.py index d161b12..47334c4 100644 --- a/generator/simple/finetune.py +++ b/generator/simple/finetune.py @@ -8,7 +8,7 @@ if not os.path.isdir(os.path.join("models", model_name)): print("Downloading ", model_name, " model...") gpt2.download_gpt2(model_name=model_name) # model is saved into current directory under /models/124M/ -file_name = "text_adventures.txt" +file_name = "writing_prompts.txt" sess = gpt2.start_tf_sess() gpt2.finetune(sess, @@ -17,6 +17,9 @@ gpt2.finetune(sess, batch_size=8, learning_rate=0.0001, model_name=model_name, - steps=100) + sample_every=1000, + max_checkpoints=1, + save_every=200, + steps=600) gpt2.generate(sess) diff --git a/install.sh b/install.sh index d8546db..00573f7 100755 --- a/install.sh +++ b/install.sh @@ -1,10 +1,26 @@ -MODEL_DIRECTORY=aidungeon/generator/gpt2/models/model_v4 +MODELS_DIRECTORY=generator/gpt2/models +MODEL_VERSION=model_v4 +MODEL_NAME=model-200 +DOWNLOAD_URL=https://students.cs.byu.edu/~nickwalt -if [ -d "$MODEL_DIRECTORY" ]; then +if [ -d "${MODELS_DIRECTORY}/${MODEL_VERSION}" ]; then echo "AIDungeon2 is already installed" else - echo "Downloading AIDungeon2 Model" - gsutil -m cp -r gs://aidungeon2model/model_v4 ./generator/gpt2/models + echo "Downloading AIDungeon2 Model... (this may take a few minutes)" + cd ${MODELS_DIRECTORY} + mkdir ${MODEL_VERSION} + cd ${MODEL_VERSION} + apt-get install aria2 > /dev/null + aria2c -x 16 -s 32 "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.data-00000-of-00001" + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/checkpoint" > /dev/null + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/encoder.json" > /dev/null + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/hparams.json" > /dev/null + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.index" > /dev/null + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.meta" > /dev/null + wget "${DOWNLOAD_URL}/${MODEL_VERSION}/vocab.bpe" > /dev/null + echo "Download Complete!" + cd ../../../.. + pip install -r requirements.txt > /dev/null fi diff --git a/play.py b/play.py index ef3f419..4a9a052 100644 --- a/play.py +++ b/play.py @@ -1,24 +1,50 @@ from story.story_manager import * from generator.gpt2.gpt2_generator import * from story.utils import * -from story.custom_story import * from termios import tcflush, TCIFLUSH -import time,sys +import time, sys, os +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" def select_game(): - print("Which game would you like to play?") - options = ["zombies", "hospital", "apocalypse", "classic", "knight", "necromancer", "custom"] - for i, option in enumerate(options): - console_print(str(i) + ") " + option + "\n") + with open(YAML_FILE, 'r') as stream: + data = yaml.safe_load(stream) - choice = get_num_options(len(options)) - if options[choice] == "custom": - context, prompt = make_custom_story() + print("Pick a setting.") + settings = data["settings"].keys() + for i, setting in enumerate(settings): + print_str = str(i) + ") " + setting + if setting == "fantasy": + print_str += " (recommended for new players)" + console_print(print_str) + console_print(str(len(settings)) + ") custom (for advanced players)") + choice = get_num_options(len(settings)+1) - else: - game = options[choice] - prompt = get_story_start(game) - context = get_context(game) + if choice == len(settings): + + console_print("Enter a sentence or two that describes the context of who your character is. Ex. ' " + + "You are a knight living in the king of Larion. You have a sword and shield. '") + context = input("Context: ") + console_print("Enter the first couple sentences to start your adventure off. Ex. " + + "'You enter the forest searching for the dragon and see' ") + prompt = input("Starting Prompt: ") + return context, prompt + + setting_key = list(settings)[choice] + + print("\nPick a character") + characters = data["settings"][setting_key]["characters"] + for i, character in enumerate(characters): + console_print(str(i) + ") " + character) + character_key = list(characters)[get_num_options(len(characters))] + + name = input("\nWhat is your name? ") + setting_description = data["settings"][setting_key]["description"] + character = data["settings"][setting_key]["characters"][character_key] + + context = "You are " + name + ", a " + character_key + " " + setting_description + \ + "You have a " + character["item1"] + " and a " + character["item2"] + ". " + prompt_num = np.random.randint(0, len(character["prompts"])) + prompt = character["prompts"][prompt_num] return context, prompt @@ -29,25 +55,33 @@ def instructions(): text += '\n* Finally if you want to end your game and start a new one just enter "restart" for any action. ' return text - def play_aidungeon_2(): - print("Initializing AI Dungeon! (This might take a few minutes)") + save_story = input("Help AIDungeon by letting us store your adventure to improve the model? (Y/n) ") + if save_story.lower() in ["no", "No", "n"]: + upload_story = False + else: + upload_story = True + + print("\nInitializing AI Dungeon! (This might take a few minutes)\n") generator = GPT2Generator() story_manager = UnconstrainedStoryManager(generator) - print("\n\n\n\n") + print("\n") with open('opening.txt', 'r') as file: starter = file.read() print(starter) while True: + if story_manager.story != None: + del story_manager.story print("\n\n") context, prompt = select_game() console_print(instructions()) + print("\nGenerating story...") - story_manager.start_new_story(prompt, context=context) + story_manager.start_new_story(prompt, context=context, upload_story=upload_story) print("\n") console_print(context + str(story_manager.story)) @@ -56,6 +90,8 @@ def play_aidungeon_2(): action = input("> ") if action == "restart": break + elif action == "quit": + exit() if action != "" and action.lower() != "continue": action = action.strip() @@ -69,15 +105,12 @@ def play_aidungeon_2(): action = action + "." action = "\n> " + action + "\n" - # action = remove_profanity(action) - #action = first_to_second_person(action) result = "\n" + story_manager.act(action) - if player_died(result): - console_print(result + "\nGAME OVER") - break - elif player_won(result): + + if player_won(result): console_print(result + "\n CONGRATS YOU WIN") + break else: console_print(result) diff --git a/story/custom_story.py b/story/custom_story.py deleted file mode 100644 index 5ef25fc..0000000 --- a/story/custom_story.py +++ /dev/null @@ -1,53 +0,0 @@ -from story.utils import * -import numpy as np - -YAML_FILE = "story/story_data.yaml" - - -def make_custom_story(): - - with open(YAML_FILE, 'r') as stream: - data = yaml.safe_load(stream)["custom"] - - print("Pick a setting.") - settings = data["settings"].keys() - for i, setting in enumerate(settings): - console_print(str(i) + ") " + setting) - setting_key = list(settings)[get_num_options(len(settings))] - - print("Pick a character") - characters = data["settings"][setting_key]["characters"] - for i, character in enumerate(characters): - console_print(str(i) + ") " + character) - character_key = list(characters)[get_num_options(len(characters))] - - name = input("What is your name? ") - setting_description = data["settings"][setting_key]["description"] - character = data["settings"][setting_key]["characters"][character_key] - - context = "You are " + name + ", a " + character_key + " " + setting_description + \ - "You have a " + character["item1"] + " and a " + character["item2"] + ". " - prompt_num = np.random.randint(0,len(character["prompts"])) - prompt = character["prompts"][prompt_num] - - return context, prompt - -if __name__=='__main__': - c, p = make_custom_story() - print(c) - print(p) - - # print("Pick a setting.") - # for i, setting in enumerate(settings): - # console_print(str(i) + ") " + setting) - # setting_choice = get_num_options(len(settings)) - # - # - # - # print("") - # options = ["zombies", "hospital", "peasant", "apocalypse", "classic", "knight", "necromancer"] - # for i, option in enumerate(options): - # console_print(str(i) + ") " + option + ": " + get_context(option) + "\n") - # - # choice = get_num_options(len(options)) - # return options[choice] \ No newline at end of file diff --git a/story/story_data.yaml b/story/story_data.yaml index 3338445..bef1dcc 100644 --- a/story/story_data.yaml +++ b/story/story_data.yaml @@ -1,3 +1,4 @@ +<<<<<<< HEAD settings: fantasy: @@ -15,6 +16,20 @@ settings: item1: "gold tiara" item2: "silver necklace" +======= + +settings: + + fantasy: + description: "living in the kingdom of Larion. " + characters: + + noble: + prompts: ["You are awakened by one of your servants who tells you that your keep is under attack. You look out the window and see"] + + item1: "pouch of gold" + item2: "small dagger" +>>>>>>> b4a9cb520ad6e85573e786fa7a967a0d74969534 knight: prompts: ["You enter the forest where you believe the ogre that has been terrorizing your home has been hiding. You step inside and"] @@ -75,7 +90,11 @@ settings: item2: "syringe of poison" manager: +<<<<<<< HEAD prompts: ["It's late at night when you decide to head home after a long day of work. You step into the parking lot and suddenly"] +======= + prompt: ["It's late at night when you decide to head home after a long day of work. You step into the parking lot and suddenly"] +>>>>>>> b4a9cb520ad6e85573e786fa7a967a0d74969534 item1: "wallet full of cash" item2: "bowler hat" diff --git a/story/story_manager.py b/story/story_manager.py index a0c3b25..11a1dc4 100644 --- a/story/story_manager.py +++ b/story/story_manager.py @@ -1,12 +1,17 @@ from story.utils import * import json - +import uuid +from subprocess import Popen +import subprocess +import os class Story(): - def __init__(self, story_start, context ="", seed=None, game_state=None): + def __init__(self, story_start, context ="", seed=None, game_state=None, upload_story=False): self.story_start = story_start self.context = context + self.rating = -1 + self.upload_story = upload_story # list of actions. First action is the prompt length should always equal that of story blocks self.actions = [] @@ -18,12 +23,22 @@ class Story(): self.seed = seed self.choices = [] self.possible_action_results = None + self.uuid = str(uuid.uuid1()) if game_state is None: game_state = dict() self.game_state = game_state - self.memory = 10 + self.memory = 8 + def __del__(self): + if self.upload_story: + rating = input("Please rate the story quality from 1-10: ") + try: + rating_float = float(rating) + self.rating = rating_float + self.save_to_storage() + except: + pass def initialize_from_json(self, json_string): story_dict = json.loads(json_string) @@ -35,6 +50,12 @@ class Story(): self.possible_action_results = story_dict["possible_action_results"] self.game_state = story_dict["game_state"] self.context = story_dict["context"] + self.uuid = story_dict["uuid"] + + if "rating" in story_dict.keys(): + self.rating = story_dict["rating"] + else: + self.rating = -1 def add_to_story(self, action, story_block): self.actions.append(action) @@ -75,19 +96,32 @@ class Story(): story_dict["possible_action_results"] = self.possible_action_results story_dict["game_state"] = self.game_state story_dict["context"] = self.context + story_dict["uuid"] = self.uuid + story_dict["rating"] = self.rating return json.dumps(story_dict) + def save_to_storage(self): + story_json = self.to_json() + file_name = "story" + str(self.uuid) + ".json" + f = open(file_name, "w") + f.write(story_json) + f.close() + + FNULL = open(os.devnull, 'w') + p = Popen(['gsutil', 'cp', file_name, 'gs://aidungeonstories'], stdout=FNULL, stderr=subprocess.STDOUT) + class StoryManager(): def __init__(self, generator): self.generator = generator + self.story = None - def start_new_story(self, story_prompt, context="", game_state=None): + def start_new_story(self, story_prompt, context="", game_state=None, upload_story=False): block = self.generator.generate(context + story_prompt) block = cut_trailing_sentence(block) - self.story = Story(story_prompt + block, context=context, game_state=game_state) + self.story = Story(story_prompt + block, context=context, game_state=game_state, upload_story=upload_story) return self.story def load_story(self, story, from_json=False): diff --git a/story/utils.py b/story/utils.py index cf5cb07..ba5a46e 100644 --- a/story/utils.py +++ b/story/utils.py @@ -21,7 +21,6 @@ def console_print(text, width=75): i += 1 print(text) - def get_num_options(num): while True: @@ -35,31 +34,6 @@ def get_num_options(num): except ValueError: print("Error invalid choice. ") -def get_context(key): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["contexts"][key] - -def get_allowed_ctrl_verbs(): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["ctrl_verbs"]["movement"] + data_loaded["ctrl_verbs"]["non_movement"] - -def get_story_start(key): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["prompts"][key] - - -def get_action_verbs(key): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["action_verbs"][key] - def player_died(text): @@ -78,21 +52,6 @@ def player_won(text): return True return False - -def get_ctrl_verbs(key): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["ctrl_verbs"][key] - - -def get_rooms(key): - with open(YAML_FILE, 'r') as stream: - data_loaded = yaml.safe_load(stream) - - return data_loaded["rooms"][key] - - def remove_profanity(text): return pf.censor(text) @@ -146,7 +105,37 @@ def replace_outside_quotes(text, current_word, repl_word): output = reg_expr.sub(repl_word, text) return output - + +def is_first_person(text): + + count = 0 + for pair in first_to_second_mappings: + variations = mapping_variation_pairs(pair) + for variation in variations: + reg_expr = re.compile(variation[0] + '(?=([^"]*"[^"]*")*[^"]*$)') + matches = re.findall(reg_expr, text) + count += len(matches) + + if count > 3: + return True + else: + return False + + +def is_second_person(text): + count = 0 + for pair in second_to_first_mappings: + variations = mapping_variation_pairs(pair) + for variation in variations: + reg_expr = re.compile(variation[0] + '(?=([^"]*"[^"]*")*[^"]*$)') + matches = re.findall(reg_expr, text) + count += len(matches) + + if count > 3: + return True + else: + return False + def capitalize(word): return word[0].upper() + word[1:]