update

data/.gitignore

@@ -0,0 +1,2 @@
+writingprompts
+*.txt

data/build_training_data.py

@@ -1,5 +1,6 @@
 import csv
 import json
+from story.utils import *
 
 
 def load_tree(filename):
@@ -7,9 +8,18 @@ def load_tree(filename):
         tree = json.load(fp)
     return tree
 
+def remove_phrase(text):
+    phrases = ["Years pass...", "Years pass"]
+    for phrase in phrases:
+        text = text.replace(phrase, "")
+    return text
+
 def make_stories(current_story, tree):
     stories = []
-    current_story += ("\n> " + tree["action"] + "\n" + tree["result"])
+    action = first_to_second_person(tree["action"])
+    action = remove_phrase(action)
+    result = remove_phrase(tree["result"])
+    current_story += ("\n> " + action + "\n" + result)
 
     action_results = tree["action_results"]
     if len(action_results) == 0 or action_results[0] is None:

data/make_reddit_data.py

@@ -0,0 +1,56 @@
+import json
+from story.utils import *
+import os
+
+def load_stories(file):
+
+
+        try:
+            with open(file) as fp:
+                stories = json.load(fp)
+                return stories
+        except:
+            with open(file) as fp:
+                stories = []
+                for line in fp:
+                    if len(line) > 10:
+                        story = json.loads(line)
+                        stories.append(story)
+                return stories
+
+
+def modify_story(story):
+
+    text = story["body"]
+    if len(text) < 100:
+        return None
+
+    first_person = is_first_person(text)
+    second_person = is_second_person(text)
+    if first_person or second_person:
+        return first_to_second_person(text)
+    else:
+        return None
+
+current = os.getcwd()
+files = os.listdir(current + "/writingprompts")
+output_file_path = "writing_prompts.txt"
+with open(output_file_path, 'w') as output_file:
+    filenames = ["writingprompts/" + file for file in files]
+    cleaned_stories = []
+    for filename in filenames:
+        print("Processing file ", filename)
+        stories = load_stories(filename)
+        for story in stories:
+            cleaned_story = modify_story(story)
+            if cleaned_story is not None:
+                cleaned_stories.append(cleaned_story)
+
+    raw_text = ""
+    start_token = "<|startoftext|>"
+    end_token = "<|endoftext|>"
+    for story in cleaned_stories:
+        raw_text += start_token + story + end_token + "\n"
+        print(len(raw_text))
+
+    output_file.write(raw_text)

generator/gpt2/gpt2_generator.py

@@ -10,7 +10,7 @@ import numpy as np
 
 class GPT2Generator:
 
-    def __init__(self,  generate_num=60, temperature=0.3, top_k=40, top_p=0.9):
+    def __init__(self,  generate_num=60, temperature=0.4, top_k=40, top_p=0.9):
         self.generate_num=generate_num
         self.temp = temperature
         self.top_k = top_k

generator/gpt2/src/sample.py

@@ -17,7 +17,7 @@ def penalize_used(logits, output):
 
     return tf.compat.v1.where(
             bool_tensor,
-            logits / 1.2,
+            logits * 0.85,
             logits)
 
 

generator/simple/finetune.py

@@ -8,7 +8,7 @@ if not os.path.isdir(os.path.join("models", model_name)):
     print("Downloading ", model_name, " model...")
     gpt2.download_gpt2(model_name=model_name)  # model is saved into current directory under /models/124M/
 
-file_name = "text_adventures.txt"
+file_name = "writing_prompts.txt"
 
 sess = gpt2.start_tf_sess()
 gpt2.finetune(sess,
@@ -17,6 +17,9 @@ gpt2.finetune(sess,
               batch_size=8,
               learning_rate=0.0001,
               model_name=model_name,
-              steps=100)
+              sample_every=1000,
+              max_checkpoints=1,
+              save_every=200,
+              steps=600)
 
 gpt2.generate(sess)

install.sh

@@ -1,10 +1,26 @@
-MODEL_DIRECTORY=aidungeon/generator/gpt2/models/model_v4
+MODELS_DIRECTORY=generator/gpt2/models
+MODEL_VERSION=model_v4
+MODEL_NAME=model-200
+DOWNLOAD_URL=https://students.cs.byu.edu/~nickwalt
 
-if [ -d "$MODEL_DIRECTORY" ]; then
+if [ -d "${MODELS_DIRECTORY}/${MODEL_VERSION}" ]; then
     echo "AIDungeon2 is already installed"
 
 else
-    echo "Downloading AIDungeon2 Model"
-    gsutil -m cp -r gs://aidungeon2model/model_v4 ./generator/gpt2/models
+	echo "Downloading AIDungeon2 Model... (this may take a few minutes)"
+    cd ${MODELS_DIRECTORY}
+    mkdir ${MODEL_VERSION}
+    cd ${MODEL_VERSION}
+    apt-get install aria2 > /dev/null
+    aria2c -x 16 -s 32 "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.data-00000-of-00001"
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/checkpoint" > /dev/null
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/encoder.json" > /dev/null
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/hparams.json" > /dev/null
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.index" > /dev/null
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.meta" > /dev/null
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/vocab.bpe" > /dev/null
+    echo "Download Complete!"
+    cd ../../../..
+
     pip install -r requirements.txt > /dev/null
 fi

play.py

@@ -1,24 +1,50 @@
 from story.story_manager import *
 from generator.gpt2.gpt2_generator import *
 from story.utils import *
-from story.custom_story import *
 from termios import tcflush, TCIFLUSH
-import time,sys
+import time, sys, os
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
 
 def select_game():
-    print("Which game would you like to play?")
-    options = ["zombies", "hospital", "apocalypse", "classic", "knight", "necromancer", "custom"]
-    for i, option in enumerate(options):
-        console_print(str(i) + ") " + option + "\n")
+    with open(YAML_FILE, 'r') as stream:
+        data = yaml.safe_load(stream)
 
-    choice = get_num_options(len(options))
-    if options[choice] == "custom":
-        context, prompt = make_custom_story()
+    print("Pick a setting.")
+    settings = data["settings"].keys()
+    for i, setting in enumerate(settings):
+        print_str = str(i) + ") " + setting
+        if setting == "fantasy":
+            print_str += " (recommended for new players)"
+        console_print(print_str)
+    console_print(str(len(settings)) + ") custom (for advanced players)")
+    choice = get_num_options(len(settings)+1)
 
-    else:
-        game = options[choice]
-        prompt = get_story_start(game)
-        context = get_context(game)
+    if choice == len(settings):
+
+        console_print("Enter a sentence or two that describes the context of who your character is. Ex. ' " +
+                        "You are a knight living in the king of Larion. You have a sword and shield. '")
+        context = input("Context: ")
+        console_print("Enter the first couple sentences to start your adventure off. Ex. " +
+                       "'You enter the forest searching for the dragon and see' ")
+        prompt = input("Starting Prompt: ")
+        return context, prompt
+
+    setting_key = list(settings)[choice]
+
+    print("\nPick a character")
+    characters = data["settings"][setting_key]["characters"]
+    for i, character in enumerate(characters):
+        console_print(str(i) + ") " + character)
+    character_key = list(characters)[get_num_options(len(characters))]
+
+    name = input("\nWhat is your name? ")
+    setting_description = data["settings"][setting_key]["description"]
+    character = data["settings"][setting_key]["characters"][character_key]
+
+    context = "You are " + name + ", a " + character_key + " " + setting_description + \
+              "You have a " + character["item1"] + " and a " + character["item2"] + ". "
+    prompt_num = np.random.randint(0, len(character["prompts"]))
+    prompt = character["prompts"][prompt_num]
 
     return context, prompt
 
@@ -29,25 +55,33 @@ def instructions():
     text += '\n* Finally if you want to end your game and start a new one just enter "restart" for any action. '
     return text
 
-
 def play_aidungeon_2():
 
-    print("Initializing AI Dungeon! (This might take a few minutes)")
+    save_story = input("Help AIDungeon by letting us store your adventure to improve the model? (Y/n) ")
+    if save_story.lower() in ["no", "No", "n"]:
+        upload_story = False
+    else:
+        upload_story = True
+
+    print("\nInitializing AI Dungeon! (This might take a few minutes)\n")
     generator = GPT2Generator()
     story_manager = UnconstrainedStoryManager(generator)
-    print("\n\n\n\n")
+    print("\n")
 
     with open('opening.txt', 'r') as file:
         starter = file.read()
     print(starter)
 
     while True:
+        if story_manager.story != None:
+            del story_manager.story
 
         print("\n\n")
         context, prompt = select_game()
         console_print(instructions())
+        print("\nGenerating story...")
 
-        story_manager.start_new_story(prompt, context=context)
+        story_manager.start_new_story(prompt, context=context, upload_story=upload_story)
 
         print("\n")
         console_print(context + str(story_manager.story))
@@ -56,6 +90,8 @@ def play_aidungeon_2():
             action = input("> ")
             if action == "restart":
                 break
+            elif action == "quit":
+                exit()
 
             if action != "" and action.lower() != "continue":
                 action = action.strip()
@@ -69,15 +105,12 @@ def play_aidungeon_2():
                     action = action + "."
 
                 action = "\n> " + action + "\n"
-                # action = remove_profanity(action)
-                #action = first_to_second_person(action)
 
             result = "\n" + story_manager.act(action)
-            if player_died(result):
-                console_print(result + "\nGAME OVER")
-                break
-            elif player_won(result):
+
+            if player_won(result):
                 console_print(result + "\n CONGRATS YOU WIN")
+                break
             else:
                 console_print(result)
 

story/custom_story.py

@@ -1,53 +0,0 @@
-from story.utils import *
-import numpy as np
-
-YAML_FILE = "story/story_data.yaml"
-
-
-def make_custom_story():
-
-    with open(YAML_FILE, 'r') as stream:
-        data = yaml.safe_load(stream)["custom"]
-
-    print("Pick a setting.")
-    settings = data["settings"].keys()
-    for i, setting in enumerate(settings):
-        console_print(str(i) + ") " + setting)
-    setting_key = list(settings)[get_num_options(len(settings))]
-
-    print("Pick a character")
-    characters = data["settings"][setting_key]["characters"]
-    for i, character in enumerate(characters):
-        console_print(str(i) + ") " + character)
-    character_key = list(characters)[get_num_options(len(characters))]
-
-    name = input("What is your name? ")
-    setting_description = data["settings"][setting_key]["description"]
-    character = data["settings"][setting_key]["characters"][character_key]
-
-    context = "You are " + name + ", a " + character_key + " " + setting_description + \
-              "You have a " + character["item1"] + " and a " + character["item2"] + ". "
-    prompt_num = np.random.randint(0,len(character["prompts"]))
-    prompt = character["prompts"][prompt_num]
-
-    return context, prompt
-
-if __name__=='__main__':
-    c, p = make_custom_story()
-    print(c)
-    print(p)
-
-    # print("Pick a setting.")
-    # for i, setting in enumerate(settings):
-    #     console_print(str(i) + ") " + setting)
-    # setting_choice = get_num_options(len(settings))
-    #
-    #
-    #
-    # print("")
-    # options = ["zombies", "hospital", "peasant", "apocalypse", "classic", "knight", "necromancer"]
-    # for i, option in enumerate(options):
-    #     console_print(str(i) + ") " + option + ": " + get_context(option) + "\n")
-    #
-    # choice = get_num_options(len(options))
-    # return options[choice]

story/story_data.yaml

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 settings:
 
   fantasy:
@@ -15,6 +16,20 @@ settings:
 
         item1: "gold tiara"
         item2: "silver necklace"
+=======
+
+settings:
+
+  fantasy:
+    description: "living in the kingdom of Larion. "
+    characters:
+
+      noble:
+        prompts: ["You are awakened by one of your servants who tells you that your keep is under attack. You look out the window and see"]
+
+        item1: "pouch of gold"
+        item2: "small dagger"
+>>>>>>> b4a9cb520ad6e85573e786fa7a967a0d74969534
 
       knight:
         prompts: ["You enter the forest where you believe the ogre that has been terrorizing your home has been hiding. You step inside and"]
@@ -75,7 +90,11 @@ settings:
         item2: "syringe of poison"
 
       manager:
+<<<<<<< HEAD
         prompts: ["It's late at night when you decide to head home after a long day of work. You step into the parking lot and suddenly"]
+=======
+        prompt: ["It's late at night when you decide to head home after a long day of work. You step into the parking lot and suddenly"]
+>>>>>>> b4a9cb520ad6e85573e786fa7a967a0d74969534
 
         item1: "wallet full of cash"
         item2: "bowler hat"

story/story_manager.py

@@ -1,12 +1,17 @@
 from story.utils import *
 import json
-
+import uuid
+from subprocess import Popen
+import subprocess
+import os
 
 class Story():
 
-    def __init__(self, story_start, context ="", seed=None, game_state=None):
+    def __init__(self, story_start, context ="", seed=None, game_state=None, upload_story=False):
         self.story_start = story_start
         self.context = context
+        self.rating = -1
+        self.upload_story = upload_story
 
         # list of actions. First action is the prompt length should always equal that of story blocks
         self.actions = []
@@ -18,12 +23,22 @@ class Story():
         self.seed = seed
         self.choices = []
         self.possible_action_results = None
+        self.uuid = str(uuid.uuid1())
 
         if game_state is None:
             game_state = dict()
         self.game_state = game_state
-        self.memory = 10
+        self.memory = 8
 
+    def __del__(self):
+        if self.upload_story:
+            rating = input("Please rate the story quality from 1-10: ")
+            try:
+                rating_float = float(rating)
+                self.rating = rating_float
+                self.save_to_storage()
+            except:
+                pass
 
     def initialize_from_json(self, json_string):
         story_dict = json.loads(json_string)
@@ -35,6 +50,12 @@ class Story():
         self.possible_action_results = story_dict["possible_action_results"]
         self.game_state = story_dict["game_state"]
         self.context = story_dict["context"]
+        self.uuid = story_dict["uuid"]
+
+        if "rating" in story_dict.keys():
+            self.rating = story_dict["rating"]
+        else:
+            self.rating = -1
 
     def add_to_story(self, action, story_block):
         self.actions.append(action)
@@ -75,19 +96,32 @@ class Story():
         story_dict["possible_action_results"] = self.possible_action_results
         story_dict["game_state"] = self.game_state
         story_dict["context"] = self.context
+        story_dict["uuid"] = self.uuid
+        story_dict["rating"] = self.rating
 
         return json.dumps(story_dict)
 
+    def save_to_storage(self):
+        story_json = self.to_json()
+        file_name = "story" + str(self.uuid) + ".json"
+        f = open(file_name, "w")
+        f.write(story_json)
+        f.close()
+
+        FNULL = open(os.devnull, 'w')
+        p = Popen(['gsutil', 'cp', file_name, 'gs://aidungeonstories'], stdout=FNULL, stderr=subprocess.STDOUT)
+
 
 class StoryManager():
 
     def __init__(self, generator):
         self.generator = generator
+        self.story = None
         
-    def start_new_story(self, story_prompt, context="", game_state=None):
+    def start_new_story(self, story_prompt, context="", game_state=None, upload_story=False):
         block = self.generator.generate(context + story_prompt)
         block = cut_trailing_sentence(block)
-        self.story = Story(story_prompt + block, context=context, game_state=game_state)
+        self.story = Story(story_prompt + block, context=context, game_state=game_state, upload_story=upload_story)
         return self.story
     
     def load_story(self, story, from_json=False):

story/utils.py

@@ -21,7 +21,6 @@ def console_print(text, width=75):
         i += 1
     print(text)
 
-
 def get_num_options(num):
 
     while True:
@@ -35,31 +34,6 @@ def get_num_options(num):
         except ValueError:
             print("Error invalid choice. ")
 
-def get_context(key):
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["contexts"][key]
-
-def get_allowed_ctrl_verbs():
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["ctrl_verbs"]["movement"] + data_loaded["ctrl_verbs"]["non_movement"]
-
-def get_story_start(key):
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["prompts"][key]
-
-
-def get_action_verbs(key):
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["action_verbs"][key]
-
 
 def player_died(text):
 
@@ -78,21 +52,6 @@ def player_won(text):
             return True
     return False
 
-
-def get_ctrl_verbs(key):
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["ctrl_verbs"][key]
-
-
-def get_rooms(key):
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["rooms"][key]
-
-
 def remove_profanity(text):
     return pf.censor(text)
 
@@ -146,7 +105,37 @@ def replace_outside_quotes(text, current_word, repl_word):
     output = reg_expr.sub(repl_word, text)
     return output
 
-    
+
+def is_first_person(text):
+
+    count = 0
+    for pair in first_to_second_mappings:
+        variations = mapping_variation_pairs(pair)
+        for variation in variations:
+            reg_expr = re.compile(variation[0] + '(?=([^"]*"[^"]*")*[^"]*$)')
+            matches = re.findall(reg_expr, text)
+            count += len(matches)
+
+    if count > 3:
+        return True
+    else:
+        return False
+
+
+def is_second_person(text):
+    count = 0
+    for pair in second_to_first_mappings:
+        variations = mapping_variation_pairs(pair)
+        for variation in variations:
+            reg_expr = re.compile(variation[0] + '(?=([^"]*"[^"]*")*[^"]*$)')
+            matches = re.findall(reg_expr, text)
+            count += len(matches)
+
+    if count > 3:
+        return True
+    else:
+        return False
+
 
 def capitalize(word):
     return word[0].upper() + word[1:]