Develop (#7)

* generatesettings

* no_settings

* no_settings

* moresettings

* moresettings

* moresettings

* moresettings

* custom

* update

* new install script

* update

* update

* update

* fixed zombie data

* Updated message

* updated to get out tensorflow logs

* changed msg

data/.gitignore

@@ -0,0 +1,2 @@
+writingprompts
+*.txt

data/build_training_data.py

@@ -1,5 +1,6 @@
 import csv
 import json
+from story.utils import *
 
 
 def load_tree(filename):
@@ -7,9 +8,18 @@ def load_tree(filename):
         tree = json.load(fp)
     return tree
 
+def remove_phrase(text):
+    phrases = ["Years pass...", "Years pass"]
+    for phrase in phrases:
+        text = text.replace(phrase, "")
+    return text
+
 def make_stories(current_story, tree):
     stories = []
-    current_story += ("\n> " + tree["action"] + "\n" + tree["result"])
+    action = first_to_second_person(tree["action"])
+    action = remove_phrase(action)
+    result = remove_phrase(tree["result"])
+    current_story += ("\n> " + action + "\n" + result)
 
     action_results = tree["action_results"]
     if len(action_results) == 0 or action_results[0] is None:

data/make_reddit_data.py

@@ -0,0 +1,56 @@
+import json
+from story.utils import *
+import os
+
+def load_stories(file):
+
+
+        try:
+            with open(file) as fp:
+                stories = json.load(fp)
+                return stories
+        except:
+            with open(file) as fp:
+                stories = []
+                for line in fp:
+                    if len(line) > 10:
+                        story = json.loads(line)
+                        stories.append(story)
+                return stories
+
+
+def modify_story(story):
+
+    text = story["body"]
+    if len(text) < 100:
+        return None
+
+    first_person = is_first_person(text)
+    second_person = is_second_person(text)
+    if first_person or second_person:
+        return first_to_second_person(text)
+    else:
+        return None
+
+current = os.getcwd()
+files = os.listdir(current + "/writingprompts")
+output_file_path = "writing_prompts.txt"
+with open(output_file_path, 'w') as output_file:
+    filenames = ["writingprompts/" + file for file in files]
+    cleaned_stories = []
+    for filename in filenames:
+        print("Processing file ", filename)
+        stories = load_stories(filename)
+        for story in stories:
+            cleaned_story = modify_story(story)
+            if cleaned_story is not None:
+                cleaned_stories.append(cleaned_story)
+
+    raw_text = ""
+    start_token = "<|startoftext|>"
+    end_token = "<|endoftext|>"
+    for story in cleaned_stories:
+        raw_text += start_token + story + end_token + "\n"
+        print(len(raw_text))
+
+    output_file.write(raw_text)

generator/simple/finetune.py

@@ -8,7 +8,7 @@ if not os.path.isdir(os.path.join("models", model_name)):
     print("Downloading ", model_name, " model...")
     gpt2.download_gpt2(model_name=model_name)  # model is saved into current directory under /models/124M/
 
-file_name = "text_adventures.txt"
+file_name = "writing_prompts.txt"
 
 sess = gpt2.start_tf_sess()
 gpt2.finetune(sess,
@@ -17,6 +17,9 @@ gpt2.finetune(sess,
               batch_size=8,
               learning_rate=0.0001,
               model_name=model_name,
-              steps=100)
+              sample_every=1000,
+              max_checkpoints=1,
+              save_every=200,
+              steps=600)
 
 gpt2.generate(sess)

install.sh

@@ -1,10 +1,26 @@
-MODEL_DIRECTORY=aidungeon/generator/gpt2/models/model_v4
+MODELS_DIRECTORY=generator/gpt2/models
+MODEL_VERSION=model_v4
+MODEL_NAME=model-200
+DOWNLOAD_URL=https://students.cs.byu.edu/~nickwalt
 
-if [ -d "$MODEL_DIRECTORY" ]; then
+if [ -d "${MODELS_DIRECTORY}/${MODEL_VERSION}" ]; then
     echo "AIDungeon2 is already installed"
 
 else
-    echo "Downloading AIDungeon2 Model"
-    gsutil -m cp -r gs://aidungeon2model/model_v4 ./generator/gpt2/models
+	echo "Downloading AIDungeon2 Model... (this may take a few minutes)"
+    cd ${MODELS_DIRECTORY}
+    mkdir ${MODEL_VERSION}
+    cd ${MODEL_VERSION}
+    apt-get install aria2 > /dev/null
+    aria2c -x 16 -s 32 "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.data-00000-of-00001"
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/checkpoint" > /dev/null
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/encoder.json" > /dev/null
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/hparams.json" > /dev/null
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.index" > /dev/null
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/${MODEL_NAME}.meta" > /dev/null
+    wget "${DOWNLOAD_URL}/${MODEL_VERSION}/vocab.bpe" > /dev/null
+    echo "Download Complete!"
+    cd ../../../..
+
     pip install -r requirements.txt > /dev/null
 fi

play.py

@@ -1,24 +1,50 @@
 from story.story_manager import *
 from generator.gpt2.gpt2_generator import *
 from story.utils import *
-from story.custom_story import *
 from termios import tcflush, TCIFLUSH
-import time,sys
+import time, sys, os
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
 
 def select_game():
-    print("Which game would you like to play?")
-    options = ["zombies", "hospital", "apocalypse", "classic", "knight", "necromancer", "custom"]
-    for i, option in enumerate(options):
-        console_print(str(i) + ") " + option + "\n")
+    with open(YAML_FILE, 'r') as stream:
+        data = yaml.safe_load(stream)
 
-    choice = get_num_options(len(options))
-    if options[choice] == "custom":
-        context, prompt = make_custom_story()
+    print("Pick a setting.")
+    settings = data["settings"].keys()
+    for i, setting in enumerate(settings):
+        print_str = str(i) + ") " + setting
+        if setting == "fantasy":
+            print_str += " (recommended for new players)"
+        console_print(print_str)
+    console_print(str(len(settings)) + ") custom (for advanced players)")
+    choice = get_num_options(len(settings)+1)
 
-    else:
-        game = options[choice]
-        prompt = get_story_start(game)
-        context = get_context(game)
+    if choice == len(settings):
+
+        console_print("Enter a sentence or two that describes the context of who your character is. Ex. ' " +
+                        "You are a knight living in the king of Larion. You have a sword and shield. '")
+        context = input("Context: ")
+        console_print("Enter the first couple sentences to start your adventure off. Ex. " +
+                       "'You enter the forest searching for the dragon and see' ")
+        prompt = input("Starting Prompt: ")
+        return context, prompt
+
+    setting_key = list(settings)[choice]
+
+    print("Pick a character")
+    characters = data["settings"][setting_key]["characters"]
+    for i, character in enumerate(characters):
+        console_print(str(i) + ") " + character)
+    character_key = list(characters)[get_num_options(len(characters))]
+
+    name = input("What is your name? ")
+    setting_description = data["settings"][setting_key]["description"]
+    character = data["settings"][setting_key]["characters"][character_key]
+
+    context = "You are " + name + ", a " + character_key + " " + setting_description + \
+              "You have a " + character["item1"] + " and a " + character["item2"] + ". "
+    prompt_num = np.random.randint(0, len(character["prompts"]))
+    prompt = character["prompts"][prompt_num]
 
     return context, prompt
 
@@ -31,7 +57,7 @@ def instructions():
 
 def play_aidungeon_2():
 
-    save_story = input("Help improve AIDungeon by enabling story saving? (Y/n) ")
+    save_story = input("Help AIDungeon by letting us store your adventure to improve the model? (Y/n) ")
     if save_story.lower() in ["no", "No", "n"]:
         upload_story = False
     else:
@@ -40,14 +66,13 @@ def play_aidungeon_2():
     print("\nInitializing AI Dungeon! (This might take a few minutes)\n")
     generator = GPT2Generator()
     story_manager = UnconstrainedStoryManager(generator)
-    print("\n\n\n\n")
+    print("\n")
 
     with open('opening.txt', 'r') as file:
         starter = file.read()
     print(starter)
 
     while True:
-
         if story_manager.story != None:
             del story_manager.story
 
@@ -80,8 +105,6 @@ def play_aidungeon_2():
                     action = action + "."
 
                 action = "\n> " + action + "\n"
-                # action = remove_profanity(action)
-                #action = first_to_second_person(action)
 
             result = "\n" + story_manager.act(action)
 

story/custom_story.py

@@ -1,53 +0,0 @@
-from story.utils import *
-import numpy as np
-
-YAML_FILE = "story/story_data.yaml"
-
-
-def make_custom_story():
-
-    with open(YAML_FILE, 'r') as stream:
-        data = yaml.safe_load(stream)["custom"]
-
-    print("Pick a setting.")
-    settings = data["settings"].keys()
-    for i, setting in enumerate(settings):
-        console_print(str(i) + ") " + setting)
-    setting_key = list(settings)[get_num_options(len(settings))]
-
-    print("Pick a character")
-    characters = data["settings"][setting_key]["characters"]
-    for i, character in enumerate(characters):
-        console_print(str(i) + ") " + character)
-    character_key = list(characters)[get_num_options(len(characters))]
-
-    name = input("What is your name? ")
-    setting_description = data["settings"][setting_key]["description"]
-    character = data["settings"][setting_key]["characters"][character_key]
-
-    context = "You are " + name + ", a " + character_key + " " + setting_description + \
-              "You have a " + character["item1"] + " and a " + character["item2"] + ". "
-    prompt_num = np.random.randint(0,len(character["prompts"]))
-    prompt = character["prompts"][prompt_num]
-
-    return context, prompt
-
-if __name__=='__main__':
-    c, p = make_custom_story()
-    print(c)
-    print(p)
-
-    # print("Pick a setting.")
-    # for i, setting in enumerate(settings):
-    #     console_print(str(i) + ") " + setting)
-    # setting_choice = get_num_options(len(settings))
-    #
-    #
-    #
-    # print("")
-    # options = ["zombies", "hospital", "peasant", "apocalypse", "classic", "knight", "necromancer"]
-    # for i, option in enumerate(options):
-    #     console_print(str(i) + ") " + option + ": " + get_context(option) + "\n")
-    #
-    # choice = get_num_options(len(options))
-    # return options[choice]

story/story_data.yaml

@@ -1,99 +1,134 @@
-prompts:
-  classic: "You enter a dungeon with your trusty sword and shield. You are searching for the evil necromancer who killed your family. You've heard that he resides at the bottom of the dungeon, guarded by legions of the undead. You enter the first door and see"
 
-  hospital: "You wake up in an old rundown hospital with no memory of how you got there. You take a look around the room and see"
+settings:
 
-  peasant: "You were kicked out of your home and have set out to see if you can find work in the next town. You step onto the road and see"
+  fantasy:
+    description: "living in the kingdom of Larion. "
+    characters:
 
-  apocalypse: "You walk for two hours and take a break. You've left the town you were in and are now in a more rural area. You look around and see "
+      noble:
+        prompts: ["You are awakened by one of your servants who tells you that your keep is under attack. You look out the window and see"]
 
-  zombies: "You're on top of a building. You look over the city and see"
+        item1: "pouch of gold"
+        item2: "small dagger"
 
-  knight: "You wake up and continue your journey towards Larion Kingdom. You enter a dark forest that you must pass through. "
+      knight:
+        prompts: ["You enter the forest where you believe the ogre that has been terrorizing your home has been hiding. You step inside and"]
 
-  necromancer: "It’s been five years since the Great Lich Lord was defeated and now it’s time to take over again.\nA messenger runs towards you and tells y that your new necromancer lord has already begun laying waste to the lands under your control. You look around at your necromancer captains and ask their advice. "
+        item1: "steel longsword"
+        item2: "wooden shield"
 
-  scifi: "You have crashlanded on an some planet. You have a laser pistol and a pack of supplies. You leave your escape pod and see"
+      squire:
+        prompts: ["You follow Sir Theo as he enters the forest, he turns to you and says"]
 
-  vague: "You are a "
+        item1: "spear"
+        item2: "pack of supplies"
 
-contexts:
-  zombies: "A few months ago a zombie infection broke out. You now are trying to survive on the ruins of what's left in the midst of zombie hordes. "
+      wizard:
+        prompts: ["You finish your long journey and finally arrive at the ruin you've been looking for. You look around and see"]
 
-  hospital: "You forgot everything before the hospital and are now trying to figure out what to do next. "
+        item1: "staff"
+        item2: "spellbook"
 
-  peasant: "You are a poor medieval peasant. "
+      ranger:
+        prompts: ["You spot the deer and are ready to finish your hunt when suddenly"]
 
-  vague: ""
+        item1: "hunting bow"
+        item2: "quiver of arrows"
 
-  apocalypse: "Long ago the bombs fell and the world ended. You are one of the few who is still alive. You are trying to survive by scavenging among the ruins of what is left behind. "
+      peasant:
+        prompts: ["You wake up and begin working in the fields. You see"]
 
-  classic: "You are a knight in a deep dungeon. "
+        item1: "pitchfork"
+        item2: "nothing else"
 
-  knight: "You are a knight on a quest to defeat the great dragon of Larion. You are armed with your sword and shield. "
+      rogue:
+        prompts: ["You successfully sneak into the merchant's house. You look around and see"]
 
-  necromancer: "You are a necromancer in a world where undead spell casters rule and fight one another for power and control. "
+        item1: "long steel dagger"
+        item2: "length of rope"
 
-  scifi: "You are a UN space marine with information crucial to the war effort that must get back to Earth. "
+  mystery:
+    description: "living in Chicago. "
+    characters:
 
-custom:
-  settings:
-#    apocalyptic:
-#      description: "You live in a post apocalyptic world trying to survive by scavenging among the ruins of what is left behind. "
-#
-#    zombies:
-#      description: "A few months ago a zombie infection broke out. You now are trying to survive on the ruins of what's left in the midst of zombie hordes. "
-#
-#    modern:
-#      description: ""
+      patient:
+        prompts: ["You wake up in an old rundown hospital with no memory of how you got there. You take a look around the room and see"]
 
-#   context is "you are [name] a [character] [description]. You have a [item1] and [item2].
+        item1: "hospital bracelet"
+        item2: "pack of bandages"
 
-    fantasy:
-      description: "living in the kingdom of Larion. "
-      characters:
+      detective:
+        prompts: ["You enter the forest where you believe the criminal you're searching for fled to. Suddenly"]
 
-        noble:
-          prompts: ["You are awakened by one of your servants who tells you that your keep is under attack. You look out the window and see"]
+        item1: "pistol"
+        item2: "police badge"
 
-          item1: "pouch of gold"
-          item2: "small dagger"
+      spy:
+        prompts: ["You listen to the russian diplomats and hear them discussing"]
 
-        knight:
-          prompts: ["You enter the forest where you believe the ogre that has been terrorizing your home has been hiding. You step inside and"]
+        item1: "concealed pistol"
+        item2: "syringe of poison"
 
-          item1: "steel longsword"
-          item2: "wooden shield"
+      manager:
+        prompt: ["It's late at night when you decide to head home after a long day of work. You step into the parking lot and suddenly"]
 
-        squire:
-          prompts: ["You follow Sir Theo as he enters the forest, he turns to you and says"]
+        item1: "wallet full of cash"
+        item2: "bowler hat"
 
-          item1: "spear"
-          item2: "pack of supplies"
+  apocalyptic:
+    description: " trying to survive in a post apocalyptic world by scavenging among the ruins of what is left. "
 
-        wizard:
-          prompts: ["You finish your long journey and finally arrive at the ruin you've been looking for. You look around and see"]
+    characters:
 
-          item1: "staff"
-          item2: "spellbook"
+      soldier:
+        prompts: ["You have lived in this military bunker since the bombs went off. You wake up and report for duty to your commanding officer and"]
 
-        ranger:
-          prompts: ["You spot the deer and are ready to finish your hunt when suddenly"]
+        item1: "automatic rifle"
+        item2: "grenade"
 
-          item1: "hunting bow"
-          item2: "quiver of arrows"
+      scavenger:
+        prompts: ["You walk for two hours and take a break. You've left your town in search of food. You look around and see "]
 
-        peasant:
-          prompts: ["You wake up and begin working in the fields. You see"]
+        item1: "rusty knife"
+        item2: "canteen"
 
-          item1: "pitchfork"
-          item2: "nothing else"
+      survivor:
+        prompts: ["You live in a small community of survivors who have managed to build a piece of civilization after
+        the bombs fell. You are talking to one of the leaders of the community when"]
 
-        rogue:
-          prompts: ["You successfully sneak into the merchant's house. You look around and see"]
+        item1: "hatchet"
+        item2: "hunting rifle"
 
-          item1: "long steel dagger"
-          item2: "length of rope"
+      courier:
+        prompts: ["It's a long and dangerous road from Boston to Charleston, but you're one of the only people who
+        knows the roads well enough to get your parcel of letters there. You set out in the morning and"]
+
+        item1: "parcel of letters"
+        item2: "small pistol"
+
+  zombies:
+    description: " trying to survive in a world filled with infected zombies everywhere. "
+
+    characters:
+      soldier:
+        prompts: ["Your unit lost a lot of men when the infection broke, but you've managed to keep the small town
+        you're stationed near safe for now. You look over the town and think"]
+
+        item1: "automatic rifle"
+        item2: "grenade"
+
+      survivor:
+        prompts: ["You have managed to survive several months avoiding zombies and scavenging food.
+       You cautiously enter a rundown store and hear"]
+
+        item1: "pistol"
+        item2: "backpack"
+
+      scientist:
+        prompts: ["You pound your fist on the table angry that you still haven't found the cure to the infection. You turn to your assistant and"]
+
+        item1: "backpack"
+        item2: "solar powered tablet"
 
 
 

story/story_manager.py

@@ -28,16 +28,17 @@ class Story():
         if game_state is None:
             game_state = dict()
         self.game_state = game_state
-        self.memory = 10
+        self.memory = 8
 
     def __del__(self):
-        rating = input("Please rate the story quality from 1-10: ")
-        try:
-            rating_float = float(rating)
-            self.rating = rating_float
-            self.save_to_storage()
-        except:
-            pass
+        if self.upload_story:
+            rating = input("Please rate the story quality from 1-10: ")
+            try:
+                rating_float = float(rating)
+                self.rating = rating_float
+                self.save_to_storage()
+            except:
+                pass
 
     def initialize_from_json(self, json_string):
         story_dict = json.loads(json_string)

story/utils.py

@@ -21,7 +21,6 @@ def console_print(text, width=75):
         i += 1
     print(text)
 
-
 def get_num_options(num):
 
     while True:
@@ -35,31 +34,6 @@ def get_num_options(num):
         except ValueError:
             print("Error invalid choice. ")
 
-def get_context(key):
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["contexts"][key]
-
-def get_allowed_ctrl_verbs():
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["ctrl_verbs"]["movement"] + data_loaded["ctrl_verbs"]["non_movement"]
-
-def get_story_start(key):
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["prompts"][key]
-
-
-def get_action_verbs(key):
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["action_verbs"][key]
-
 
 def player_died(text):
 
@@ -78,21 +52,6 @@ def player_won(text):
             return True
     return False
 
-
-def get_ctrl_verbs(key):
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["ctrl_verbs"][key]
-
-
-def get_rooms(key):
-    with open(YAML_FILE, 'r') as stream:
-        data_loaded = yaml.safe_load(stream)
-
-    return data_loaded["rooms"][key]
-
-
 def remove_profanity(text):
     return pf.censor(text)
 
@@ -146,7 +105,37 @@ def replace_outside_quotes(text, current_word, repl_word):
     output = reg_expr.sub(repl_word, text)
     return output
 
-    
+
+def is_first_person(text):
+
+    count = 0
+    for pair in first_to_second_mappings:
+        variations = mapping_variation_pairs(pair)
+        for variation in variations:
+            reg_expr = re.compile(variation[0] + '(?=([^"]*"[^"]*")*[^"]*$)')
+            matches = re.findall(reg_expr, text)
+            count += len(matches)
+
+    if count > 3:
+        return True
+    else:
+        return False
+
+
+def is_second_person(text):
+    count = 0
+    for pair in second_to_first_mappings:
+        variations = mapping_variation_pairs(pair)
+        for variation in variations:
+            reg_expr = re.compile(variation[0] + '(?=([^"]*"[^"]*")*[^"]*$)')
+            matches = re.findall(reg_expr, text)
+            count += len(matches)
+
+    if count > 3:
+        return True
+    else:
+        return False
+
 
 def capitalize(word):
     return word[0].upper() + word[1:]