made training data

data/scraper/old_scraper.py

@@ -1,41 +0,0 @@
-import scrapy
-from scrapy.crawler import CrawlerProcess
-from scrapy import signals
-from scrapy.utils.project import get_project_settings
-from scrapy.signalmanager import dispatcher
-
-class Scraper(scrapy.Spider):
-
-    name = 'redditbot'
-    #allowed_domains = ['www.reddit.com/r/gameofthrones/']
-    #start_urls = ['https://en.wikipedia.org/wiki/Web_scraping']
-    start_urls = ['http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10638']
-    custom_settings = {
-        'DEPTH_LIMIT': 1
-    }
-
-    def parse(self, response):
-        links = response.css("a")[4:]
-        for next_page in links:
-            yield response.follow(next_page, self.parse)
-
-        for text in response.css('div::text'):
-            yield {"text": text.extract()}
-
-def spider_results():
-    results = []
-
-    def crawler_results(signal, sender, item, response, spider):
-        results.append(item)
-
-    dispatcher.connect(crawler_results, signal=signals.item_passed)
-
-    process = CrawlerProcess(get_project_settings())
-    process.crawl(Scraper)
-    process.start()  # the script will block here until the crawling is finished
-    return results
-
-
-if __name__ == '__main__':
-    results = [result["text"] for result in spider_results() if "\r\n" not in result["text"]]
-    print("".join(results))

data/scraper/scraper.py

@@ -29,8 +29,13 @@ class Scraper:
         exec_path = "/usr/bin/chromedriver"
         self.driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=exec_path)
         self.max_depth = 10
-        self.end_actions = ["End Game and Leave Comments",
-                            "See How Well You Did (you can still back-page afterwards if you like)"]
+        self.end_actions = {"End Game and Leave Comments",
+                            "Click here to End the Game and Leave Comments",
+                            "See How Well You Did (you can still back-page afterwards if you like)",
+                            "You have died.",
+                            "You have died",
+                            "Epilogue",
+                            "Save Game"}
         self.texts = set()
 
     def GoToURL(self, url):
@@ -65,7 +70,7 @@ class Scraper:
         action_result = {}
 
         action = old_actions[action_num]
-        print("Action is ", action)
+        print("Action is ", repr(action))
         action_result["action"] = action
 
         links = self.GetLinks()
@@ -87,7 +92,7 @@ class Scraper:
         action_result["action_results"] = []
 
         for i, action in enumerate(actions):
-            if actions[i] not in self.end_actions and actions[i] != "Epilogue":
+            if actions[i] not in self.end_actions:
                 sub_action_result = self.BuildTreeHelper(result, i, depth, actions)
                 if action_result is not None:
                     action_result["action_results"].append(sub_action_result)
@@ -111,6 +116,8 @@ class Scraper:
                 action_result = self.BuildTreeHelper(text, i, 0, actions)
                 if action_result is not None:
                     story_dict["action_results"].append(action_result)
+            else:
+                print("done")
 
         return story_dict
 
@@ -129,7 +136,6 @@ urls = ["http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10638",
         "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=7393",
         "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=13875",
         "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=37696",
-        "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=31353",
         "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=31013",
         "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=45375",
         "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=41698",
@@ -139,12 +145,13 @@ urls = ["http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10638",
         "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=18988",
         "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10359",
         "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=5466",
-        "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=28030"
+        "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=28030",
+        "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=56515"
         ]
 
-for i, url in enumerate(urls):
-    print("****** Extracting Adventure ", url, " ***********")
-    tree = scraper.BuildStoryTree(url)
+for i in range(19, len(urls)):
+    print("****** Extracting Adventure ", urls[i], " ***********")
+    tree = scraper.BuildStoryTree(urls[i])
     save_tree(tree, "stories/story" + str(i) + ".json")
 
 print("done")

data/scraper/stories/build_training_data.py

@@ -0,0 +1,44 @@
+import csv
+import json
+
+
+def load_tree(filename):
+    with open(filename, 'r') as fp:
+        tree = json.load(fp)
+    return tree
+
+def make_stories(current_story, tree):
+    stories = []
+    current_story += ("\n> " + tree["action"] + "\n" + tree["result"])
+
+    action_results = tree["action_results"]
+    if len(action_results) == 0 or action_results[0] is None:
+        return [current_story]
+    else:
+        stories += make_stories(current_story, action_results[0])
+
+        for i in range(1, len(action_results)):
+            if action_results[i] is not None:
+                stories += make_stories(tree["result"], action_results[i])
+
+    return stories
+
+def get_stories(filename):
+    tree = load_tree(filename)
+    stories = []
+    for action_result in tree["action_results"]:
+        stories += make_stories(tree["first_story_block"], action_result)
+    return stories
+
+
+output_file_path = "text_adventures.csv"
+with open(output_file_path, 'w') as output_file:
+    writer = csv.writer(output_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
+    filenames = ["story0.json"]
+    stories = []
+    for filename in filenames:
+        stories += get_stories(filename)
+
+    for story in stories:
+        writer.writerow([story])
+