diff --git a/data/adventure/adventure_scraper.py b/data/adventure/adventure_scraper.py deleted file mode 100644 index 1fca879..0000000 --- a/data/adventure/adventure_scraper.py +++ /dev/null @@ -1,93 +0,0 @@ -from adventure import load_advent_dat -from adventure.game import Game -import re - - - -fail_phrases = ["I SEE NO ", - "I DON'T KNOW HOW TO APPLY THAT", - "REQUIRES", "GOOD TRY, BUT THAT", - "THERE IS NO WAY TO GO THAT DIRECTION", - "THERE IS NOTHING HERE TO ", - "SORRY, BUT I NO LONGER SEEM TO REMEMBER HOW IT WAS YOU GOT", - "BREAK WHAT", - "OKAY, FROM NOW ON I'LL ONLY DESCRIBE A PLACE IN FULL THE FIRST TIME", - "WHAT?", - "I DON'T KNOW WHERE", - "THERE IS NOTHING HERE WITH", - "I AM UNSURE HOW YOU ARE FACING", - "WHICH WAY?", - "IT WOULD BE ADVISABLE TO USE THE EXIT", - "DIGGING WITHOUT A SHOVEL IS QUITE IMPRACTICAL", - "YOU HAVE NO", - "OK", - "WHAT'S THE MATTER", - "NOTHING HAPPENS", - "YOU AREN'T CARRYING IT", - "I DON'T KNOW HOW", - "I DON'T KNOW THAT WORD", - "I DON'T UNDERSTAND THAT", - "YOU CAN'T BE SERIOUS", - "DON'T BE RIDICULOUS"] - -def failed(text): - for fail_phrase in fail_phrases: - if fail_phrase in text: - return True - - return False - -def get_possible_lines(commands, objects): - lines = [] - for command in commands: - for object in objects: - lines.append(command + " " + object) - - return lines + commands - -def save_game(game, file): - print(game.do_command(["save", file])) - -def load_game(game, file): - game.resume(file) - -fp = open('verbs.txt', 'r') -commands = [] -for line in fp: - commands.append(line[:-1]) - -fp = open('objects.txt', 'r') -objects = [] -for line in fp: - if line[:-1] not in commands: - objects.append(line[:-1]) - -possible_lines = get_possible_lines(commands, objects) - -game = Game() -load_advent_dat(game) -game.start() -print(game.do_command("no")) -successes = 0 -save_game(game, "file") -while not game.is_finished: - - for i, line in enumerate(possible_lines[:]): - load_game(game, "file") - action = re.findall(r'\w+', line) - result = game.do_command(action) - if "ARE YOU TRYING TO" in result: - game.do_command("no") - continue - if failed(result): - # print("FAILED: ", " ".join(action)) - # print(result) - continue - else: - print("> ", " ".join(action)) - print(result) - successes += 1 - save_game(game, "file" + str(i)) - - print("Successes is ", successes, " out of ", len(possible_lines)) - break diff --git a/data/adventure/objects.txt b/data/adventure/objects.txt deleted file mode 100644 index 0cc50c7..0000000 --- a/data/adventure/objects.txt +++ /dev/null @@ -1,284 +0,0 @@ -above -abracadabra -across -ascend -attack -awkward -axe -back -barren -bars -batteries -beans -beanstalk -bear -bed -bedquilt -bird -blast -blowup -bottle -box -break -brief -broke -broken -build -building -cage -calm -canyon -capture -carpet -carry -catch -cave -cavern -chain -chant -chasm -chest -clam -climb -close -cobble -coins -continue -crack -crawl -cross -dark -debris -depression -descend -describe -detonate -devour -diamonds -dig -discard -disturb -dome -door -downs -downstream -downward -dragon -drawings -drink -drop -dump -dwarf -dwarves -east -eat -egg -eggs -emera -emerald -enter -entrance -examine -excavate -exit -explore -extinguish -fee -feed -fie -fight -figure -fill -find -fissure -floor -foe -follow -foo -food -fores -forest -fork -forward -free -fum -get -geyser -giant -go -gold -goto -grate -gully -h2o -hall -headlamp -hill -hit -hocus -hole -hours -house -ignite -in -info -information -inside -inventory -inward -issue -jar -jewel -jewelry -jump -keep -key -keys -kill -knife -knives -lamp -lantern -leave -left -light -lock -look -lost -low -machine -magazine -main -message -ming -mirror -mist -moss -mumble -n -ne -nest -north -nothing -nowhere -nuggets -null -nw -off -office -oil -on -onward -open -opens -opensesame -oriental -out -outdo -outdoors -outside -over -oyster -passage -pearl -persian -peruse -pillow -pirate -pit -placate -plant -platinum -plover -plugh -pocus -pottery -pour -proceed -pyramid -ratio -rations -read -release -reservoir -retreat -return -right -road -rock -rod -room -rub -rug -run -save -say -se -secret -sesame -shadow -shake -shard -shards -shatter -shazam -shell -silver -sing -slab -slabroom -slit -smash -snake -south -spelunker -spice -spices -stair -staircase -stalactite -steal -steps -stop -stream -strike -surface -sw -swim -swing -table -tablet -take -tame -throw -toss -tote -touch -travel -treas -treasure -tree -trees -trident -troll -tunnel -turn -unlock -up -upstream -upward -utter -valle -valley -vase -velvet -vending -view -volcano -wake -walk -wall -water -wave -west -where \ No newline at end of file diff --git a/data/adventure/verbs.txt b/data/adventure/verbs.txt deleted file mode 100644 index 54f4e4c..0000000 --- a/data/adventure/verbs.txt +++ /dev/null @@ -1,89 +0,0 @@ -attack -blast -blowup -break -build -capture -carry -catch -chant -climb -close -continue -crack -crawl -cross -descend -describe -detonate -devour -dig -discard -disturb -drink -drop -dump -eat -egg -enter -examine -excavate -exit -explore -extinguish -feed -fight -figure -fill -find -follow -free -get -go -goto -hit -ignite -jump -keep -kill -leave -lock -look -message -mumble -open -outdo -peruse -placate -pour -proceed -read -release -retreat -return -rub -rug -say -shake -shatter -sing -slit -smash -steal -stop -strike -swim -swing -take -tame -throw -toss -tote -touch -travel -turn -unlock -view -wake -walk -wave \ No newline at end of file diff --git a/data/app.py b/data/app.py deleted file mode 100644 index 274d64b..0000000 --- a/data/app.py +++ /dev/null @@ -1,13 +0,0 @@ -import os -import flask -import Flask -app = Flask(__name__) - -@app.route("/") -def hellp_world(): - target = os.environ.get('TARGET', 'World') - return 'Hello {}!\n'.format(target) - - -if __name__ == "__main__": - app.run(debug=True, host='0.0.0.0', port=int(os.environ.get('PORT', 8080))) diff --git a/data/scraper/scraper.py b/data/scraper/scraper.py index dde842f..c7f7a73 100644 --- a/data/scraper/scraper.py +++ b/data/scraper/scraper.py @@ -1,7 +1,23 @@ from selenium import webdriver from selenium.webdriver.chrome.options import Options import time +import json +""" +format of tree is +dict { + tree_id: tree_id_text + context: context text? + first_story_block + action_results: [act_res1, act_res2, act_res3...] +} +where each action_result's format is: +dict{ + action: action_text + result: result_text + action_results: [act_res1, act_res2, act_res3...] +} +""" class Scraper: @@ -12,13 +28,15 @@ class Scraper: chrome_options.add_argument("--window-size=1920x1080") exec_path = "/usr/bin/chromedriver" self.driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=exec_path) - self.max_depth = 20 + self.max_depth = 10 + self.end_action = "End Game and Leave Comments" + self.texts = set() def GoToURL(self, url): self.driver.get(url) time.sleep(0.5) - def GatherText(self): + def GetText(self): div_elements = self.driver.find_elements_by_css_selector("div") text = div_elements[3].text return text @@ -27,23 +45,82 @@ class Scraper: return self.driver.find_elements_by_css_selector("a") def GoBack(self): - self.GetLinks().click() + self.GetLinks()[0].click() + time.sleep(0.2) - def ClickAction(self, action_num): - link = self.GetLinks()[action_num+4] - if link.text != "self.GetLinks()[action_num+4]": - self.GetLinks()[action_num+4].click() - else: - print("at the end so can no longer go forward") + def ClickAction(self, links, action_num): + links[action_num+4].click() + time.sleep(0.2) + + def GetActions(self): + return [link.text for link in self.GetLinks()[4:]] def NumActions(self): return len(self.GetLinks()) - 4 + def BuildTreeHelper(self, parent_story, action_num, depth, old_actions): + depth += 1 + action_result = {} + + action = old_actions[action_num] + print("Action is ", action) + action_result["action"] = action + + links = self.GetLinks() + if action_num+4 >= len(links): + return None + + self.ClickAction(links, action_num) + result = self.GetText() + if result == parent_story or result in self.texts: + self.GoBack() + return None + + self.texts.add(result) + print(len(self.texts)) + + action_result["result"] = result + + actions = self.GetActions() + action_result["action_results"] = [] + + for i, action in enumerate(actions): + if actions[i] != self.end_action and actions[i] != "Epilogue": + sub_action_result = self.BuildTreeHelper(result, i, depth, actions) + if action_result is not None: + action_result["action_results"].append(sub_action_result) + + self.GoBack() + return action_result + + + def BuildStoryTree(self, url): + scraper.GoToURL(url) + text = scraper.GetText() + actions = self.GetActions() + story_dict = {} + story_dict["tree_id"] = url + story_dict["context"] = "" + story_dict["first_story_block"] = text + story_dict["action_results"] = [] + + for i, action in enumerate(actions): + if action != self.end_action: + action_result = self.BuildTreeHelper(text, i, 0, actions) + if action_result is not None: + story_dict["action_results"].append(action_result) + + return story_dict + +def save_tree(tree, filename): + with open(filename, 'w') as fp: + json.dump(tree, fp) + scraper = Scraper() + urls = ["http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10638", "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=11246"] -scraper.GoToURL(urls[1]) -scraper.ClickAction(0) -text = scraper.GatherText() -links = scraper.GetLinks() -print("Done") \ No newline at end of file + +tree = scraper.BuildStoryTree(urls[0]) +save_tree(tree, "scraped_story.json") +print("done")