mirror of
https://github.com/wassname/Clover-Edition.git
synced 2026-09-09 11:13:26 +08:00
updated version of scraper
This commit is contained in:
Nick
@@ -1,93 +0,0 @@
|
||||
from adventure import load_advent_dat
|
||||
from adventure.game import Game
|
||||
import re
|
||||
|
||||
|
||||
|
||||
fail_phrases = ["I SEE NO ",
|
||||
"I DON'T KNOW HOW TO APPLY THAT",
|
||||
"REQUIRES", "GOOD TRY, BUT THAT",
|
||||
"THERE IS NO WAY TO GO THAT DIRECTION",
|
||||
"THERE IS NOTHING HERE TO ",
|
||||
"SORRY, BUT I NO LONGER SEEM TO REMEMBER HOW IT WAS YOU GOT",
|
||||
"BREAK WHAT",
|
||||
"OKAY, FROM NOW ON I'LL ONLY DESCRIBE A PLACE IN FULL THE FIRST TIME",
|
||||
"WHAT?",
|
||||
"I DON'T KNOW WHERE",
|
||||
"THERE IS NOTHING HERE WITH",
|
||||
"I AM UNSURE HOW YOU ARE FACING",
|
||||
"WHICH WAY?",
|
||||
"IT WOULD BE ADVISABLE TO USE THE EXIT",
|
||||
"DIGGING WITHOUT A SHOVEL IS QUITE IMPRACTICAL",
|
||||
"YOU HAVE NO",
|
||||
"OK",
|
||||
"WHAT'S THE MATTER",
|
||||
"NOTHING HAPPENS",
|
||||
"YOU AREN'T CARRYING IT",
|
||||
"I DON'T KNOW HOW",
|
||||
"I DON'T KNOW THAT WORD",
|
||||
"I DON'T UNDERSTAND THAT",
|
||||
"YOU CAN'T BE SERIOUS",
|
||||
"DON'T BE RIDICULOUS"]
|
||||
|
||||
def failed(text):
|
||||
for fail_phrase in fail_phrases:
|
||||
if fail_phrase in text:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def get_possible_lines(commands, objects):
|
||||
lines = []
|
||||
for command in commands:
|
||||
for object in objects:
|
||||
lines.append(command + " " + object)
|
||||
|
||||
return lines + commands
|
||||
|
||||
def save_game(game, file):
|
||||
print(game.do_command(["save", file]))
|
||||
|
||||
def load_game(game, file):
|
||||
game.resume(file)
|
||||
|
||||
fp = open('verbs.txt', 'r')
|
||||
commands = []
|
||||
for line in fp:
|
||||
commands.append(line[:-1])
|
||||
|
||||
fp = open('objects.txt', 'r')
|
||||
objects = []
|
||||
for line in fp:
|
||||
if line[:-1] not in commands:
|
||||
objects.append(line[:-1])
|
||||
|
||||
possible_lines = get_possible_lines(commands, objects)
|
||||
|
||||
game = Game()
|
||||
load_advent_dat(game)
|
||||
game.start()
|
||||
print(game.do_command("no"))
|
||||
successes = 0
|
||||
save_game(game, "file")
|
||||
while not game.is_finished:
|
||||
|
||||
for i, line in enumerate(possible_lines[:]):
|
||||
load_game(game, "file")
|
||||
action = re.findall(r'\w+', line)
|
||||
result = game.do_command(action)
|
||||
if "ARE YOU TRYING TO" in result:
|
||||
game.do_command("no")
|
||||
continue
|
||||
if failed(result):
|
||||
# print("FAILED: ", " ".join(action))
|
||||
# print(result)
|
||||
continue
|
||||
else:
|
||||
print("> ", " ".join(action))
|
||||
print(result)
|
||||
successes += 1
|
||||
save_game(game, "file" + str(i))
|
||||
|
||||
print("Successes is ", successes, " out of ", len(possible_lines))
|
||||
break
|
||||
@@ -1,284 +0,0 @@
|
||||
above
|
||||
abracadabra
|
||||
across
|
||||
ascend
|
||||
attack
|
||||
awkward
|
||||
axe
|
||||
back
|
||||
barren
|
||||
bars
|
||||
batteries
|
||||
beans
|
||||
beanstalk
|
||||
bear
|
||||
bed
|
||||
bedquilt
|
||||
bird
|
||||
blast
|
||||
blowup
|
||||
bottle
|
||||
box
|
||||
break
|
||||
brief
|
||||
broke
|
||||
broken
|
||||
build
|
||||
building
|
||||
cage
|
||||
calm
|
||||
canyon
|
||||
capture
|
||||
carpet
|
||||
carry
|
||||
catch
|
||||
cave
|
||||
cavern
|
||||
chain
|
||||
chant
|
||||
chasm
|
||||
chest
|
||||
clam
|
||||
climb
|
||||
close
|
||||
cobble
|
||||
coins
|
||||
continue
|
||||
crack
|
||||
crawl
|
||||
cross
|
||||
dark
|
||||
debris
|
||||
depression
|
||||
descend
|
||||
describe
|
||||
detonate
|
||||
devour
|
||||
diamonds
|
||||
dig
|
||||
discard
|
||||
disturb
|
||||
dome
|
||||
door
|
||||
downs
|
||||
downstream
|
||||
downward
|
||||
dragon
|
||||
drawings
|
||||
drink
|
||||
drop
|
||||
dump
|
||||
dwarf
|
||||
dwarves
|
||||
east
|
||||
eat
|
||||
egg
|
||||
eggs
|
||||
emera
|
||||
emerald
|
||||
enter
|
||||
entrance
|
||||
examine
|
||||
excavate
|
||||
exit
|
||||
explore
|
||||
extinguish
|
||||
fee
|
||||
feed
|
||||
fie
|
||||
fight
|
||||
figure
|
||||
fill
|
||||
find
|
||||
fissure
|
||||
floor
|
||||
foe
|
||||
follow
|
||||
foo
|
||||
food
|
||||
fores
|
||||
forest
|
||||
fork
|
||||
forward
|
||||
free
|
||||
fum
|
||||
get
|
||||
geyser
|
||||
giant
|
||||
go
|
||||
gold
|
||||
goto
|
||||
grate
|
||||
gully
|
||||
h2o
|
||||
hall
|
||||
headlamp
|
||||
hill
|
||||
hit
|
||||
hocus
|
||||
hole
|
||||
hours
|
||||
house
|
||||
ignite
|
||||
in
|
||||
info
|
||||
information
|
||||
inside
|
||||
inventory
|
||||
inward
|
||||
issue
|
||||
jar
|
||||
jewel
|
||||
jewelry
|
||||
jump
|
||||
keep
|
||||
key
|
||||
keys
|
||||
kill
|
||||
knife
|
||||
knives
|
||||
lamp
|
||||
lantern
|
||||
leave
|
||||
left
|
||||
light
|
||||
lock
|
||||
look
|
||||
lost
|
||||
low
|
||||
machine
|
||||
magazine
|
||||
main
|
||||
message
|
||||
ming
|
||||
mirror
|
||||
mist
|
||||
moss
|
||||
mumble
|
||||
n
|
||||
ne
|
||||
nest
|
||||
north
|
||||
nothing
|
||||
nowhere
|
||||
nuggets
|
||||
null
|
||||
nw
|
||||
off
|
||||
office
|
||||
oil
|
||||
on
|
||||
onward
|
||||
open
|
||||
opens
|
||||
opensesame
|
||||
oriental
|
||||
out
|
||||
outdo
|
||||
outdoors
|
||||
outside
|
||||
over
|
||||
oyster
|
||||
passage
|
||||
pearl
|
||||
persian
|
||||
peruse
|
||||
pillow
|
||||
pirate
|
||||
pit
|
||||
placate
|
||||
plant
|
||||
platinum
|
||||
plover
|
||||
plugh
|
||||
pocus
|
||||
pottery
|
||||
pour
|
||||
proceed
|
||||
pyramid
|
||||
ratio
|
||||
rations
|
||||
read
|
||||
release
|
||||
reservoir
|
||||
retreat
|
||||
return
|
||||
right
|
||||
road
|
||||
rock
|
||||
rod
|
||||
room
|
||||
rub
|
||||
rug
|
||||
run
|
||||
save
|
||||
say
|
||||
se
|
||||
secret
|
||||
sesame
|
||||
shadow
|
||||
shake
|
||||
shard
|
||||
shards
|
||||
shatter
|
||||
shazam
|
||||
shell
|
||||
silver
|
||||
sing
|
||||
slab
|
||||
slabroom
|
||||
slit
|
||||
smash
|
||||
snake
|
||||
south
|
||||
spelunker
|
||||
spice
|
||||
spices
|
||||
stair
|
||||
staircase
|
||||
stalactite
|
||||
steal
|
||||
steps
|
||||
stop
|
||||
stream
|
||||
strike
|
||||
surface
|
||||
sw
|
||||
swim
|
||||
swing
|
||||
table
|
||||
tablet
|
||||
take
|
||||
tame
|
||||
throw
|
||||
toss
|
||||
tote
|
||||
touch
|
||||
travel
|
||||
treas
|
||||
treasure
|
||||
tree
|
||||
trees
|
||||
trident
|
||||
troll
|
||||
tunnel
|
||||
turn
|
||||
unlock
|
||||
up
|
||||
upstream
|
||||
upward
|
||||
utter
|
||||
valle
|
||||
valley
|
||||
vase
|
||||
velvet
|
||||
vending
|
||||
view
|
||||
volcano
|
||||
wake
|
||||
walk
|
||||
wall
|
||||
water
|
||||
wave
|
||||
west
|
||||
where
|
||||
@@ -1,89 +0,0 @@
|
||||
attack
|
||||
blast
|
||||
blowup
|
||||
break
|
||||
build
|
||||
capture
|
||||
carry
|
||||
catch
|
||||
chant
|
||||
climb
|
||||
close
|
||||
continue
|
||||
crack
|
||||
crawl
|
||||
cross
|
||||
descend
|
||||
describe
|
||||
detonate
|
||||
devour
|
||||
dig
|
||||
discard
|
||||
disturb
|
||||
drink
|
||||
drop
|
||||
dump
|
||||
eat
|
||||
egg
|
||||
enter
|
||||
examine
|
||||
excavate
|
||||
exit
|
||||
explore
|
||||
extinguish
|
||||
feed
|
||||
fight
|
||||
figure
|
||||
fill
|
||||
find
|
||||
follow
|
||||
free
|
||||
get
|
||||
go
|
||||
goto
|
||||
hit
|
||||
ignite
|
||||
jump
|
||||
keep
|
||||
kill
|
||||
leave
|
||||
lock
|
||||
look
|
||||
message
|
||||
mumble
|
||||
open
|
||||
outdo
|
||||
peruse
|
||||
placate
|
||||
pour
|
||||
proceed
|
||||
read
|
||||
release
|
||||
retreat
|
||||
return
|
||||
rub
|
||||
rug
|
||||
say
|
||||
shake
|
||||
shatter
|
||||
sing
|
||||
slit
|
||||
smash
|
||||
steal
|
||||
stop
|
||||
strike
|
||||
swim
|
||||
swing
|
||||
take
|
||||
tame
|
||||
throw
|
||||
toss
|
||||
tote
|
||||
touch
|
||||
travel
|
||||
turn
|
||||
unlock
|
||||
view
|
||||
wake
|
||||
walk
|
||||
wave
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
import os
|
||||
import flask
|
||||
import Flask
|
||||
app = Flask(__name__)
|
||||
|
||||
@app.route("/")
|
||||
def hellp_world():
|
||||
target = os.environ.get('TARGET', 'World')
|
||||
return 'Hello {}!\n'.format(target)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(debug=True, host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))
|
||||
+91
-14
@@ -1,7 +1,23 @@
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
import time
|
||||
import json
|
||||
"""
|
||||
format of tree is
|
||||
dict {
|
||||
tree_id: tree_id_text
|
||||
context: context text?
|
||||
first_story_block
|
||||
action_results: [act_res1, act_res2, act_res3...]
|
||||
}
|
||||
|
||||
where each action_result's format is:
|
||||
dict{
|
||||
action: action_text
|
||||
result: result_text
|
||||
action_results: [act_res1, act_res2, act_res3...]
|
||||
}
|
||||
"""
|
||||
|
||||
class Scraper:
|
||||
|
||||
@@ -12,13 +28,15 @@ class Scraper:
|
||||
chrome_options.add_argument("--window-size=1920x1080")
|
||||
exec_path = "/usr/bin/chromedriver"
|
||||
self.driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=exec_path)
|
||||
self.max_depth = 20
|
||||
self.max_depth = 10
|
||||
self.end_action = "End Game and Leave Comments"
|
||||
self.texts = set()
|
||||
|
||||
def GoToURL(self, url):
|
||||
self.driver.get(url)
|
||||
time.sleep(0.5)
|
||||
|
||||
def GatherText(self):
|
||||
def GetText(self):
|
||||
div_elements = self.driver.find_elements_by_css_selector("div")
|
||||
text = div_elements[3].text
|
||||
return text
|
||||
@@ -27,23 +45,82 @@ class Scraper:
|
||||
return self.driver.find_elements_by_css_selector("a")
|
||||
|
||||
def GoBack(self):
|
||||
self.GetLinks().click()
|
||||
self.GetLinks()[0].click()
|
||||
time.sleep(0.2)
|
||||
|
||||
def ClickAction(self, action_num):
|
||||
link = self.GetLinks()[action_num+4]
|
||||
if link.text != "self.GetLinks()[action_num+4]":
|
||||
self.GetLinks()[action_num+4].click()
|
||||
else:
|
||||
print("at the end so can no longer go forward")
|
||||
def ClickAction(self, links, action_num):
|
||||
links[action_num+4].click()
|
||||
time.sleep(0.2)
|
||||
|
||||
def GetActions(self):
|
||||
return [link.text for link in self.GetLinks()[4:]]
|
||||
|
||||
def NumActions(self):
|
||||
return len(self.GetLinks()) - 4
|
||||
|
||||
def BuildTreeHelper(self, parent_story, action_num, depth, old_actions):
|
||||
depth += 1
|
||||
action_result = {}
|
||||
|
||||
action = old_actions[action_num]
|
||||
print("Action is ", action)
|
||||
action_result["action"] = action
|
||||
|
||||
links = self.GetLinks()
|
||||
if action_num+4 >= len(links):
|
||||
return None
|
||||
|
||||
self.ClickAction(links, action_num)
|
||||
result = self.GetText()
|
||||
if result == parent_story or result in self.texts:
|
||||
self.GoBack()
|
||||
return None
|
||||
|
||||
self.texts.add(result)
|
||||
print(len(self.texts))
|
||||
|
||||
action_result["result"] = result
|
||||
|
||||
actions = self.GetActions()
|
||||
action_result["action_results"] = []
|
||||
|
||||
for i, action in enumerate(actions):
|
||||
if actions[i] != self.end_action and actions[i] != "Epilogue":
|
||||
sub_action_result = self.BuildTreeHelper(result, i, depth, actions)
|
||||
if action_result is not None:
|
||||
action_result["action_results"].append(sub_action_result)
|
||||
|
||||
self.GoBack()
|
||||
return action_result
|
||||
|
||||
|
||||
def BuildStoryTree(self, url):
|
||||
scraper.GoToURL(url)
|
||||
text = scraper.GetText()
|
||||
actions = self.GetActions()
|
||||
story_dict = {}
|
||||
story_dict["tree_id"] = url
|
||||
story_dict["context"] = ""
|
||||
story_dict["first_story_block"] = text
|
||||
story_dict["action_results"] = []
|
||||
|
||||
for i, action in enumerate(actions):
|
||||
if action != self.end_action:
|
||||
action_result = self.BuildTreeHelper(text, i, 0, actions)
|
||||
if action_result is not None:
|
||||
story_dict["action_results"].append(action_result)
|
||||
|
||||
return story_dict
|
||||
|
||||
def save_tree(tree, filename):
|
||||
with open(filename, 'w') as fp:
|
||||
json.dump(tree, fp)
|
||||
|
||||
scraper = Scraper()
|
||||
|
||||
urls = ["http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10638",
|
||||
"http://chooseyourstory.com/story/viewer/default.aspx?StoryId=11246"]
|
||||
scraper.GoToURL(urls[1])
|
||||
scraper.ClickAction(0)
|
||||
text = scraper.GatherText()
|
||||
links = scraper.GetLinks()
|
||||
print("Done")
|
||||
|
||||
tree = scraper.BuildStoryTree(urls[0])
|
||||
save_tree(tree, "scraped_story.json")
|
||||
print("done")
|
||||
|
||||
Reference in New Issue
Block a user