got scraper working

data/scraper/old_scraper.py

@@ -0,0 +1,41 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from scrapy import signals
+from scrapy.utils.project import get_project_settings
+from scrapy.signalmanager import dispatcher
+
+class Scraper(scrapy.Spider):
+
+    name = 'redditbot'
+    #allowed_domains = ['www.reddit.com/r/gameofthrones/']
+    #start_urls = ['https://en.wikipedia.org/wiki/Web_scraping']
+    start_urls = ['http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10638']
+    custom_settings = {
+        'DEPTH_LIMIT': 1
+    }
+
+    def parse(self, response):
+        links = response.css("a")[4:]
+        for next_page in links:
+            yield response.follow(next_page, self.parse)
+
+        for text in response.css('div::text'):
+            yield {"text": text.extract()}
+
+def spider_results():
+    results = []
+
+    def crawler_results(signal, sender, item, response, spider):
+        results.append(item)
+
+    dispatcher.connect(crawler_results, signal=signals.item_passed)
+
+    process = CrawlerProcess(get_project_settings())
+    process.crawl(Scraper)
+    process.start()  # the script will block here until the crawling is finished
+    return results
+
+
+if __name__ == '__main__':
+    results = [result["text"] for result in spider_results() if "\r\n" not in result["text"]]
+    print("".join(results))

data/scraper/scraper.py

@@ -0,0 +1,41 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+import time
+
+
+class Scraper:
+
+    def __init__(self):
+        chrome_options = Options()
+        chrome_options.add_argument("--binary=/path/to/other/chrome/binary")
+        chrome_options.add_argument("--incognito")
+        chrome_options.add_argument("--window-size=1920x1080")
+        exec_path = "/usr/bin/chromedriver"
+        self.driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=exec_path)
+
+    def GoToURL(self, url):
+        self.driver.get(url)
+        time.sleep(0.5)
+
+    def GatherText(self):
+        div_elements = self.driver.find_elements_by_css_selector("div")
+        text = div_elements[3].text
+        return text
+
+    def GetLinks(self):
+        return self.driver.find_elements_by_css_selector("a")
+
+    def GoBack(self):
+        self.GetLinks().click()
+
+    def ClickAction(self, action_num):
+        self.GetLinks()[action_num+4].click()
+
+
+scraper = Scraper()
+urls = ["http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10638",
+        "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=11246"]
+scraper.GoToURL(urls[1])
+scraper.ClickAction(0)
+text = scraper.GatherText()
+print("Done")