diff --git a/data/scraper/old_scraper.py b/data/scraper/old_scraper.py new file mode 100644 index 0000000..2167709 --- /dev/null +++ b/data/scraper/old_scraper.py @@ -0,0 +1,41 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy import signals +from scrapy.utils.project import get_project_settings +from scrapy.signalmanager import dispatcher + +class Scraper(scrapy.Spider): + + name = 'redditbot' + #allowed_domains = ['www.reddit.com/r/gameofthrones/'] + #start_urls = ['https://en.wikipedia.org/wiki/Web_scraping'] + start_urls = ['http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10638'] + custom_settings = { + 'DEPTH_LIMIT': 1 + } + + def parse(self, response): + links = response.css("a")[4:] + for next_page in links: + yield response.follow(next_page, self.parse) + + for text in response.css('div::text'): + yield {"text": text.extract()} + +def spider_results(): + results = [] + + def crawler_results(signal, sender, item, response, spider): + results.append(item) + + dispatcher.connect(crawler_results, signal=signals.item_passed) + + process = CrawlerProcess(get_project_settings()) + process.crawl(Scraper) + process.start() # the script will block here until the crawling is finished + return results + + +if __name__ == '__main__': + results = [result["text"] for result in spider_results() if "\r\n" not in result["text"]] + print("".join(results)) diff --git a/data/scraper/scraper.py b/data/scraper/scraper.py new file mode 100644 index 0000000..72685a9 --- /dev/null +++ b/data/scraper/scraper.py @@ -0,0 +1,41 @@ +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +import time + + +class Scraper: + + def __init__(self): + chrome_options = Options() + chrome_options.add_argument("--binary=/path/to/other/chrome/binary") + chrome_options.add_argument("--incognito") + chrome_options.add_argument("--window-size=1920x1080") + exec_path = "/usr/bin/chromedriver" + self.driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=exec_path) + + def GoToURL(self, url): + self.driver.get(url) + time.sleep(0.5) + + def GatherText(self): + div_elements = self.driver.find_elements_by_css_selector("div") + text = div_elements[3].text + return text + + def GetLinks(self): + return self.driver.find_elements_by_css_selector("a") + + def GoBack(self): + self.GetLinks().click() + + def ClickAction(self, action_num): + self.GetLinks()[action_num+4].click() + + +scraper = Scraper() +urls = ["http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10638", + "http://chooseyourstory.com/story/viewer/default.aspx?StoryId=11246"] +scraper.GoToURL(urls[1]) +scraper.ClickAction(0) +text = scraper.GatherText() +print("Done") \ No newline at end of file