mirror of
https://github.com/wassname/Clover-Edition.git
synced 2026-09-20 12:20:53 +08:00
42 lines
1.3 KiB
Python
42 lines
1.3 KiB
Python
import scrapy
|
|
from scrapy.crawler import CrawlerProcess
|
|
from scrapy import signals
|
|
from scrapy.utils.project import get_project_settings
|
|
from scrapy.signalmanager import dispatcher
|
|
|
|
class Scraper(scrapy.Spider):
|
|
|
|
name = 'redditbot'
|
|
#allowed_domains = ['www.reddit.com/r/gameofthrones/']
|
|
#start_urls = ['https://en.wikipedia.org/wiki/Web_scraping']
|
|
start_urls = ['http://chooseyourstory.com/story/viewer/default.aspx?StoryId=10638']
|
|
custom_settings = {
|
|
'DEPTH_LIMIT': 1
|
|
}
|
|
|
|
def parse(self, response):
|
|
links = response.css("a")[4:]
|
|
for next_page in links:
|
|
yield response.follow(next_page, self.parse)
|
|
|
|
for text in response.css('div::text'):
|
|
yield {"text": text.extract()}
|
|
|
|
def spider_results():
|
|
results = []
|
|
|
|
def crawler_results(signal, sender, item, response, spider):
|
|
results.append(item)
|
|
|
|
dispatcher.connect(crawler_results, signal=signals.item_passed)
|
|
|
|
process = CrawlerProcess(get_project_settings())
|
|
process.crawl(Scraper)
|
|
process.start() # the script will block here until the crawling is finished
|
|
return results
|
|
|
|
|
|
if __name__ == '__main__':
|
|
results = [result["text"] for result in spider_results() if "\r\n" not in result["text"]]
|
|
print("".join(results))
|