Improve exception handling for the http://bloodshedverse.com/ adapter

This commit is contained in:
cryzed
2014-06-08 12:48:01 +02:00
parent f24c363d3b
commit 6e38557454
@@ -39,11 +39,16 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
self._setURL(self.READ_URL_TEMPLATE % story_no)
self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
def _customized_fetch_url(self, url, exception, parameters=None):
try:
def _customized_fetch_url(self, url, exception=None, parameters=None):
if exception:
try:
data = self._fetchUrl(url, parameters)
except urllib2.HTTPError:
raise exception(self.url)
# Just let self._fetchUrl throw the exception, don't catch and
# customize it.
else:
data = self._fetchUrl(url, parameters)
except urllib2.HTTPError:
raise exception(self.url)
return BeautifulSoup.BeautifulSoup(data)
@@ -64,9 +69,9 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
return url
def extractChapterUrlsAndMetadata(self):
# Use FailedToDownload exception because that's the only error that can
# Use URLError exception because that's the only error that can
# occur here, missing stories don't return a 404 header
soup = self._customized_fetch_url(self.url, exceptions.FailedToDownload)
soup = self._customized_fetch_url(self.url)
# Since no 404 error code we have to raise the exception ourselves.
# A title that is just 'by' indicates that there is no author name
@@ -82,7 +87,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
# Get the URL to the author's page and find the correct story entry to
# scrape the metadata
author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
soup = self._customized_fetch_url(author_url, exceptions.FailedToDownload)
soup = self._customized_fetch_url(author_url)
story_no = self.story.getMetadata('storyId')
# Ignore first list_box div, it only contains the author information
@@ -176,7 +181,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
raise exceptions.AdultCheckRequired(self.url)
def getChapterText(self, url):
soup = self._customized_fetch_url(url, exceptions.FailedToDownload)
soup = self._customized_fetch_url(url)
storytext_div = soup.find('div', {'class': 'storytext'})
if self.getConfig('strip_text_links'):