wassname/FanFicFare
1
0
Fork 0
mirror of https://github.com/wassname/FanFicFare.git synced 2026-09-12 12:02:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

bloodshedverse & spikelover added tags in title, etc. Use stripHTML more.

Browse Source
This commit is contained in:
Jim Miller
2014-07-24 14:41:52 -05:00
parent d380f8b05c
commit df6599a9cc
2 changed files with 18 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fanficdownloader/adapters/adapter_bloodshedversecom.py
+7 -6
View File
@@ -4,6 +4,7 @@ import urllib2
import urlparse
from .. import BeautifulSoup
from ..htmlcleanup import stripHTML
from base_adapter import BaseSiteAdapter, makeDate
from .. import exceptions
@@ -74,11 +75,11 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
# Since no 404 error code we have to raise the exception ourselves.
# A title that is just 'by' indicates that there is no author name
# and no story title available.
if soup.title.string.strip() == 'by':
if stripHTML(soup.title) == 'by':
raise exceptions.StoryDoesNotExist(self.url)
for option in soup.find('select', {'name': 'chapter'}):
title = option.string.strip()
title = stripHTML(option)
url = self.READ_URL_TEMPLATE % option['value']
self.chapterUrls.append((title, url))
@@ -101,15 +102,15 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
raise exceptions.FailedToDownload(self.url)
title_anchor = list_box.find('a', {'class': 'fictitle'})
self.story.setMetadata('title', title_anchor.string.strip())
self.story.setMetadata('title', stripHTML(title_anchor))
author_anchor = title_anchor.findNextSibling('a')
self.story.setMetadata('author', author_anchor.string.strip())
self.story.setMetadata('author', stripHTML(author_anchor))
self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
list_review = list_box.find('div', {'class': 'list_review'})
reviews = list_review.a.string.strip().split(' ', 1)[0]
reviews = stripHTML(list_review.a).split(' ', 1)[0]
self.story.setMetadata('reviews', reviews)
summary_div = list_box.find('div', {'class': 'list_summary'})
@@ -122,7 +123,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
# I'm assuming this to be the category, not sure what else it could be
first_listinfo = list_box.find('div', {'class': 'list_info'})
self.story.addToList('category', first_listinfo.a.string.strip())
self.story.addToList('category', stripHTML(first_listinfo.a))
for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
for b_tag in list_info('b'):
fanficdownloader/adapters/adapter_spikeluvercom.py
+11 -10
View File
@@ -3,6 +3,7 @@ import urllib2
import urlparse
from .. import BeautifulSoup
from ..htmlcleanup import stripHTML
from base_adapter import BaseSiteAdapter, makeDate
from .. import exceptions
@@ -88,25 +89,25 @@ class SpikeluverComAdapter(BaseSiteAdapter):
soup = self._customized_fetch_url(url)
pagetitle_div = soup.find('div', id='pagetitle')
self.story.setMetadata('title', pagetitle_div.a.string.strip())
self.story.setMetadata('title', stripHTML(pagetitle_div.a))
author_anchor = pagetitle_div.a.findNextSibling('a')
url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
components = urlparse.urlparse(url)
query_data = urlparse.parse_qs(components.query)
self.story.setMetadata('author', author_anchor.string.strip())
self.story.setMetadata('author', stripHTML(author_anchor))
self.story.setMetadata('authorId', query_data['uid'])
self.story.setMetadata('authorUrl', url)
sort_div = soup.find('div', id='sort')
self.story.setMetadata('reviews', sort_div('a')[1].string.strip())
self.story.setMetadata('reviews', stripHTML(sort_div('a')[1]))
listbox_tag = soup.find('div', {'class': 'listbox'})
for span_tag in listbox_tag('span'):
key = span_tag.string.strip(' :')
try:
value = span_tag.nextSibling.string.strip()
value = stripHTML(span_tag.nextSibling)
# This can happen with some fancy markup in the summary. Just
# ignore this error and set value to None, the summary parsing
# takes care of this
@@ -145,27 +146,27 @@ class SpikeluverComAdapter(BaseSiteAdapter):
if sibling.name == 'br':
break
self.story.addToList('category', sibling.string.strip())
self.story.addToList('category', stripHTML(sibling))
# Seems to be always "None" for some reason
elif key == 'Characters':
for sibling in span_tag.findNextSiblings(['a', 'br']):
if sibling.name == 'br':
break
self.story.addToList('characters', sibling.string.strip())
self.story.addToList('characters', stripHTML(sibling))
elif key == 'Genres':
for sibling in span_tag.findNextSiblings(['a', 'br']):
if sibling.name == 'br':
break
self.story.addToList('genre', sibling.string.strip())
self.story.addToList('genre', stripHTML(sibling))
elif key == 'Warnings':
for sibling in span_tag.findNextSiblings(['a', 'br']):
if sibling.name == 'br':
break
self.story.addToList('warnings', sibling.string.strip())
self.story.addToList('warnings', stripHTML(sibling))
# Challenges
@@ -173,7 +174,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
a = span_tag.findNextSibling('a')
if not a:
continue
self.story.setMetadata('series', a.string.strip())
self.story.setMetadata('series', stripHTML(a))
self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href']))
elif key == 'Chapters':
@@ -196,7 +197,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
if not chapter_anchor:
continue
title = chapter_anchor.string.strip()
title = stripHTML(chapter_anchor)
url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
self.chapterUrls.append((title, url))