From df6599a9cc9d9266928f45d0d6c71a1a7210b9f1 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Thu, 24 Jul 2014 14:41:52 -0500 Subject: [PATCH] bloodshedverse & spikelover added tags in title, etc. Use stripHTML more. --- .../adapters/adapter_bloodshedversecom.py | 13 ++++++------ .../adapters/adapter_spikeluvercom.py | 21 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/fanficdownloader/adapters/adapter_bloodshedversecom.py b/fanficdownloader/adapters/adapter_bloodshedversecom.py index 66b2712..a47db27 100644 --- a/fanficdownloader/adapters/adapter_bloodshedversecom.py +++ b/fanficdownloader/adapters/adapter_bloodshedversecom.py @@ -4,6 +4,7 @@ import urllib2 import urlparse from .. import BeautifulSoup +from ..htmlcleanup import stripHTML from base_adapter import BaseSiteAdapter, makeDate from .. import exceptions @@ -74,11 +75,11 @@ class BloodshedverseComAdapter(BaseSiteAdapter): # Since no 404 error code we have to raise the exception ourselves. # A title that is just 'by' indicates that there is no author name # and no story title available. - if soup.title.string.strip() == 'by': + if stripHTML(soup.title) == 'by': raise exceptions.StoryDoesNotExist(self.url) for option in soup.find('select', {'name': 'chapter'}): - title = option.string.strip() + title = stripHTML(option) url = self.READ_URL_TEMPLATE % option['value'] self.chapterUrls.append((title, url)) @@ -101,15 +102,15 @@ class BloodshedverseComAdapter(BaseSiteAdapter): raise exceptions.FailedToDownload(self.url) title_anchor = list_box.find('a', {'class': 'fictitle'}) - self.story.setMetadata('title', title_anchor.string.strip()) + self.story.setMetadata('title', stripHTML(title_anchor)) author_anchor = title_anchor.findNextSibling('a') - self.story.setMetadata('author', author_anchor.string.strip()) + self.story.setMetadata('author', stripHTML(author_anchor)) self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who']) self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href'])) list_review = list_box.find('div', {'class': 'list_review'}) - reviews = list_review.a.string.strip().split(' ', 1)[0] + reviews = stripHTML(list_review.a).split(' ', 1)[0] self.story.setMetadata('reviews', reviews) summary_div = list_box.find('div', {'class': 'list_summary'}) @@ -122,7 +123,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter): # I'm assuming this to be the category, not sure what else it could be first_listinfo = list_box.find('div', {'class': 'list_info'}) - self.story.addToList('category', first_listinfo.a.string.strip()) + self.story.addToList('category', stripHTML(first_listinfo.a)) for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}): for b_tag in list_info('b'): diff --git a/fanficdownloader/adapters/adapter_spikeluvercom.py b/fanficdownloader/adapters/adapter_spikeluvercom.py index 62fa40d..ae37c37 100644 --- a/fanficdownloader/adapters/adapter_spikeluvercom.py +++ b/fanficdownloader/adapters/adapter_spikeluvercom.py @@ -3,6 +3,7 @@ import urllib2 import urlparse from .. import BeautifulSoup +from ..htmlcleanup import stripHTML from base_adapter import BaseSiteAdapter, makeDate from .. import exceptions @@ -88,25 +89,25 @@ class SpikeluverComAdapter(BaseSiteAdapter): soup = self._customized_fetch_url(url) pagetitle_div = soup.find('div', id='pagetitle') - self.story.setMetadata('title', pagetitle_div.a.string.strip()) + self.story.setMetadata('title', stripHTML(pagetitle_div.a)) author_anchor = pagetitle_div.a.findNextSibling('a') url = urlparse.urljoin(self.BASE_URL, author_anchor['href']) components = urlparse.urlparse(url) query_data = urlparse.parse_qs(components.query) - self.story.setMetadata('author', author_anchor.string.strip()) + self.story.setMetadata('author', stripHTML(author_anchor)) self.story.setMetadata('authorId', query_data['uid']) self.story.setMetadata('authorUrl', url) sort_div = soup.find('div', id='sort') - self.story.setMetadata('reviews', sort_div('a')[1].string.strip()) + self.story.setMetadata('reviews', stripHTML(sort_div('a')[1])) listbox_tag = soup.find('div', {'class': 'listbox'}) for span_tag in listbox_tag('span'): key = span_tag.string.strip(' :') try: - value = span_tag.nextSibling.string.strip() + value = stripHTML(span_tag.nextSibling) # This can happen with some fancy markup in the summary. Just # ignore this error and set value to None, the summary parsing # takes care of this @@ -145,27 +146,27 @@ class SpikeluverComAdapter(BaseSiteAdapter): if sibling.name == 'br': break - self.story.addToList('category', sibling.string.strip()) + self.story.addToList('category', stripHTML(sibling)) # Seems to be always "None" for some reason elif key == 'Characters': for sibling in span_tag.findNextSiblings(['a', 'br']): if sibling.name == 'br': break - self.story.addToList('characters', sibling.string.strip()) + self.story.addToList('characters', stripHTML(sibling)) elif key == 'Genres': for sibling in span_tag.findNextSiblings(['a', 'br']): if sibling.name == 'br': break - self.story.addToList('genre', sibling.string.strip()) + self.story.addToList('genre', stripHTML(sibling)) elif key == 'Warnings': for sibling in span_tag.findNextSiblings(['a', 'br']): if sibling.name == 'br': break - self.story.addToList('warnings', sibling.string.strip()) + self.story.addToList('warnings', stripHTML(sibling)) # Challenges @@ -173,7 +174,7 @@ class SpikeluverComAdapter(BaseSiteAdapter): a = span_tag.findNextSibling('a') if not a: continue - self.story.setMetadata('series', a.string.strip()) + self.story.setMetadata('series', stripHTML(a)) self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href'])) elif key == 'Chapters': @@ -196,7 +197,7 @@ class SpikeluverComAdapter(BaseSiteAdapter): if not chapter_anchor: continue - title = chapter_anchor.string.strip() + title = stripHTML(chapter_anchor) url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href']) self.chapterUrls.append((title, url))