From 863ee5c44b2c00ee6fbd47d97491019d32422cd3 Mon Sep 17 00:00:00 2001 From: doe Date: Tue, 5 Aug 2014 17:01:43 +0200 Subject: [PATCH] Adapter for The Maple Bookshelf (themaplebookshelf.com) --- fanficdownloader/adapters/__init__.py | 1 + .../adapters/adapter_themaplebookshelf.py | 132 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 fanficdownloader/adapters/adapter_themaplebookshelf.py diff --git a/fanficdownloader/adapters/__init__.py b/fanficdownloader/adapters/__init__.py index 6ab3c6c..3770252 100644 --- a/fanficdownloader/adapters/__init__.py +++ b/fanficdownloader/adapters/__init__.py @@ -132,6 +132,7 @@ import adapter_fanfictioncsodaidokhu import adapter_fictionmaniatv import adapter_bdsmgeschichten import adapter_tolkienfanfiction +import adapter_themaplebookshelf ## This bit of complexity allows adapters to be added by just adding ## importing. It eliminates the long if/else clauses we used to need diff --git a/fanficdownloader/adapters/adapter_themaplebookshelf.py b/fanficdownloader/adapters/adapter_themaplebookshelf.py new file mode 100644 index 0000000..8deba23 --- /dev/null +++ b/fanficdownloader/adapters/adapter_themaplebookshelf.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + + +class TheMapleBookshelfComSiteAdapter(BaseSiteAdapter): + """ + Use Printable version which is easier to parse and has everything in one + page and cache between extractChapterUrlsAndMetadata and getChapterText + """ + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','maplebook') + self.decode = ["Windows-1252", "utf8"] + self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId')) + self.dateformat = "%b %d, %Y" + + @staticmethod + def getSiteDomain(): + return 'themaplebookshelf.com' + + @classmethod + def getSiteExampleURLs(self): + return [ + "http://www.themaplebookshelf.com/Literati/viewstory.php?sid=227", + "http://themaplebookshelf.com/Literati/viewstory.php?sid=227&chapter=2", + ] + + def getSiteURLPattern(self): + return r"http://themaplebookshelf.com/Literati/viewstory.php\?sid=(?P\d+)" + + def extractChapterUrlsAndMetadata(self): + logger.debug(self.url) + self._setURL(self.url + "&action=printable") + try: + html = self._fetchUrl(self.url) + soup = bs.BeautifulSoup(html) + # #strip comments from soup + # [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))] + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + ## title + author + pagetitleDiv = soup.find("div", {"id": "pagetitle"}) + self.story.setMetadata('title', pagetitleDiv.find("a").text) + authorLink = pagetitleDiv.findAll("a")[1] + self.story.setMetadata('author', authorLink.text) + self.story.setMetadata('authorUrl', "http://" + self.getSiteDomain() + "/" + authorLink['href']) + self.story.setMetadata('authorId', re.search("\d+", authorLink['href']).group(0)) + + ## Description + description = "" + summaryEnd = soup.find("div", "content").find("span", "label").nextSibling + while summaryEnd is not None: + description += stripHTML(summaryEnd) + summaryEnd = summaryEnd.nextSibling + if type(summaryEnd) != bs.NavigableString and summaryEnd.name == 'br': + break + self.story.setMetadata('description', description) + + ## General Metadata + for kSpan in soup.findAll("span", "label"): + k = kSpan.text.strip().replace(':', '') + vSpan = kSpan.nextSibling + if k == 'Summary:' or not vSpan or not vSpan.string: + continue + v = vSpan.string.strip() + if v == 'None': + continue + logger.debug("%s '%s'" %(k, v)) + if k == 'Genre': + for genre in v.split(", "): + self.story.addToList('genre', genre) + elif k == 'Chapters': + self.story.setMetadata('numChapters', int(v)) + elif k == 'Word count': + self.story.setMetadata('numWords', v) + elif k == 'Published': + self.story.setMetadata('datePublished', makeDate(v, self.dateformat)) + elif k == 'Updated': + self.story.setMetadata('dateUpdated', makeDate(v, self.dateformat)) + # TODO: Series, Warnings + + ## Chapter URLs (fragment identifiers in the document, so we don' need to fetch so much) + for chapterNumB in soup.findAll("b", text=re.compile("^\d+\.$")): + self.chapterUrls.append(( + chapterNumB.parent.parent.find("a").text, + self.url + chapterNumB.parent.parent.find("a")["href"] + )) + + ## fix all local image 'src' to absolute + for img in soup.findAll("img", {"src": re.compile("^(?!http)")}): + img['src'] = re.sub("viewstory.php\?.*", "", self.url) + img['src'] + + self.html = soup + + def getChapterText(self, url): + logger.debug('Getting chapter text from <%s>' % url) + anchor = url.replace(self.url, "") + anchor = anchor.replace("#", "") + chapterDiv = self.html.find("a", {"name": anchor}).parent.findNext("div", "chapter") + return self.utf8FromSoup(self.url, chapterDiv) + +def getClass(): + return TheMapleBookshelfComSiteAdapter