From dc785b911e1e04f5cab9df20db4721d6a1edf131 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 18 Apr 2014 14:17:26 -0500 Subject: [PATCH] Add fix_fimf_blockquotes feature, bump versions. --- app.yaml | 2 +- calibre-plugin/__init__.py | 2 +- defaults.ini | 5 +++++ .../adapters/adapter_fimfictionnet.py | 17 +++++++++++++---- index.html | 4 ++-- plugin-defaults.ini | 5 +++++ 6 files changed, 27 insertions(+), 8 deletions(-) diff --git a/app.yaml b/app.yaml index ee9d9b6..78c2f25 100644 --- a/app.yaml +++ b/app.yaml @@ -1,6 +1,6 @@ # ffd-retief-hrd fanfictiondownloader application: fanfictiondownloader -version: 4-4-98 +version: 4-4-99 runtime: python27 api_version: 1 threadsafe: true diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index 9d19ec7..22ea482 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase): description = _('UI plugin to download FanFiction stories from various sites.') supported_platforms = ['windows', 'osx', 'linux'] author = 'Jim Miller' - version = (1, 8, 17) + version = (1, 8, 18) minimum_calibre_version = (1, 13, 0) #: This field defines the GUI plugin class that contains all the code diff --git a/defaults.ini b/defaults.ini index eabdc1c..a869ae3 100644 --- a/defaults.ini +++ b/defaults.ini @@ -1314,6 +1314,11 @@ extra_valid_entries:reviews,favs,follows ## when updating to enforce accurate chapters. #do_update_hook:false +## fimfiction.net is reported to misinterprete some BBCode with +## blockquotes incorrectly. This fixes those instances and defaults +## to on, but can be switched off if it is found to cause problems. +fix_fimf_blockquotes:true + ## Site dedicated to these categories/characters/ships extracategories:My Little Pony: Friendship is Magic diff --git a/fanficdownloader/adapters/adapter_fimfictionnet.py b/fanficdownloader/adapters/adapter_fimfictionnet.py index 2829f3f..2e78bc0 100644 --- a/fanficdownloader/adapters/adapter_fimfictionnet.py +++ b/fanficdownloader/adapters/adapter_fimfictionnet.py @@ -85,7 +85,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): # Unfortunately, we still need to load the story index # page to parse the characters. And chapters, now, too. - data = self._fetchUrl(self.url) + data = self.do_fix_blockquotes(self._fetchUrl(self.url)) soup = bs.BeautifulSoup(data) except urllib2.HTTPError, e: if e.code == 404: @@ -283,12 +283,21 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): print("Existing epub has %s chapters\nNewest chapter is %s. Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1)) self.oldchapters = self.oldchapters[:self.newestChapterNum] return len(self.oldchapters) - + + def do_fix_blockquotes(self,data): + if self.getConfig('fix_fimf_blockquotes'): + #

+ #

+ # include > in re groups so there's always something in the group. + data = re.sub(r']*>\s*)]*>)',r'\s*)

',r'',data) + return data def getChapterText(self, url): logger.debug('Getting chapter text from: %s' % url) - - soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'}) + + data = self.do_fix_blockquotes(self._fetchUrl(url)) + soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'}) if soup == None: raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) return self.utf8FromSoup(url,soup) diff --git a/index.html b/index.html index 5133c92..8051f25 100644 --- a/index.html +++ b/index.html @@ -64,7 +64,7 @@

Changes:

    -
  • Fix for a problem with literotica.com multi-page chapters on Kobo readers.
    • +
  • Work around for a problem fimfiction.net blockquotes.

@@ -75,7 +75,7 @@ If you have any problems with this application, please report them in the FanFictionDownLoader Google Group. The - Previous Version is also available for you to use if necessary. + Previous Version is also available for you to use if necessary.

{{ error_message }} diff --git a/plugin-defaults.ini b/plugin-defaults.ini index e100989..6a90701 100644 --- a/plugin-defaults.ini +++ b/plugin-defaults.ini @@ -1307,6 +1307,11 @@ extra_valid_entries:reviews,favs,follows ## when updating to enforce accurate chapters. #do_update_hook:false +## fimfiction.net is reported to misinterprete some BBCode with +## blockquotes incorrectly. This fixes those instances and defaults +## to on, but can be switched off if it is found to cause problems. +fix_fimf_blockquotes:true + ## Site dedicated to these categories/characters/ships extracategories:My Little Pony: Friendship is Magic