Update CLI download zip.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-94
+version: 4-4-99
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 8, 13)
+    version             = (1, 8, 18)
     minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -32,6 +32,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.utils.config import prefs as calibre_prefs
 from calibre.utils.date import local_tz
 from calibre.library.comments import sanitize_comments_html
 from calibre.constants import config_dir as calibre_config_dir
@@ -856,7 +857,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         series = story.getMetadata('series')
         if not merge and series and prefs['checkforseriesurlid']:
             # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
+            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
             identicalbooks = db.search_getting_ids(searchstr, None)
             # print("searchstr:%s"%searchstr)
             # print("identicalbooks:%s"%identicalbooks)
@@ -940,13 +941,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 if len(identicalbooks) < 1:
                     # find dups
                     authlist = story.getList("author", removeallentities=True)
-                    if len(authlist) > 100 and calibre_version < (0, 8, 61):
-                        ## should be fixed from 0.8.61 on.  In the
-                        ## meantime, if it matches the title *and* first
-                        ## 100 authors, I'm prepared to assume it's a
-                        ## match.
-                        logger.debug("reduce author list to 100 only when calibre < 0.8.61")
-                        authlist = authlist[:100]
                     mi = MetaInformation(story.getMetadata("title", removeallentities=True),
                                          authlist)
                     identicalbooks = db.find_identical_books(mi)
@@ -1456,8 +1450,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
             for fmt in fmts:
                 if fmt != formmapping[options['fileform']]:
-                    logger.debug("remove f:"+fmt)
+                    logger.debug("deleteotherforms remove f:"+fmt)
                     db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+        elif prefs['autoconvert']:
+            ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+            ## the format is already there.
+            fmt = calibre_prefs['output_format']
+            # delete if there, but not if the format we just made.
+            if fmt != formmapping[options['fileform']] and \
+                    db.has_format(book_id,fmt,index_is_id=True):
+                logger.debug("autoconvert remove f:"+fmt)
+                db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                
 
         return book_id
 
@@ -1621,25 +1625,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if 'authorUrl' in book['all_metadata']:
             authurls = book['all_metadata']['authorUrl'].split(", ")
             authorlist = [ a.replace('&',';') for a in book['author'] ]
-            if hasattr(db, 'new_api'): # new_api starts in calibre 1.0.0
-                authorids = db.new_api.get_item_ids('authors',authorlist)
-                authordata = db.new_api.author_data(authorids.values())
-                # print("\n\nauthorids:%s"%authorids)
-                # print("authordata:%s"%authordata)
+            authorids = db.new_api.get_item_ids('authors',authorlist)
+            authordata = db.new_api.author_data(authorids.values())
+            # print("\n\nauthorids:%s"%authorids)
+            # print("authordata:%s"%authordata)
 
-                author_id_to_link_map = dict()
-                for i, author in enumerate(authorlist):
-                    author_id_to_link_map[authorids[author]] = authurls[i]
+            author_id_to_link_map = dict()
+            for i, author in enumerate(authorlist):
+                author_id_to_link_map[authorids[author]] = authurls[i]
 
-                # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
-                db.new_api.set_link_for_authors(author_id_to_link_map)
-            else:
-                # keep for pre-calibre 1.0.0
-                for i, auth in enumerate(authorlist):
-                    #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                    autid=db.get_author_id(auth)
-                    db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
+            # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
+            db.new_api.set_link_for_authors(author_id_to_link_map)
+            
         db.commit()
 
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
@@ -1688,7 +1685,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
 
                 if prefs['gc_polish_cover'] and \
-                        options['fileform'] == "epub" and calibre_version >= (0, 9, 39):
+                        options['fileform'] == "epub":
                     # set cover inside epub from calibre's polish feature
                     from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
                     from calibre.utils.logging import Log

calibre-plugin/jobs.py

@@ -93,16 +93,13 @@ def do_download_worker(book_list, options,
         logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
         logger.info(job.details)
 
+
+
         if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            logger.info("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
-            logger.info("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\nSuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                      filter(lambda x: x['good'], book_list) ] ) ) )
+            logger.info("\nUnsuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
             break
 
     server.close()

defaults.ini

@@ -182,6 +182,11 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
 
+## How long to wait for each HTTP connection to finish.  Longer times
+## are better for sites that are slow to respond.  Shorter times
+## prevent excessive wait when your network or the site is down.
+connect_timeout:60.0
+
 ## For use only with stand-alone CLI version--run a command on the
 ## generated file after it's produced.  All of the titlepage_entries
 ## values are available, plus output_filename.
@@ -568,6 +573,12 @@ include_in_freefromtags:freeformtags
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
+## AO3 chapters can include several different types of notes.  We've
+## traditional included them all in the chapter text, but this allows
+## you to customize which you include.  Copy this parameter to your
+## personal.ini and list the ones you don't want.
+#exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
+
 [ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1303,6 +1314,11 @@ extra_valid_entries:reviews,favs,follows
 ## when updating to enforce accurate chapters.
 #do_update_hook:false
 
+## fimfiction.net is reported to misinterprete some BBCode with
+## blockquotes incorrectly.  This fixes those instances and defaults
+## to on, but can be switched off if it is found to cause problems.
+fix_fimf_blockquotes:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
 

downloader.py

@@ -270,26 +270,24 @@ def main(argv,
            elif chaptercount == 0:
                print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
            else:
-               if not options.metaonly:
+               # update now handled by pre-populating the old
+               # images and chapters in the adapter rather than
+               # merging epubs.
+               (url,
+                chaptercount,
+                adapter.oldchapters,
+                adapter.oldimgs,
+                adapter.oldcover,
+                adapter.calibrebookmark,
+                adapter.logfile) = get_update_data(output_filename)
 
-                   # update now handled by pre-populating the old
-                   # images and chapters in the adapter rather than
-                   # merging epubs.
-                   (url,
-                    chaptercount,
-                    adapter.oldchapters,
-                    adapter.oldimgs,
-                    adapter.oldcover,
-                    adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(output_filename)
+               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
 
-                   print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+               if not (options.update and chaptercount == urlchaptercount) \
+                       and adapter.getConfig("do_update_hook"):
+                   chaptercount = adapter.hookForUpdates(chaptercount)
 
-                   if not (options.update and chaptercount == urlchaptercount) \
-                           and adapter.getConfig("do_update_hook"):
-                       chaptercount = adapter.hookForUpdates(chaptercount)
-
-                   writeStory(configuration,adapter,"epub")
+               writeStory(configuration,adapter,"epub")
                    
        else:
            # regular download

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -84,7 +84,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
         # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
+        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -321,25 +321,31 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-		
-        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-        if headnotes != None:
-            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+
+        exclude_notes=self.getConfigList('exclude_notes')
+
+        if 'authorheadnotes' not in exclude_notes:
+            headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
             if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(headnotes)
+                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+                if headnotes != None:
+                    chapter.append("<b>Author's Note:</b>")
+                    chapter.append(headnotes)
         
-        chapsumm = soup.find('div', {'id' : "summary"})
-        if chapsumm != None:
-            chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
-            chapter.append(chapsumm)
-        chapnotes = soup.find('div', {'id' : "notes"})
-        if chapnotes != None:
-            chapnotes = chapnotes.find('blockquote')
+        if 'chaptersummary' not in exclude_notes:
+            chapsumm = soup.find('div', {'id' : "summary"})
+            if chapsumm != None:
+                chapsumm = chapsumm.find('blockquote')
+                chapter.append("<b>Summary for the Chapter:</b>")
+                chapter.append(chapsumm)
+                
+        if 'chapterheadnotes' not in exclude_notes:
+            chapnotes = soup.find('div', {'id' : "notes"})
             if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapnotes)
+                chapnotes = chapnotes.find('blockquote')
+                if chapnotes != None:
+                    chapter.append("<b>Notes for the Chapter:</b>")
+                    chapter.append(chapnotes)
 		
         text = soup.find('div', {'class' : "userstuff module"})
         chtext = text.find('h3', {'class' : "landmark heading"})
@@ -347,17 +353,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chtext.extract()
         chapter.append(text)
 		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-        if chapfoot != None:
-            chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
-            chapter.append(chapfoot)
+        if 'chapterfootnotes' not in exclude_notes:
+            chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+            if chapfoot != None:
+                chapfoot = chapfoot.find('blockquote')
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapfoot)
 		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        if footnotes != None:
-            footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
-            chapter.append(footnotes)
+        if 'authorfootnotes' not in exclude_notes:
+            footnotes = soup.find('div', {'id' : "work_endnotes"})
+            if footnotes != None:
+                footnotes = footnotes.find('blockquote')
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(footnotes)
 			
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -41,10 +41,10 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
+        return "https://www.fictionpress.com/s/1234/1/ https://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"https?://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
 
 def getClass():
     return FictionPressComSiteAdapter

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -85,7 +85,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             
             # Unfortunately, we still need to load the story index
             # page to parse the characters.  And chapters, now, too.
-            data = self._fetchUrl(self.url)
+            data = self.do_fix_blockquotes(self._fetchUrl(self.url))
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -283,12 +283,21 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
             self.oldchapters = self.oldchapters[:self.newestChapterNum]
         return len(self.oldchapters)
-            
+
+    def do_fix_blockquotes(self,data):
+        if self.getConfig('fix_fimf_blockquotes'):
+            # <p class="double"><blockquote>
+            # </blockquote></p>
+            # include > in re groups so there's always something in the group.
+            data = re.sub(r'<p([^>]*>\s*)<blockquote([^>]*>)',r'<blockquote\2<p\1',data)
+            data = re.sub(r'</blockquote(>\s*)</p>',r'</p\1</blockquote>',data)
+        return data
         
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
+
+        data = self.do_fix_blockquotes(self._fetchUrl(url))
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -160,6 +160,11 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
             if 'Warnings' in label:
                 warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                 for warning in warnings:

fanficdownloader/adapters/adapter_literotica.py

@@ -203,7 +203,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         # get story text
         story1 = soup1.find('div', 'b-story-body-x').p
         story1.name='div'
-        story1.append('<br>')
+        story1.append('<br />')
         storytext = self.utf8FromSoup(url,story1)
 
         # find num pages
@@ -220,7 +220,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
                 [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
                 story2 = soup2.find('div', 'b-story-body-x').p
                 story2.name='div'
-                story2.append('<br>')
+                story2.append('<br />')
                 storytext += self.utf8FromSoup(url,story2)
             except urllib2.HTTPError, e:
                 if e.code == 404:

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -163,12 +163,11 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
+                # limit date values, there's some extra chars.
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y"))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y"))
 
         try:
             # Find Series name from series URL.

fanficdownloader/adapters/base_adapter.py

@@ -160,13 +160,13 @@ class BaseSiteAdapter(Configurable):
         req = u2.Request(url,
                          data=urllib.urlencode(parameters),
                          headers=headers)
-        return self._decode(self.opener.open(req).read())
+        return self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())
 
     def _fetchUrlRaw(self, url, parameters=None):
         if parameters != None:
-            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters)).read()
+            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
         else:
-            return self.opener.open(url.replace(' ','%20')).read()
+            return self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
 
     def set_sleep(self,val):
         print("\n===========\n set sleep time %s\n==========="%val)

index.html

@@ -60,14 +60,13 @@
 	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
-	  <!-- put announcements here, h3 is a good title size.
+	  <!-- put announcements here, h3 is a good title size. -->
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>Fix for cover image.</li>
+              <li>Work around for a problem fimfiction.net blockquotes.</li>
 	    </ul>
 	  </p>
-           -->
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -76,7 +75,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-92.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-98.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -154,6 +154,11 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
 
+## How long to wait for each HTTP connection to finish.  Longer times
+## are better for sites that are slow to respond.  Shorter times
+## prevent excessive wait when your network or the site is down.
+connect_timeout:60.0
+
 ## Use regular expressions to find and replace (or remove) metadata.
 ## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
 ## etc.  See http://docs.python.org/library/re.html (look for re.sub)
@@ -537,6 +542,12 @@ include_in_freefromtags:freeformtags
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
+## AO3 chapters can include several different types of notes.  We've
+## traditional included them all in the chapter text, but this allows
+## you to customize which you include.  Copy this parameter to your
+## personal.ini and list the ones you don't want.
+#exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
+
 [ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1296,6 +1307,11 @@ extra_valid_entries:reviews,favs,follows
 ## when updating to enforce accurate chapters.
 #do_update_hook:false
 
+## fimfiction.net is reported to misinterprete some BBCode with
+## blockquotes incorrectly.  This fixes those instances and defaults
+## to on, but can be switched off if it is found to cause problems.
+fix_fimf_blockquotes:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic