Update CLI download zip.

Also don't retry 404 errors and bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-89
+version: 4-4-95
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 8, 06)
+    version             = (1, 8, 14)
     minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -55,7 +55,7 @@ from calibre_plugins.fanfictiondownloader_plugin.dialogs \
             EditTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
-    import (getConfigSections, getNormalStoryURL)
+    import getConfigSections
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
@@ -209,6 +209,7 @@ class ConfigWidget(QWidget):
         prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
         prefs['mark'] = self.basic_tab.mark.isChecked()
         prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
+        prefs['autoconvert'] = self.basic_tab.autoconvert.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
@@ -248,6 +249,7 @@ class ConfigWidget(QWidget):
                 #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
         prefs['gc_site_settings'] = gc_site_settings
         prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
+        prefs['gc_polish_cover'] = self.generatecover_tab.gc_polish_cover.isChecked()
 
         # Count Pages tab
         countpagesstats = []
@@ -422,6 +424,11 @@ class BasicTab(QWidget):
         self.showmarked.setChecked(prefs['showmarked'])
         self.l.addWidget(self.showmarked)
 
+        self.autoconvert = QCheckBox(_("Automatically Convert new/update books?"),self)
+        self.autoconvert.setToolTip(_("Automatically call calibre's Convert for new/update books.\nConverts to the current output format as chosen in calibre's\nPreferences->Behavior settings."))
+        self.autoconvert.setChecked(prefs['autoconvert'])
+        self.l.addWidget(self.autoconvert)
+
         gui_gb = groupbox = QGroupBox(_("GUI Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
@@ -747,6 +754,11 @@ class GenerateCoverTab(QWidget):
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
             
+        self.gc_polish_cover = QCheckBox(_("Use calibre's Polish feature to inject/update the cover"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated cover into the ebook, EPUB only."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        self.l.addWidget(self.gc_polish_cover)
+            
 class CountPagesTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):

calibre-plugin/ffdl_plugin.py

@@ -32,6 +32,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.utils.config import prefs as calibre_prefs
 from calibre.utils.date import local_tz
 from calibre.library.comments import sanitize_comments_html
 from calibre.constants import config_dir as calibre_config_dir
@@ -660,13 +661,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         #print("update_dialog()")
         
         db = self.gui.current_db
-        book_list = map( self.make_book_id_only, id_list )
+        books = map( self.make_book_id_only, id_list )
 
-        for j, book in enumerate(book_list):
+        for j, book in enumerate(books):
             book['listorder'] = j
-            
+
         LoopProgressDialog(self.gui,
-                           book_list,
+                           books,
                            partial(self.populate_book_from_calibre_id, db=self.gui.current_db),
                            self.update_dialog_finish,
                            init_label=_("Collecting stories for update..."),
@@ -718,6 +719,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             url_list = split_text_to_urls(books)
             books = self.convert_urls_to_books(url_list)
 
+        ## for tweak_fg_sleep
+        options['ffnetcount']=len(filter(lambda x : x['site']=='www.fanfiction.net', books))
+
         options['version'] = self.version
         logger.debug(self.version)
         
@@ -815,6 +819,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
         options['personal.ini'] = get_ffdl_personalini()
         adapter = get_ffdl_adapter(url,fileform)
+        # reduce foreground sleep time for ffnet when few books.
+        if 'ffnetcount' in options and \
+                adapter.getConfig('tweak_fg_sleep') and \
+                adapter.getSiteDomain() == 'www.fanfiction.net':
+            minslp = float(adapter.getConfig('min_fg_sleep'))
+            maxslp = float(adapter.getConfig('max_fg_sleep'))
+            dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+            m = (maxslp-minslp) / (dwnlds-1)
+            b = minslp - m
+            slp = min(maxslp,m*float(options['ffnetcount'])+b)
+            #print("m:%s b:%s = %s"%(m,b,slp))
+            adapter.set_sleep(slp)
 
         ## three tries, that's enough if both user/pass & is_adult needed,
         ## or a couple tries of one or the other
@@ -841,7 +857,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         series = story.getMetadata('series')
         if not merge and series and prefs['checkforseriesurlid']:
             # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
+            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
             identicalbooks = db.search_getting_ids(searchstr, None)
             # print("searchstr:%s"%searchstr)
             # print("identicalbooks:%s"%identicalbooks)
@@ -1138,6 +1154,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 func, args=args,
                 description=desc)
         
+        self.gui.jobs_pointer.start()
         self.gui.status_bar.show_message(_('Starting %d FanFictionDownLoads')%len(book_list),3000)
 
     def update_books_loop(self,book,db=None,
@@ -1232,6 +1249,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             cp_plugin = self.gui.iactions['Count Pages']
             cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
 
+        if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
+            self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
+            self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
+
     def download_list_completed(self, job, options={},merge=False):
         if job.failed:
             self.gui.job_exception(job, dialog_title='Failed to Download Stories')
@@ -1335,7 +1356,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         bad_list = sorted(bad_list,key=lambda x : x['listorder'])
         
         self.gui.status_bar.show_message(_('Merging %s books.')%total_good)
-
         
         existingbook = None
         if 'mergebook' in options:
@@ -1393,7 +1413,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if prefs['mark'] or (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
             self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
             self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.')%len(book_list))
-            label = custom_columns[prefs['errorcol']]['label']
+            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
+                label = custom_columns[prefs['errorcol']]['label']
+            else:
+                label = None
             LoopProgressDialog(self.gui,
                                book_list,
                                partial(self.update_error_column_loop, db=self.gui.current_db, label=label),
@@ -1402,12 +1425,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                win_title=_("Update calibre for BAD FanFiction stories"),
                                status_prefix=_("Updated"))
 
-    def update_error_column_loop(self,book,db=None,label='errorcol'):
-        if book['calibre_id']:
-            custom_columns = self.gui.library_view.model().custom_columns
-            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
-                logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
-                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+    def update_error_column_loop(self,book,db=None,label=None):
+        if book['calibre_id'] and label:
+            logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
 
     def add_book_or_update_format(self,book,options,prefs,mi=None):
         db = self.gui.current_db
@@ -1436,8 +1457,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
             for fmt in fmts:
                 if fmt != formmapping[options['fileform']]:
-                    logger.debug("remove f:"+fmt)
+                    logger.debug("deleteotherforms remove f:"+fmt)
                     db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+        elif prefs['autoconvert']:
+            ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+            ## the format is already there.
+            fmt = calibre_prefs['output_format']
+            # delete if there, but not if the format we just made.
+            if fmt != formmapping[options['fileform']] and \
+                    db.has_format(book_id,fmt,index_is_id=True):
+                logger.debug("autoconvert remove f:"+fmt)
+                db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                
 
         return book_id
 
@@ -1545,8 +1576,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )
 
                     if meta not in book['all_metadata']:
-                        logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
-                        continue
+                        # if double quoted, use as a literal value.
+                        if meta[0] == '"' and meta[-1] == '"':
+                            val = meta[1:-1]
+                            logger.debug("No metadata value for %s, setting custom column(%s) literally to %s."%(meta,custcol,val))
+                        else:
+                            logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
+                            continue
+                    else:
+                        val = book['all_metadata'][meta]    
                     
                     if custcol not in custom_columns:
                         continue
@@ -1558,12 +1596,19 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
                             if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
                                 # re-split list, strip commas, convert to floats, sum up.
-                                val = sum([ float(x.replace(",","")) for x in book['all_metadata'][meta].split(", ") ])
+                                val = sum([ float(x.replace(",","")) for x in val.split(", ") ])
                             else:
-                                val = unicode(book['all_metadata'][meta]).replace(",","")
+                                val = unicode(val).replace(",","")
                         else:
-                            val = book['all_metadata'][meta]
+                            val = val
                         if val != '':
+                            if coldef['datatype'] == 'bool':
+                                if val.lower() in ('t','true','1','yes','y'):
+                                    val = True
+                                elif val.lower() in ('f','false','0','no','n'):
+                                    val = False
+                                else:
+                                    val = None # for tri-state 'booleans'. Yes/No/Null
                             db.set_custom(book_id, val, label=label, commit=False)
 
                     if flag == 'a':
@@ -1577,8 +1622,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         except:
                             pass
 
-                        if book['all_metadata'][meta]:
-                            vallist = [book['all_metadata'][meta]]
+                        if val:
+                            vallist = [val]
                             
                         db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)
 
@@ -1610,7 +1655,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
 
-            logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
+            #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
             
             # force a refresh if generating cover so complex composite
             # custom columns are current and correct
@@ -1653,6 +1698,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 realmi = db.get_metadata(book_id, index_is_id=True)
                 gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
 
+                if prefs['gc_polish_cover'] and \
+                        options['fileform'] == "epub" and calibre_version >= (0, 9, 39):
+                    # set cover inside epub from calibre's polish feature
+                    from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+                    from calibre.utils.logging import Log
+                    from collections import namedtuple
+
+                    # Couldn't find a better way to get the cover path.
+                    cover_path = os.path.join(db.library_path, db.path(book_id, index_is_id=True), 'cover.jpg')
+                    data = {'cover':cover_path}
+                    #print("cover_path:%s"%cover_path)
+                    opts = ALL_OPTS.copy()
+                    opts.update(data)
+                    O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+                    opts = O(**opts)
+                
+                    log = Log(level=Log.DEBUG)
+                    outfile = db.format_abspath(book_id, formmapping[options['fileform']], index_is_id=True)
+                    #print("polish cover outfile:%s"%outfile)
+                    polish({outfile:outfile}, opts, log, logger.info)
+                
+
     def get_clean_reading_lists(self,lists):
         if lists == None or lists.strip() == "" :
             return []
@@ -1741,6 +1808,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['end'] = None
         book['comment'] = '' # note this is a comment on the d/l or update.
         book['url'] = ''
+        book['site'] = ''
         book['added'] = False
         return book
     
@@ -1817,13 +1885,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             book['status'] = _('Not Found')
         else:
             # get normalized url or None.
-            book['url'] = self.is_good_downloader_url(url)
-            if book['url'] == None:
+            urlsitetuple = adapters.getNormalStoryURLSite(url)
+            if urlsitetuple == None:
                 book['url'] = url
                 book['comment'] = _("URL is not a valid story URL.")
                 book['good'] = False
                 book['icon']='dialog_error.png'
                 book['status'] = _('Bad URL')
+            else:
+                (book['url'],book['site'])=urlsitetuple
     
     def get_story_url(self, db, book_id=None, path=None):
         if book_id == None:

calibre-plugin/jobs.py

@@ -93,16 +93,13 @@ def do_download_worker(book_list, options,
         logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
         logger.info(job.details)
 
+
+
         if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            logger.info("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
-            logger.info("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\nSuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                      filter(lambda x: x['good'], book_list) ] ) ) )
+            logger.info("\nUnsuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
             break
 
     server.close()

calibre-plugin/prefs.py

@@ -34,6 +34,7 @@ default_prefs['suppressauthorsort'] = False
 default_prefs['suppresstitlesort'] = False
 default_prefs['mark'] = False
 default_prefs['showmarked'] = False
+default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'
@@ -56,6 +57,7 @@ default_prefs['addtolistsonread'] = False
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
+default_prefs['gc_polish_cover'] = False
 
 default_prefs['countpagesstats'] = []
 

defaults.ini

@@ -182,6 +182,11 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
 
+## How long to wait for each HTTP connection to finish.  Longer times
+## are better for sites that are slow to respond.  Shorter times
+## prevent excessive wait when your network or the site is down.
+connect_timeout:60.0
+
 ## For use only with stand-alone CLI version--run a command on the
 ## generated file after it's produced.  All of the titlepage_entries
 ## values are available, plus output_filename.
@@ -817,12 +822,24 @@ dislikes_label:Dislikes
 ## Clear FanFiction from defaults, site is original fiction.
 extratags:
 
-extra_valid_entries:size,universe,codes
-#extra_titlepage_entries:size,universe,codes
+extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
+#extra_titlepage_entries:size,universeHTML,codes,notice
 
 size_label:Size
 universe_label:Universe
+universeUrl_label:Universe URL
+universeHTML_label:Universe
 codes_label:Codes
+notice_label:Notice
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:universe
+
+## storiesonline.net stories can be in a series or a universe, but not
+## both.  By default, universe will be populated in 'series' with
+## index=0
+universe_as_series: true
 
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
@@ -1203,6 +1220,11 @@ user_agent:
 ## Change this to false to use them anyway.
 never_make_cover: true
 
+## fanfiction.net is blocking people more aggressively.  If you
+## download fewer stories less often you can likely get by with
+## reducing this sleep.
+slow_down_sleep_time:4
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
@@ -1211,13 +1233,20 @@ extra_valid_entries:reviews,favs,follows
 ## to be romantic pairings.
 ships_label:Pairings
 
-## Date formats used by FFDL.  Published and Update don't have time.
+## Date formats used by FFDL.  Published and Update don't usually have
+## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
 #dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
 datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
 dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 
+## ffnet used to have a tendency to send out update notices in email
+## before all their servers were showing the update on the first
+## chapter.  It generates another server request and doesn't seem to
+## be needed lately, so now default it to off.
+check_next_chapter:false
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In

fanficdownloader/adapters/__init__.py

@@ -146,6 +146,13 @@ for x in imports():
             __domain_map[site]=cls
 
 def getNormalStoryURL(url):
+    r = getNormalStoryURLSite(url)
+    if r:
+        return r[0]
+    else:
+        return None
+
+def getNormalStoryURLSite(url):
     if not getNormalStoryURL.__dummyconfig:
         getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
     # pulling up an adapter is pretty low over-head.  If
@@ -153,10 +160,11 @@ def getNormalStoryURL(url):
     try:
         adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
         url = adapter.url
+        site = adapter.getSiteDomain()
         del adapter
-        return url
+        return (url,site)
     except:
-        return None;
+        return None
 
 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None
@@ -193,7 +201,7 @@ def getConfigSectionFor(url):
 
 def getClassFor(url):
     ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
     if not fixedurl.startswith("http"):
         fixedurl = "http://%s"%url
     ## remove any trailing '#' locations.

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -84,7 +84,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
         # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
+        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -87,7 +87,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
     
     def setHeader(self):
         "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
-        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
         
     @classmethod
     def getSiteExampleURLs(self):

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -57,6 +57,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             ## accept m(mobile)url, but use www.
             self.origurl = self.origurl.replace("https://m.","https://www.")
 
+        self.opener.addheaders.append(('Referer',self.origurl))
+
     @staticmethod
     def getSiteDomain():
         return 'www.fanfiction.net'
@@ -105,28 +107,31 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if "not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
 
-        try:
-            # rather nasty way to check for a newer chapter.  ffnet has a
-            # tendency to send out update notices in email before all
-            # their servers are showing the update on the first chapter.
+        if self.getConfig('check_next_chapter'):
             try:
-                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
-            # get chapter part of url.
+                ## ffnet used to have a tendency to send out update
+                ## notices in email before all their servers were
+                ## showing the update on the first chapter.  It
+                ## generates another server request and doesn't seem
+                ## to be needed lately, so now default it to off.
+                try:
+                    chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+                # get chapter part of url.
+                except:
+                    chapcount = 1
+                chapter = url.split('/',)[5]
+                tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                                self.story.getMetadata('storyId'),
+                                                chapcount+1)
+                logger.debug('=Trying newer chapter: %s' % tryurl)
+                newdata = self._fetchUrl(tryurl)
+                if "not found. Please check to see you are not using an outdated url." \
+                        not in newdata:
+                    logger.debug('=======Found newer chapter: %s' % tryurl)
+                    soup = bs.BeautifulSoup(newdata)
             except:
-                chapcount = 1
-            chapter = url.split('/',)[5]
-            tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            chapcount+1)
-            logger.debug('=Trying newer chapter: %s' % tryurl)
-            newdata = self._fetchUrl(tryurl)
-            if "not found. Please check to see you are not using an outdated url." \
-                    not in newdata:
-                logger.debug('=======Found newer chapter: %s' % tryurl)
-                soup = bs.BeautifulSoup(newdata)
-        except:
-            pass
-        
+                pass
+            
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -101,7 +101,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         # if "/images/missing_story.png" in data:
         #     raise exceptions.StoryDoesNotExist(self.url)
         
-        if "This story has been marked as having adult content." in data:
+        if "This story has been marked as having adult content. Please click below to confirm you are of legal age to view adult material in your country." in data:
             raise exceptions.AdultCheckRequired(self.url)
         
         if self.password:
@@ -249,18 +249,20 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
 
-        #Sequel links and group links are each bundled into story_group_list containers.
-        #Rather than mess around examining the header text, which is outside the containers,
-        #one can tell the two link types apart by examining them directly.
+        ## Groups and sequels code from FaceDeer
         allGroupLists = soup.findAll('ul', {'id':'story_group_list'})
         for groupList in allGroupLists:
             for groupName in groupList.findAll('a', {'href':re.compile('^/group/')}):
                 self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) 
                 self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
-            for sequel in groupList.findAll('a', {'class':'story_link'}):
+
+        sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels")
+        if not sequelStoryHeader == None:
+            sequelContainer = sequelStoryHeader.parent.parent
+            for sequel in sequelContainer.findAll('a', {'class':'story_link'}):
                 self.story.addToList("sequelsUrl", 'http://'+self.host+sequel["href"]) 
                 self.story.addToList("sequels", stripHTML(sequel).replace(',', ';'))
-
+                
         #The link to the prequel is embedded in the description text, so erring
         #on the side of caution and wrapping this whole thing in a try block.
         #If anything goes wrong this probably wasn't a valid prequel link.

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,11 +37,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
@@ -51,7 +46,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
         
-        
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
         

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -128,23 +128,22 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # <span class="label">Rated:</span> NC-17<br /> etc
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
-            value = labelspan.nextSibling
+            val = labelspan.nextSibling
+            value = unicode('')
+            while val and not defaultGetattr(val,'class') == 'label':
+                value += unicode(val)
+                val = val.nextSibling
             label = labelspan.string
+            #print("label:%s\nvalue:%s"%(label,value))
 
             if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,value)
 
             if 'Rated' in label:
-                self.story.setMetadata('rating', value)
+                self.story.setMetadata('rating', stripHTML(value))
 
             if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
+                self.story.setMetadata('numWords', stripHTML(value))
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
@@ -167,7 +166,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                     self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
-                if 'Yes' in value:
+                if 'Yes' in stripHTML(value):
                     self.story.setMetadata('status', 'Completed')
                 else:
                     self.story.setMetadata('status', 'In-Progress')

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -163,12 +163,11 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
+                # limit date values, there's some extra chars.
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y"))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y"))
 
         try:
             # Find Series name from series URL.

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -37,11 +37,6 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
@@ -147,6 +142,10 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         a = soup.find('h1')
         self.story.setMetadata('title',stripHTML(a))
         
+        notice = soup.find('div', {'class' : 'notice'})
+        if notice:
+            self.story.setMetadata('notice',unicode(notice))
+
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"/a/\w+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -185,23 +184,74 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('size', lc2.findNext('td', {'class' : 'num'}).text)
         
         lc4 = lc2.findNext('td', {'class' : 'lc4'})        
-        
+        desc = lc4.contents[0]
+
         try:
             a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
-            i = a.parent.text.split('(')[1].split(')')[0]
-            self.setSeries(stripHTML(a), i)
-            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
+            if a:
+                # if there's a number after the series name, series_contents is a two element list:
+                # [<a href="...">Title</a>, u' (2)']
+                series_contents = a.parent.contents
+                i = 0 if len(series_contents) == 1 else series_contents[1].strip(' ()')
+                seriesUrl = 'http://'+self.host+a['href']
+                self.story.setMetadata('seriesUrl',seriesUrl)
+                series_name = stripHTML(a)
+                logger.debug("Series name= %s" % series_name)
+                series_soup = bs.BeautifulSoup(self._fetchUrl(seriesUrl))
+                if series_soup:
+                    logger.debug("Retrieving Series - looking for name")
+                    series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0]
+                    logger.debug("Series name: '{0}'".format(series_name))
+                self.setSeries(series_name, i)
+                desc = lc4.contents[2]
+                # Check if series is in a universe
+                universes_soup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')  + "&type=uni")) 
+#                logger.debug("Universe page=", universes_soup)
+                if universes_soup:
+                    universes = universes_soup.findAll('div', {'class' : 'ser-box'})
+                    logger.debug("Number of Universes: %d" % len(universes))
+                    for universe in universes:
+                        logger.debug("universe.find('a')={0}".format(universe.find('a')))
+                        # The universe id is in an "a" tag that has an id but nothing else. It is the first tag.
+                        # The id is prefixed with the letter "u".
+                        universe_id = universe.find('a')['id'][1:]
+                        logger.debug("universe_id='%s'" % universe_id)
+                        universe_name = universe.find('div', {'class' : 'ser-name'}).text.partition(' ')[2]
+                        logger.debug("universe_name='%s'" % universe_name)
+                        # If there is link to the story, we have the right universe
+                        story_a = universe.find('a', {'href' : '/s/'+self.story.getMetadata('storyId')})
+                        if story_a:
+                            logger.debug("Story is in a series that is in a universe! The universe is '%s'" % universe_name)
+                            self.story.setMetadata("universe", universe_name)
+                            self.story.setMetadata('universeUrl','http://'+self.host+ '/library/universe.php?id=' + universe_id)
+                            break
         except:
             pass
         try:
             a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
             if a:
                 self.story.setMetadata("universe",stripHTML(a))
+                desc = lc4.contents[2]
+                # Assumed only one universe, but it does have a URL--use universeHTML
+                universe_name = stripHTML(a)
+                universeUrl = 'http://'+self.host+a['href']
+                logger.debug("Retrieving Universe - about to get page")
+                universe_soup = bs.BeautifulSoup(self._fetchUrl(universeUrl))
+                logger.debug("Retrieving Universe - have page")
+                if universe_soup:
+                    logger.debug("Retrieving Universe - looking for name")
+                    universe_name = universe_soup.find('span', {'id' : 'ptitle'}).text.partition(' &mdash;')[0]
+                    logger.debug("Universes name: '{0}'".format(universe_name))
+
+                self.story.setMetadata('universeUrl',universeUrl)
+                logger.debug("Setting universe name: '{0}'".format(universe_name))
+                self.story.setMetadata('universe',universe_name)
+                if self.getConfig("universe_as_series"):
+                    self.setSeries(universe_name, 0)
+                    self.story.setMetadata('seriesUrl',universeUrl)
         except:
             pass
-            
 
-        desc = lc4.contents[0]
         self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
             
         for b in lc4.findAll('b'):
@@ -255,8 +305,18 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         # some big chapters are split over several pages
         pager = div.find('span', {'class' : 'pager'})
         if pager != None:
+            a = pager.previousSibling
+            while a != None:
+                logger.debug("before pager: {0}".format(a))
+                b = a.previousSibling
+                a.extract()
+                a = b
+
             urls=pager.findAll('a')
             urls=urls[:len(urls)-1]
+            pager.extract()
+            div.contents = div.contents[2:]
+#            logger.debug(div)
                         
             for ur in urls:
                 soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
@@ -264,38 +324,76 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         
                 div1 = soup.find('div', {'id' : 'story'})
                 
-                # appending next section
-                last=div.findAll('p')
-                next=div1.find('span', {'class' : 'conTag'}).nextSibling
-            
-                last[len(last)-1]=last[len(last)-1].append(next)
-                div.append(div1)
-            
-        # removing all the left-over stuff    
-        for a in div.findAll('span'):
-            a.extract() 
-            
-        for a in div.findAll('h1'):
-            a.extract()
-        for a in div.findAll('h2'):
-            a.extract()
-        for a in div.findAll('h3'):
-            a.extract()
-        for a in div.findAll('h4'):
-            a.extract()
-        for a in div.findAll('br'):
-            a.extract()
-        for a in div.findAll('div', {'class' : 'date'}):
-            a.extract()
-            
-        a = div.find('form')
-        if a != None:
-            b = a.nextSibling
-            while b != None:
+                # Find the "Continues" marker on the current page and remove everything after that. 
+                continues = div.find('span', {'class' : 'conTag'})
+                if continues != None:
+                    while continues != None:
+#                        logger.debug("removing end: {0}".format(continues))
+                        b = continues.nextSibling
+                        continues.extract()
+                        continues = b
+
+                # Find the "Continued" marker and delete everything before that
+                continued = div1.find('span', {'class' : 'conTag'})
+                if continued != None:
+                    a = continued.previousSibling
+                    while a != None:
+#                        logger.debug("before conTag: {0}".format(a))
+                        b = a.previousSibling
+                        a.extract()
+                        a = b
+                # Remove the pager from the end if this is the last page
+                endPager = div1.find('span', {'class' : 'pager'})
+                if endPager != None:
+                    b = endPager.nextSibling
+                    while endPager != None:
+                        logger.debug("removing end: {0}".format(endPager))
+                        b = endPager.nextSibling
+                        endPager.extract()
+                        endPager = b
+                    div1.contents = div1.contents[:len(div1) - 2]
+#                logger.debug("after removing pager: {0}".format(div1))
+                for tag in div1.contents[2:]:
+                    div.append(tag)
+
+        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus 
+        # everything before them can be discarded. 
+        postedDates = div.findAll('div', {'class' : 'date'})
+        if postedDates:
+            a = postedDates[0].previousSibling
+            while a != None:
+#                logger.debug("before dates: {0}".format(a))
+                b = a.previousSibling
                 a.extract()
-                a=b
-                b=b.nextSibling
-        
+                a = b
+            for a in div.findAll('div', {'class' : 'date'}):
+                a.extract()
+
+        # For single chapter stories, there is a copyright statement. Remove this and everything
+        # before it.
+        copy = div.find('h4', {'class': 'copy'})
+        while copy != None:
+#            logger.debug("before copyright: {0}".format(copy))
+            b = copy.previousSibling
+            copy.extract()
+            copy = b
+
+        # For a story or the last chapter, remove voting form and the in library box
+        a = div.find('div', {'id' : 'vote-form'})
+        if a != None:
+            a.extract()
+        a = div.find('div', {'id' : 'b-man-div'})
+        if a != None:
+            a.extract()
+
+        # Kill the "The End" header and everything after it.
+        a = div.find(['h2', 'h3'], {'class' : 'end'})
+        logger.debug("Chapter end= '{0}'".format(a))
+        while a != None:
+            b = a.nextSibling
+            a.extract()
+            a=b
+
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_test1.py

@@ -50,6 +50,7 @@ class TestSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
         idstr = self.story.getMetadata('storyId')
         idnum = int(idstr)
+        self.do_sleep()
 
         if idnum >= 1000:
             logger.warn("storyId:%s - Custom INI data will be used."%idstr)
@@ -296,6 +297,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
+        self.do_sleep()
         if self.story.getMetadata('storyId').startswith('670') or \
                 self.story.getMetadata('storyId').startswith('672'):
             time.sleep(1.0)

fanficdownloader/adapters/base_adapter.py

@@ -76,6 +76,8 @@ class BaseSiteAdapter(Configurable):
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
+
+        self.override_sleep = None
         
         self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         ## Specific UA because too many sites are blocking the default python UA.
@@ -147,8 +149,7 @@ class BaseSiteAdapter(Configurable):
 
     # Assumes application/x-www-form-urlencoded.  parameters, headers are dict()s
     def _postUrl(self, url, parameters={}, headers={}):
-        if self.getConfig('slow_down_sleep_time'):
-            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        self.do_sleep()
 
         ## u2.Request assumes POST when data!=None.  Also assumes data
         ## is application/x-www-form-urlencoded.
@@ -159,24 +160,38 @@ class BaseSiteAdapter(Configurable):
         req = u2.Request(url,
                          data=urllib.urlencode(parameters),
                          headers=headers)
-        return self._decode(self.opener.open(req).read())
+        return self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())
 
     def _fetchUrlRaw(self, url, parameters=None):
         if parameters != None:
-            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters)).read()
+            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
         else:
-            return self.opener.open(url.replace(' ','%20')).read()
+            return self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
+
+    def set_sleep(self,val):
+        print("\n===========\n set sleep time %s\n==========="%val)
+        self.override_sleep = val
     
+    def do_sleep(self):
+        if self.override_sleep:
+            time.sleep(float(self.override_sleep))
+        elif self.getConfig('slow_down_sleep_time'):
+            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        
     # parameters is a dict()
     def _fetchUrl(self, url, parameters=None):
-        if self.getConfig('slow_down_sleep_time'):
-            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        self.do_sleep()
 
         excpt=None
         for sleeptime in [0, 0.5, 4, 9]:
             time.sleep(sleeptime)	
             try:
                 return self._decode(self._fetchUrlRaw(url,parameters))
+            except u2.HTTPError, he:
+                if he.code == 404:
+                    logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(he)))
+                    excpt=he
+                    break # break out on 404
             except Exception, e:
                 excpt=e
                 logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
@@ -367,7 +382,7 @@ class BaseSiteAdapter(Configurable):
             
         # Don't want body tags in chapter html--writers add them.
         # This is primarily for epub updates.
-        retval = re.sub(r"</?body>\r?\n?","",retval)
+        retval = re.sub(r"</?body[^>]*>\r?\n?","",retval)
         
         if self.getConfig("replace_br_with_p"):
             # Apply heuristic processing to replace <br> paragraph

fanficdownloader/htmlheuristics.py

@@ -137,6 +137,9 @@ def replace_br_with_p(body):
             contentLinesSum += lineLen
             if lineLen > longestLineLength:
                 longestLineLength = lineLen
+                
+    if contentLines == 0:
+        contentLines = 1
 
     averageLineLength = contentLinesSum/contentLines
 

fanficdownloader/story.py

@@ -573,7 +573,13 @@ class Story(Configurable):
             imgurl = url
         else:
             parsedUrl = urlparse.urlparse(parenturl)
-            if url.startswith("/") :
+            if url.startswith("//") :
+                imgurl = urlparse.urlunparse(
+                    (parsedUrl.scheme,
+                     '',
+                     url,
+                     '','',''))
+            elif url.startswith("/") :
                 imgurl = urlparse.urlunparse(
                     (parsedUrl.scheme,
                      parsedUrl.netloc,

index.html

@@ -48,13 +48,9 @@
 
           <h3>fanfiction.net / fimfiction.net</h3>
           <p>
-            Fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-          <p>It appears that FimFiction.net is also blocking access from Google
-            App Engine now.
+            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
+            are working again.  I'd ask that users limit the number of
+            stories they download from those sites, thanks.
           </p>
 	  
       {% if authorized %}
@@ -68,11 +64,10 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>Alternate URL efiction.trekiverse.org for trekiverse.org/efiction/</li>
-              <li>Minor fixes</li>
+              <li>Fix potionsandsnitches.net dates.</li>
+              <li>Fix for AO3 https urls.</li>
 	    </ul>
 	  </p>
-          
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -81,7 +76,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-86.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-94.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -154,6 +154,11 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
 
+## How long to wait for each HTTP connection to finish.  Longer times
+## are better for sites that are slow to respond.  Shorter times
+## prevent excessive wait when your network or the site is down.
+connect_timeout:60.0
+
 ## Use regular expressions to find and replace (or remove) metadata.
 ## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
 ## etc.  See http://docs.python.org/library/re.html (look for re.sub)
@@ -691,13 +696,15 @@ cliches_label:Character Cliches
 ## 'mode'.  'r' to Replace any existing values, 'a' to Add to existing
 ## value (use with tag-like columns), and 'n' for setting on New books
 ## only.  (Default is 'r'.)
+## Literal strings can be set into custom columns using double quotes.
 ## Each metadata=>column mapping must be on a separate line and each
 ## needs to have one space at the start of each line.
 
 #custom_columns_settings:
-# cliches=>#acolumn,r
+# cliches=>#acolumn
 # themes=>#bcolumn,a
 # timeline=>#ccolumn,n
+# "FanFiction"=>#collection
 
 [efiction.esteliel.de]
 ## Site dedicated to these categories/characters/ships
@@ -802,12 +809,24 @@ dislikes_label:Dislikes
 ## Clear FanFiction from defaults, site is original fiction.
 extratags:
 
-extra_valid_entries:size,universe,codes
-#extra_titlepage_entries:size,universe,codes
+extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
+#extra_titlepage_entries:size,universeHTML,codes,notice
 
 size_label:Size
 universe_label:Universe
+universeUrl_label:Universe URL
+universeHTML_label:Universe
 codes_label:Codes
+notice_label:Notice
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:universe
+
+## storiesonline.net stories can be in a series or a universe, but not
+## both.  By default, universe will be populated in 'series' with
+## index=0
+universe_as_series: true
 
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
@@ -1188,6 +1207,20 @@ user_agent:
 ## Change this to false to use them anyway.
 never_make_cover: true
 
+## fanfiction.net is blocking people more aggressively.  If you
+## download fewer stories less often you can likely get by with
+## reducing this sleep.
+slow_down_sleep_time:4
+
+## ffnet is sensitive to too many hits.  Users are sensitive to long
+## waits during the initial metadata collection in the foreground.
+## When used, these settings will speed up metadata downloads in the
+## foreground linearly.
+tweak_fg_sleep:true
+min_fg_sleep:1.0
+max_fg_sleep:4.0
+max_fg_sleep_at_downloads: 10
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
@@ -1196,13 +1229,20 @@ extra_valid_entries:reviews,favs,follows
 ## to be romantic pairings.
 ships_label:Pairings
 
-## Date formats used by FFDL.  Published and Update don't have time.
+## Date formats used by FFDL.  Published and Update don't usually have
+## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
 #dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
 datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
 dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 
+## ffnet used to have a tendency to send out update notices in email
+## before all their servers were showing the update on the first
+## chapter.  It generates another server request and doesn't seem to
+## be needed lately, so now default it to off.
+check_next_chapter:false
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In

status.html

@@ -53,13 +53,9 @@
 	  {% if fic.failure %}
           <h3>fanfiction.net / fimfiction.net</h3>
           <p>
-            FYI, fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-          <p>It appears that FimFiction.net is also blocking access from Google
-            App Engine now.
+            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
+            are working again.  I'd ask that users limit the number of
+            stories they download from those sites, thanks.
           </p>
 	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}