Update CLI download zip.

Also don't retry 404 errors and bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-91
+version: 4-5-05
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 8, 10)
+    version             = (1, 8, 24)
     minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -55,7 +55,7 @@ from calibre_plugins.fanfictiondownloader_plugin.dialogs \
             EditTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
-    import (getConfigSections, getNormalStoryURL)
+    import getConfigSections
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
@@ -249,6 +249,7 @@ class ConfigWidget(QWidget):
                 #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
         prefs['gc_site_settings'] = gc_site_settings
         prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
+        prefs['gc_polish_cover'] = self.generatecover_tab.gc_polish_cover.isChecked()
 
         # Count Pages tab
         countpagesstats = []
@@ -753,6 +754,11 @@ class GenerateCoverTab(QWidget):
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
             
+        self.gc_polish_cover = QCheckBox(_("Use calibre's Polish feature to inject/update the cover"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated cover into the ebook, EPUB only."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        self.l.addWidget(self.gc_polish_cover)
+            
 class CountPagesTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):

calibre-plugin/ffdl_plugin.py

@@ -32,6 +32,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.utils.config import prefs as calibre_prefs
 from calibre.utils.date import local_tz
 from calibre.library.comments import sanitize_comments_html
 from calibre.constants import config_dir as calibre_config_dir
@@ -660,13 +661,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         #print("update_dialog()")
         
         db = self.gui.current_db
-        book_list = map( self.make_book_id_only, id_list )
+        books = map( self.make_book_id_only, id_list )
 
-        for j, book in enumerate(book_list):
+        for j, book in enumerate(books):
             book['listorder'] = j
-            
+
         LoopProgressDialog(self.gui,
-                           book_list,
+                           books,
                            partial(self.populate_book_from_calibre_id, db=self.gui.current_db),
                            self.update_dialog_finish,
                            init_label=_("Collecting stories for update..."),
@@ -718,6 +719,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             url_list = split_text_to_urls(books)
             books = self.convert_urls_to_books(url_list)
 
+        ## for tweak_fg_sleep
+        options['ffnetcount']=len(filter(lambda x : x['site']=='www.fanfiction.net', books))
+
         options['version'] = self.version
         logger.debug(self.version)
         
@@ -815,6 +819,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
         options['personal.ini'] = get_ffdl_personalini()
         adapter = get_ffdl_adapter(url,fileform)
+        # reduce foreground sleep time for ffnet when few books.
+        if 'ffnetcount' in options and \
+                adapter.getConfig('tweak_fg_sleep') and \
+                adapter.getSiteDomain() == 'www.fanfiction.net':
+            minslp = float(adapter.getConfig('min_fg_sleep'))
+            maxslp = float(adapter.getConfig('max_fg_sleep'))
+            dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+            m = (maxslp-minslp) / (dwnlds-1)
+            b = minslp - m
+            slp = min(maxslp,m*float(options['ffnetcount'])+b)
+            #print("m:%s b:%s = %s"%(m,b,slp))
+            adapter.set_sleep(slp)
 
         ## three tries, that's enough if both user/pass & is_adult needed,
         ## or a couple tries of one or the other
@@ -841,7 +857,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         series = story.getMetadata('series')
         if not merge and series and prefs['checkforseriesurlid']:
             # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
+            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
             identicalbooks = db.search_getting_ids(searchstr, None)
             # print("searchstr:%s"%searchstr)
             # print("identicalbooks:%s"%identicalbooks)
@@ -925,13 +941,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 if len(identicalbooks) < 1:
                     # find dups
                     authlist = story.getList("author", removeallentities=True)
-                    if len(authlist) > 100 and calibre_version < (0, 8, 61):
-                        ## should be fixed from 0.8.61 on.  In the
-                        ## meantime, if it matches the title *and* first
-                        ## 100 authors, I'm prepared to assume it's a
-                        ## match.
-                        logger.debug("reduce author list to 100 only when calibre < 0.8.61")
-                        authlist = authlist[:100]
                     mi = MetaInformation(story.getMetadata("title", removeallentities=True),
                                          authlist)
                     identicalbooks = db.find_identical_books(mi)
@@ -961,7 +970,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 if book_id and mi: # book_id and mi only set if matched by title/author.
                     liburl = self.get_story_url(db,book_id)
                     if book['url'] != liburl and prefs['checkforurlchange'] and \
-                        not (book['url'].replace('https','http') == liburl and 'fanfiction.net' in liburl):
+                        not (book['url'].replace('https','http') == liburl): # several sites have been changing to
+                                                                             # https now.  Don't flag when that's the only change.
                         # special case for ffnet urls change to https.
                         if not question_dialog(self.gui, _('Change Story URL?'),'''
                                                   <h3>%s</h3>
@@ -1138,6 +1148,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 func, args=args,
                 description=desc)
         
+        self.gui.jobs_pointer.start()
         self.gui.status_bar.show_message(_('Starting %d FanFictionDownLoads')%len(book_list),3000)
 
     def update_books_loop(self,book,db=None,
@@ -1232,7 +1243,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             cp_plugin = self.gui.iactions['Count Pages']
             cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
 
-        if prefs['autoconvert']:
+        if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
             self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
             self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
 
@@ -1339,7 +1350,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         bad_list = sorted(bad_list,key=lambda x : x['listorder'])
         
         self.gui.status_bar.show_message(_('Merging %s books.')%total_good)
-
         
         existingbook = None
         if 'mergebook' in options:
@@ -1441,8 +1451,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
             for fmt in fmts:
                 if fmt != formmapping[options['fileform']]:
-                    logger.debug("remove f:"+fmt)
+                    logger.debug("deleteotherforms remove f:"+fmt)
                     db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+        elif prefs['autoconvert']:
+            ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+            ## the format is already there.
+            fmt = calibre_prefs['output_format']
+            # delete if there, but not if the format we just made.
+            if fmt != formmapping[options['fileform']] and \
+                    db.has_format(book_id,fmt,index_is_id=True):
+                logger.debug("autoconvert remove f:"+fmt)
+                db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                
 
         return book_id
 
@@ -1576,6 +1596,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         else:
                             val = val
                         if val != '':
+                            if coldef['datatype'] == 'bool':
+                                if val.lower() in ('t','true','1','yes','y'):
+                                    val = True
+                                elif val.lower() in ('f','false','0','no','n'):
+                                    val = False
+                                else:
+                                    val = None # for tri-state 'booleans'. Yes/No/Null
                             db.set_custom(book_id, val, label=label, commit=False)
 
                     if flag == 'a':
@@ -1599,30 +1626,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if 'authorUrl' in book['all_metadata']:
             authurls = book['all_metadata']['authorUrl'].split(", ")
             authorlist = [ a.replace('&',';') for a in book['author'] ]
-            if hasattr(db, 'new_api'): # new_api starts in calibre 1.0.0
-                authorids = db.new_api.get_item_ids('authors',authorlist)
-                authordata = db.new_api.author_data(authorids.values())
-                # print("\n\nauthorids:%s"%authorids)
-                # print("authordata:%s"%authordata)
+            authorids = db.new_api.get_item_ids('authors',authorlist)
+            authordata = db.new_api.author_data(authorids.values())
+            # print("\n\nauthorids:%s"%authorids)
+            # print("authordata:%s"%authordata)
 
-                author_id_to_link_map = dict()
-                for i, author in enumerate(authorlist):
-                    author_id_to_link_map[authorids[author]] = authurls[i]
+            author_id_to_link_map = dict()
+            for i, author in enumerate(authorlist):
+                author_id_to_link_map[authorids[author]] = authurls[i]
 
-                # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
-                db.new_api.set_link_for_authors(author_id_to_link_map)
-            else:
-                # keep for pre-calibre 1.0.0
-                for i, auth in enumerate(authorlist):
-                    #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                    autid=db.get_author_id(auth)
-                    db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
+            # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
+            db.new_api.set_link_for_authors(author_id_to_link_map)
+            
         db.commit()
 
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
 
-            logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
+            #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
             
             # force a refresh if generating cover so complex composite
             # custom columns are current and correct
@@ -1665,6 +1685,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 realmi = db.get_metadata(book_id, index_is_id=True)
                 gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
 
+                if prefs['gc_polish_cover'] and \
+                        options['fileform'] == "epub":
+                    # set cover inside epub from calibre's polish feature
+                    from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+                    from calibre.utils.logging import Log
+                    from collections import namedtuple
+
+                    # Couldn't find a better way to get the cover path.
+                    cover_path = os.path.join(db.library_path, db.path(book_id, index_is_id=True), 'cover.jpg')
+                    data = {'cover':cover_path}
+                    #print("cover_path:%s"%cover_path)
+                    opts = ALL_OPTS.copy()
+                    opts.update(data)
+                    O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+                    opts = O(**opts)
+                
+                    log = Log(level=Log.DEBUG)
+                    outfile = db.format_abspath(book_id, formmapping[options['fileform']], index_is_id=True)
+                    #print("polish cover outfile:%s"%outfile)
+                    polish({outfile:outfile}, opts, log, logger.info)
+                
+
     def get_clean_reading_lists(self,lists):
         if lists == None or lists.strip() == "" :
             return []
@@ -1753,6 +1795,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['end'] = None
         book['comment'] = '' # note this is a comment on the d/l or update.
         book['url'] = ''
+        book['site'] = ''
         book['added'] = False
         return book
     
@@ -1829,13 +1872,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             book['status'] = _('Not Found')
         else:
             # get normalized url or None.
-            book['url'] = self.is_good_downloader_url(url)
-            if book['url'] == None:
+            urlsitetuple = adapters.getNormalStoryURLSite(url)
+            if urlsitetuple == None:
                 book['url'] = url
                 book['comment'] = _("URL is not a valid story URL.")
                 book['good'] = False
                 book['icon']='dialog_error.png'
                 book['status'] = _('Bad URL')
+            else:
+                (book['url'],book['site'])=urlsitetuple
     
     def get_story_url(self, db, book_id=None, path=None):
         if book_id == None:
@@ -1940,11 +1985,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             for (k,v) in b['all_metadata'].iteritems():
                 #print("merge_meta_books v:%s k:%s"%(v,k))
                 if k in ('numChapters','numWords'):
-                    if k not in book['all_metadata']:
-                        book['all_metadata'][k] = b['all_metadata'][k]
-                    else:
-                        # lot of work for a simple add.
-                        book['all_metadata'][k] = unicode(int(book['all_metadata'][k].replace(',',''))+int(b['all_metadata'][k].replace(',','')))
+                    if k in b['all_metadata'] and b['all_metadata'][k]:
+                        if k not in book['all_metadata']:
+                            book['all_metadata'][k] = b['all_metadata'][k]
+                        else:
+                            # lot of work for a simple add.
+                            book['all_metadata'][k] = unicode(int(book['all_metadata'][k].replace(',',''))+int(b['all_metadata'][k].replace(',','')))
                 elif k in ('dateUpdated','datePublished','dateCreated',
                            'series','status','title'):
                     pass # handled above, below or skip these for now, not going to do anything with them.
@@ -1959,7 +2005,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         # cust cols can convert back to numbers and
                         # add.
                         book['anthology_meta_list'][k]=True
-    
+
+        print("book['url']:%s"%book['url'])
+        configuration = get_ffdl_config(book['url'],fileform)
         if existingbook:
             book['title'] = deftitle = existingbook['title']
             book['comments'] = existingbook['comments']
@@ -1978,7 +2026,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         book['title'] = deftitle
                         break
             
-            configuration = get_ffdl_config(book['url'],fileform)
             logger.debug("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
             if configuration.getConfig('anthology_title_pattern'):
                 tmplt = Template(configuration.getConfig('anthology_title_pattern'))
@@ -1994,7 +2041,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         for v in ['Completed','In-Progress']:
             if v in book['tags']:
                 book['tags'].remove(v)
-        book['tags'].append('Anthology')
+        book['tags'].extend(configuration.getConfigList('anthology_tags'))
         book['all_metadata']['anthology'] = "true"
         
         return book

calibre-plugin/jobs.py

@@ -93,16 +93,13 @@ def do_download_worker(book_list, options,
         logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
         logger.info(job.details)
 
+
+
         if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            logger.info("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
-            logger.info("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\nSuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                      filter(lambda x: x['good'], book_list) ] ) ) )
+            logger.info("\nUnsuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
             break
 
     server.close()

calibre-plugin/prefs.py

@@ -57,6 +57,7 @@ default_prefs['addtolistsonread'] = False
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
+default_prefs['gc_polish_cover'] = False
 
 default_prefs['countpagesstats'] = []
 

defaults.ini

@@ -182,6 +182,11 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
 
+## How long to wait for each HTTP connection to finish.  Longer times
+## are better for sites that are slow to respond.  Shorter times
+## prevent excessive wait when your network or the site is down.
+connect_timeout:60.0
+
 ## For use only with stand-alone CLI version--run a command on the
 ## generated file after it's produced.  All of the titlepage_entries
 ## values are available, plus output_filename.
@@ -214,7 +219,32 @@ extratags: FanFiction
 # .*-Centered=>
 # characters=>Sam W\.=>Sam Witwicky&&category=>Transformers
 # characters=>Sam W\.=>Sam Winchester&&category=>Supernatural
- 
+
+## Include/Exclude metadata
+##
+## You can use the include/exclude metadata features to either limit
+## the values of particular metadata lists to specific values or to
+## exclude specific values.  Further, you can conditionally apply each
+## line depending on other metadata, use exact strings or regular
+## expressions(regex) to match values, and negate matches.
+##
+## The settings are:
+## include_metadata_pre
+## exclude_metadata_pre
+## include_metadata_post
+## exclude_metadata_post
+##
+## The form of each line is:
+## metakey[,metakey]==exactvalue
+## metakey[,metakey]=~regex
+## metakey[,metakey]==exactvalue&&conditionalkey==exactcondvalue
+## metakey[,metakey]=~regex&&conditionalkey==exactcondvalue
+## metakey[,metakey]==exactvalue&&conditionalkey=~condregex
+##
+## This is fairly complicated, so it's documented on its own wiki
+## page:
+## https://code.google.com/p/fanficdownloader/wiki/InExcludeMetadataFeature
+
 ## Some readers don't show horizontal rule (<hr />) tags correctly.
 ## This replaces them all with a centered '* * *'.  (Note centering
 ## doesn't work on some devices either.)
@@ -568,6 +598,12 @@ include_in_freefromtags:freeformtags
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
+## AO3 chapters can include several different types of notes.  We've
+## traditional included them all in the chapter text, but this allows
+## you to customize which you include.  Copy this parameter to your
+## personal.ini and list the ones you don't want.
+#exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
+
 [ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -590,6 +626,29 @@ extracategories:The Sentinel
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[bloodshedverse.com]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:warnings,reviews
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
+## Strips links found in the story text
+## Specific to bloodshedverse.com
+strip_text_links:true
+
 [bloodties-fans.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -731,6 +790,38 @@ extracategories:Harry Potter
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
+[fanfiction.csodaidok.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-2,auto
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,challenge
+reviews_label:Reviews
+challenge_label:Challenge
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+[fanfic.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfiction.mugglenet.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -781,6 +872,14 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [fictionpad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -937,6 +1036,17 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[nocturnal-light.net]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:readings,reviews
+readings_label:Readings
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
 [occlumency.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1027,6 +1137,16 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[spikeluver.com]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:warnings,reviews
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
 [stargate-atlantis.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
@@ -1125,6 +1245,13 @@ awards_label:Awards
 
 cover_exclusion_regexp:art/.*Awards.jpg
 
+[voracity2.e-fic.com]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,readings
+reviews_label:Reviews
+readings_label:Readings
+
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1215,6 +1342,11 @@ user_agent:
 ## Change this to false to use them anyway.
 never_make_cover: true
 
+## fanfiction.net is blocking people more aggressively.  If you
+## download fewer stories less often you can likely get by with
+## reducing this sleep.
+slow_down_sleep_time:4
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
@@ -1223,13 +1355,20 @@ extra_valid_entries:reviews,favs,follows
 ## to be romantic pairings.
 ships_label:Pairings
 
-## Date formats used by FFDL.  Published and Update don't have time.
+## Date formats used by FFDL.  Published and Update don't usually have
+## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
 #dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
 datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
 dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 
+## ffnet used to have a tendency to send out update notices in email
+## before all their servers were showing the update on the first
+## chapter.  It generates another server request and doesn't seem to
+## be needed lately, so now default it to off.
+check_next_chapter:false
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1265,14 +1404,6 @@ extratags:
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
-[ficwad.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
 [www.fimfiction.net]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1291,6 +1422,11 @@ extra_valid_entries:reviews,favs,follows
 ## when updating to enforce accurate chapters.
 #do_update_hook:false
 
+## fimfiction.net is reported to misinterprete some BBCode with
+## blockquotes incorrectly.  This fixes those instances and defaults
+## to on, but can be switched off if it is found to cause problems.
+fix_fimf_blockquotes:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
 

downloader.py

@@ -23,6 +23,7 @@ import getpass
 import string
 import ConfigParser
 from subprocess import call
+import pprint
 
 import logging
 if sys.version_info >= (2, 7):
@@ -270,31 +271,29 @@ def main(argv,
            elif chaptercount == 0:
                print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
            else:
-               if not options.metaonly:
+               # update now handled by pre-populating the old
+               # images and chapters in the adapter rather than
+               # merging epubs.
+               (url,
+                chaptercount,
+                adapter.oldchapters,
+                adapter.oldimgs,
+                adapter.oldcover,
+                adapter.calibrebookmark,
+                adapter.logfile) = get_update_data(output_filename)
 
-                   # update now handled by pre-populating the old
-                   # images and chapters in the adapter rather than
-                   # merging epubs.
-                   (url,
-                    chaptercount,
-                    adapter.oldchapters,
-                    adapter.oldimgs,
-                    adapter.oldcover,
-                    adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(output_filename)
+               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
 
-                   print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+               if not (options.update and chaptercount == urlchaptercount) \
+                       and adapter.getConfig("do_update_hook"):
+                   chaptercount = adapter.hookForUpdates(chaptercount)
 
-                   if not (options.update and chaptercount == urlchaptercount) \
-                           and adapter.getConfig("do_update_hook"):
-                       chaptercount = adapter.hookForUpdates(chaptercount)
-
-                   writeStory(configuration,adapter,"epub")
+               writeStory(configuration,adapter,"epub")
                    
        else:
            # regular download
            if options.metaonly:
-               print adapter.getStoryMetadataOnly()
+               pprint.pprint(adapter.getStoryMetadataOnly().getAllMetadata())
            
            output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
        

fanficdownloader/adapters/__init__.py

@@ -123,6 +123,12 @@ import adapter_fictionpadcom
 import adapter_storiesonlinenet
 import adapter_trekiverseorg
 import adapter_literotica
+import adapter_voracity2eficcom
+import adapter_spikeluvercom
+import adapter_bloodshedversecom
+import adapter_nocturnallightnet
+import adapter_fanfichu
+import adapter_fanfictioncsodaidokhu
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -146,6 +152,13 @@ for x in imports():
             __domain_map[site]=cls
 
 def getNormalStoryURL(url):
+    r = getNormalStoryURLSite(url)
+    if r:
+        return r[0]
+    else:
+        return None
+
+def getNormalStoryURLSite(url):
     if not getNormalStoryURL.__dummyconfig:
         getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
     # pulling up an adapter is pretty low over-head.  If
@@ -153,10 +166,11 @@ def getNormalStoryURL(url):
     try:
         adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
         url = adapter.url
+        site = adapter.getSiteDomain()
         del adapter
-        return url
+        return (url,site)
     except:
-        return None;
+        return None
 
 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None
@@ -193,12 +207,12 @@ def getConfigSectionFor(url):
 
 def getClassFor(url):
     ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
     if not fixedurl.startswith("http"):
         fixedurl = "http://%s"%url
     ## remove any trailing '#' locations.
     fixedurl = re.sub(r"#.*$","",fixedurl)
-    
+
     parsedUrl = up.urlparse(fixedurl)
     domain = parsedUrl.netloc.lower()
     if( domain != parsedUrl.netloc ):

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -84,7 +84,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
         # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
+        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -145,10 +145,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             
         except urllib2.HTTPError, e:
             if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.meta)
+                raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
-                
+
+        if "Sorry, we couldn&#x27;t find the work you were looking for." in data:
+            raise exceptions.StoryDoesNotExist(self.url)
+            
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url,data)
@@ -321,25 +324,31 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-		
-        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-        if headnotes != None:
-            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+
+        exclude_notes=self.getConfigList('exclude_notes')
+
+        if 'authorheadnotes' not in exclude_notes:
+            headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
             if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(headnotes)
+                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+                if headnotes != None:
+                    chapter.append("<b>Author's Note:</b>")
+                    chapter.append(headnotes)
         
-        chapsumm = soup.find('div', {'id' : "summary"})
-        if chapsumm != None:
-            chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
-            chapter.append(chapsumm)
-        chapnotes = soup.find('div', {'id' : "notes"})
-        if chapnotes != None:
-            chapnotes = chapnotes.find('blockquote')
+        if 'chaptersummary' not in exclude_notes:
+            chapsumm = soup.find('div', {'id' : "summary"})
+            if chapsumm != None:
+                chapsumm = chapsumm.find('blockquote')
+                chapter.append("<b>Summary for the Chapter:</b>")
+                chapter.append(chapsumm)
+                
+        if 'chapterheadnotes' not in exclude_notes:
+            chapnotes = soup.find('div', {'id' : "notes"})
             if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapnotes)
+                chapnotes = chapnotes.find('blockquote')
+                if chapnotes != None:
+                    chapter.append("<b>Notes for the Chapter:</b>")
+                    chapter.append(chapnotes)
 		
         text = soup.find('div', {'class' : "userstuff module"})
         chtext = text.find('h3', {'class' : "landmark heading"})
@@ -347,17 +356,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chtext.extract()
         chapter.append(text)
 		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-        if chapfoot != None:
-            chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
-            chapter.append(chapfoot)
+        if 'chapterfootnotes' not in exclude_notes:
+            chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+            if chapfoot != None:
+                chapfoot = chapfoot.find('blockquote')
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapfoot)
 		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        if footnotes != None:
-            footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
-            chapter.append(footnotes)
+        if 'authorfootnotes' not in exclude_notes:
+            footnotes = soup.find('div', {'id' : "work_endnotes"})
+            if footnotes != None:
+                footnotes = footnotes.find('blockquote')
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(footnotes)
 			
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_bloodshedversecom.py

@@ -0,0 +1,192 @@
+from datetime import timedelta
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return BloodshedverseComAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class BloodshedverseComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'bvc'
+    SITE_DOMAIN = 'bloodshedverse.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/'
+    READ_URL_TEMPLATE = BASE_URL + 'stories.php?go=read&no=%s'
+
+    STARTED_DATETIME_FORMAT = '%m/%d/%Y'
+    UPDATED_DATETIME_FORMAT = '%m/%d/%Y %I:%M'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_no = query_data['no'][0]
+
+        self.story.setMetadata('storyId', story_no)
+        self._setURL(self.READ_URL_TEMPLATE % story_no)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return BloodshedverseComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL + 'stories.php?go=') + r'(read|chapters)\&no=\d+$'
+
+    # Override stripURLParameters so the "no" parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if soup.title.string.strip() == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        for option in soup.find('select', {'name': 'chapter'}):
+            title = option.string.strip()
+            url = self.READ_URL_TEMPLATE % option['value']
+            self.chapterUrls.append((title, url))
+
+        # Get the URL to the author's page and find the correct story entry to
+        # scrape the metadata
+        author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_no = self.story.getMetadata('storyId')
+        # Ignore first list_box div, it only contains the author information
+        for list_box in soup('div', {'class': 'list_box'})[1:]:
+            url = list_box.find('a', {'class': 'fictitle'})['href']
+            query_data = _get_query_data(url)
+
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if query_data['no'] == story_no:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title_anchor = list_box.find('a', {'class': 'fictitle'})
+        self.story.setMetadata('title', title_anchor.string.strip())
+
+        author_anchor = title_anchor.findNextSibling('a')
+        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
+
+        list_review = list_box.find('div', {'class': 'list_review'})
+        reviews = list_review.a.string.strip().split(' ', 1)[0]
+        self.story.setMetadata('reviews', reviews)
+
+        summary_div = list_box.find('div', {'class': 'list_summary'})
+        if not self.getConfig('keep_summary_html'):
+            summary = ''.join(summary_div(text=True))
+        else:
+            summary = self.utf8FromSoup(author_url, summary_div)
+
+        self.story.setMetadata('description', summary)
+
+        # I'm assuming this to be the category, not sure what else it could be
+        first_listinfo = list_box.find('div', {'class': 'list_info'})
+        self.story.addToList('category', first_listinfo.a.string.strip())
+
+        for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
+            for b_tag in list_info('b'):
+                key = b_tag.string.strip(': ')
+                # Strip colons from the beginning, superfluous spaces and minus
+                # characters from the end, and possibly trailing commas from
+                # the warnings if only one is present
+                value = b_tag.nextSibling.string.strip(': -,')
+
+                if key == 'Genre':
+                    for genre in value.split(', '):
+                        # Ignore the "none" genre
+                        if not genre == 'none':
+                            self.story.addToList('genre', genre)
+
+                elif key == 'Rating':
+                    self.story.setMetadata('rating', value)
+
+                elif key == 'Complete':
+                    self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+                elif key == 'Warning':
+                    for warning in value.split(', '):
+                        # The string here starts with ", " before the actual list
+                        # of values sometimes, so check for an empty warning
+                        # and ignore the "none" warning.
+                        if not warning or warning == 'none':
+                            continue
+
+                        self.story.addToList('warnings', warning)
+
+                elif key == 'Chapters':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Words':
+                    # Apparently only numChapters need to be an integer for
+                    # some strange reason. Remove possible ',' characters as to
+                    # not confuse the codebase down the line
+                    self.story.setMetadata('numWords', value.replace(',', ''))
+
+                elif key == 'Started':
+                    self.story.setMetadata('datePublished', makeDate(value, self.STARTED_DATETIME_FORMAT))
+
+                elif key == 'Updated':
+                    date_string, period = value.rsplit(' ', 1)
+                    date = makeDate(date_string, self.UPDATED_DATETIME_FORMAT)
+
+                    # Rather ugly hack to work around Calibre's changing of
+                    # Python's locale setting, causing am/pm to not be properly
+                    # parsed by strptime() when using a non-english locale
+                    if period == 'pm':
+                        date += timedelta(hours=12)
+                    self.story.setMetadata('dateUpdated', date)
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        storytext_div = soup.find('div', {'class': 'storytext'})
+
+        if self.getConfig('strip_text_links'):
+            for anchor in storytext_div('a', {'class': 'FAtxtL'}):
+                navigable_string = BeautifulSoup.NavigableString(anchor.string)
+                anchor.replaceWith(navigable_string)
+
+        return self.utf8FromSoup(url, storytext_div)

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -87,7 +87,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
     
     def setHeader(self):
         "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
-        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
         
     @classmethod
     def getSiteExampleURLs(self):

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -182,6 +182,11 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         # first a tag in pagetitle is title
         self.story.setMetadata('title',stripHTML(div.find('a')))
+        div.find('a').extract()
+        # only thing left in div(pagetitle) now should be 'by' and rating.
+        rating = stripHTML(div)
+        if '[' in rating:
+            self.story.setMetadata('rating', rating[rating.index('[')+1:-1])
 
         for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
                                                        self.story.getMetadata('storyId')+'&chapter=\d+')):
@@ -234,31 +239,28 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
                 if 'Yes' in value:

fanficdownloader/adapters/adapter_fanfichu.py

@@ -0,0 +1,185 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanficHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class FanficHuAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'ffh'
+    SITE_DOMAIN = 'fanfic.hu'
+    SITE_LANGUAGE = 'Hungarian'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/merengo/'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%s'
+
+    DATE_FORMAT = '%m/%d/%Y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanficHuAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&i=1')
+
+        if soup.title.string.encode(_SOURCE_CODE_ENCODING).strip(' :') == 'írta':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        chapter_options = soup.find('form', action='viewstory.php').select('option')
+        # Remove redundant "Fejezetek" option
+        chapter_options.pop(0)
+
+        # If there is still more than one entry remove chapter overview entry
+        if len(chapter_options) > 1:
+            chapter_options.pop(0)
+
+        for option in chapter_options:
+            url = urlparse.urljoin(self.url, option['value'])
+            self.chapterUrls.append((option.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_id = self.story.getMetadata('storyId')
+        for table in soup('table', {'class': 'mainnav'}):
+            title_anchor = table.find('span', {'class': 'storytitle'}).a
+            href = title_anchor['href']
+            if href.startswith('javascript:'):
+                href = href.rsplit(' ', 1)[1].strip("'")
+            query_data = _get_query_data(href)
+
+            if query_data['sid'] == story_id:
+                break
+        else:
+            # This should never happen, the story must be found on the author's
+            # page.
+            raise exceptions.FailedToDownload(self.url)
+
+        self.story.setMetadata('title', title_anchor.string)
+
+        rows = table('tr')
+
+        anchors = rows[0].div('a')
+        author_anchor = anchors[1]
+        query_data = _get_query_data(author_anchor['href'])
+        self.story.setMetadata('author', author_anchor.string)
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.BASE_URL, author_anchor['href']))
+        self.story.setMetadata('reviews', anchors[3].string)
+
+        if self.getConfig('keep_summary_html'):
+            self.story.setMetadata('description', self.utf8FromSoup(author_url, rows[1].td))
+        else:
+            self.story.setMetadata('description', ''.join(rows[1].td(text=True)))
+
+        for row in rows[3:]:
+            index = 0
+            cells = row('td')
+
+            while index < len(cells):
+                cell = cells[index]
+                key = cell.b.string.encode(_SOURCE_CODE_ENCODING).strip(':')
+                try:
+                    value = cells[index+1].string.encode(_SOURCE_CODE_ENCODING)
+                except AttributeError:
+                    value = None
+
+                if key == 'Kategória':
+                    for anchor in cells[index+1]('a'):
+                        self.story.addToList('category', anchor.string)
+
+                elif key == 'Szereplõk':
+                    if cells[index+1].string:
+                        for name in cells[index+1].string.split(', '):
+                            self.story.addToList('character', name)
+
+                elif key == 'Korhatár':
+                    if value != 'nem korhatáros':
+                        self.story.setMetadata('rating', value)
+
+                elif key == 'Figyelmeztetések':
+                    for b_tag in cells[index+1]('b'):
+                        self.story.addToList('warnings', b_tag.string)
+
+                elif key == 'Jellemzõk':
+                    for genre in cells[index+1].string.split(', '):
+                        self.story.addToList('genre', genre)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Megjelenés':
+                    self.story.setMetadata('datePublished', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Frissítés':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Szavak':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                index += 2
+
+        if self.story.getMetadata('rating') == '18':
+            if not (self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        story_cell = soup.find('form', action='viewstory.php').parent.parent
+
+        for div in story_cell('div'):
+            div.extract()
+
+        return self.utf8FromSoup(url, story_cell)

fanficdownloader/adapters/adapter_fanfictioncsodaidokhu.py

@@ -0,0 +1,218 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanfictionCsodaidokHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FanfictionCsodaidokHuAdapter(BaseSiteAdapter):
+    _SITE_DOMAIN = 'fanfiction.csodaidok.hu'
+    _BASE_URL = 'http://' + _SITE_DOMAIN + '/'
+    _VIEW_STORY_URL_TEMPLATE = _BASE_URL + 'viewstory.php?sid=%s'
+    _VIEW_CHAPTER_URL_TEMPLATE = _VIEW_STORY_URL_TEMPLATE + '&chapter=%s'
+
+    _STORY_DOES_NOT_EXIST_PAGE_TITLE = 'Cím:  Szerző:'
+    _DATE_FORMAT = '%Y.%m.%d'
+    _SITE_LANGUAGE = 'Hungarian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self._VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self._SITE_DOMAIN)
+        self.story.setMetadata('language', self._SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanfictionCsodaidokHuAdapter._SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls._VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self._VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&chapter=1')
+
+        element = soup.find('div', id='pagetitle')
+        page_title = ''.join(element(text=True)).encode(_SOURCE_CODE_ENCODING)
+        if page_title == self._STORY_DOES_NOT_EXIST_PAGE_TITLE:
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        author_url = urlparse.urljoin(self.url, element.a['href'])
+
+        story_id = self.story.getMetadata('storyId')
+        element = soup.find('select', {'name': 'chapter'})
+        if element:
+            for option in element('option'):
+                title = option.string
+                url = self._VIEW_CHAPTER_URL_TEMPLATE % (story_id, option['value'])
+                self.chapterUrls.append((title, url))
+
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+
+        for listbox_div in soup('div', {'class': lambda klass: klass and 'listbox' in klass}):
+            a = listbox_div.div.a
+            if not a['href'].startswith('viewstory.php?sid='):
+                continue
+
+            query_data = _get_query_data(a['href'])
+            if query_data['sid'] == story_id:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = ''.join(a(text=True))
+        self.story.setMetadata('title', title)
+        if not self.chapterUrls:
+            self.chapterUrls.append((title, self.url))
+
+        element = a.findNextSibling('a')
+        self.story.setMetadata('author', element.string)
+        query_data = _get_query_data(element['href'])
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', author_url)
+
+        element = element.findNextSibling('span')
+        rating = element.nextSibling.strip(' [')
+
+        if rating.encode(_SOURCE_CODE_ENCODING) != 'Korhatár nélkül':
+            self.story.setMetadata('rating', rating)
+
+        if rating == '18':
+            raise exceptions.AdultCheckRequired(self.url)
+
+        element = element.findNextSiblings('a')[1]
+        self.story.setMetadata('reviews', element.string)
+
+        sections = listbox_div('div', {'class': lambda klass: klass and klass in ['content', 'tail']})
+        for section in sections:
+            for element in section('span', {'class': 'classification'}):
+                key = element.string.encode(_SOURCE_CODE_ENCODING).strip(' :')
+                try:
+                    value = element.nextSibling.string.encode(_SOURCE_CODE_ENCODING).strip()
+                except AttributeError:
+                    value = None
+
+                if key == 'Tartalom':
+                    contents = []
+                    keep_summary_html = self.getConfig('keep_summary_html')
+
+                    for sibling in _yield_next_siblings(element):
+                        if isinstance(sibling, BeautifulSoup.Tag):
+                            if sibling.name == 'span' and sibling.get('class', None) == 'classification':
+                                break
+
+                            if keep_summary_html:
+                                contents.append(self.utf8FromSoup(author_url, sibling))
+                            else:
+                                contents.append(''.join(sibling(text=True)))
+                        else:
+                            contents.append(sibling)
+                    self.story.setMetadata('description', ''.join(contents))
+
+                elif key == 'Kategória':
+                    for sibling in element.findNextSiblings(['a', 'span']):
+                        if sibling.name == 'span':
+                            break
+
+                        self.story.addToList('category', sibling.string)
+
+                elif key == 'Szereplők':
+                    for name in value.split(', '):
+                        self.story.addToList('characters', name)
+
+                elif key == 'Műfaj':
+                    if value != 'Nincs':
+                        self.story.setMetadata('genre', value)
+
+                elif key == 'Figyelmeztetés':
+                    if value != 'Nincs':
+                        for warning in value.split(', '):
+                            self.story.addToList('warnings', warning)
+
+                elif key == 'Kihívás':
+                    if value != 'Nincs':
+                        self.story.setMetadata('challenge', value)
+
+                elif key == 'Sorozat':
+                    if value != 'Nincs':
+                        self.story.setMetadata('series', value)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                elif key == 'Szavak száma':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Feltöltve':
+                    self.story.setMetadata('datePublished', makeDate(value, self._DATE_FORMAT))
+
+                elif key == 'Frissítve':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self._DATE_FORMAT))
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        contents = []
+
+        notes_div = soup.find('div', id='notes')
+        if notes_div:
+            contents.append(self.utf8FromSoup(url, notes_div))
+            story_div = notes_div.findNextSibling('div')
+        else:
+            element = soup.find('div', {'class': 'jumpmenu'})
+            story_div = element.findNextSibling('div')
+
+        contents.append(self.utf8FromSoup(url, story_div.span))
+        return ''.join(contents)

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -57,6 +57,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             ## accept m(mobile)url, but use www.
             self.origurl = self.origurl.replace("https://m.","https://www.")
 
+        self.opener.addheaders.append(('Referer',self.origurl))
+
     @staticmethod
     def getSiteDomain():
         return 'www.fanfiction.net'
@@ -105,28 +107,31 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if "not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
 
-        try:
-            # rather nasty way to check for a newer chapter.  ffnet has a
-            # tendency to send out update notices in email before all
-            # their servers are showing the update on the first chapter.
+        if self.getConfig('check_next_chapter'):
             try:
-                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
-            # get chapter part of url.
+                ## ffnet used to have a tendency to send out update
+                ## notices in email before all their servers were
+                ## showing the update on the first chapter.  It
+                ## generates another server request and doesn't seem
+                ## to be needed lately, so now default it to off.
+                try:
+                    chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+                # get chapter part of url.
+                except:
+                    chapcount = 1
+                chapter = url.split('/',)[5]
+                tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                                self.story.getMetadata('storyId'),
+                                                chapcount+1)
+                logger.debug('=Trying newer chapter: %s' % tryurl)
+                newdata = self._fetchUrl(tryurl)
+                if "not found. Please check to see you are not using an outdated url." \
+                        not in newdata:
+                    logger.debug('=======Found newer chapter: %s' % tryurl)
+                    soup = bs.BeautifulSoup(newdata)
             except:
-                chapcount = 1
-            chapter = url.split('/',)[5]
-            tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            chapcount+1)
-            logger.debug('=Trying newer chapter: %s' % tryurl)
-            newdata = self._fetchUrl(tryurl)
-            if "not found. Please check to see you are not using an outdated url." \
-                    not in newdata:
-                logger.debug('=======Found newer chapter: %s' % tryurl)
-                soup = bs.BeautifulSoup(newdata)
-        except:
-            pass
-        
+                pass
+            
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -133,6 +133,7 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
         author = tables['users'][0]
         story = tables['stories'][0]
         story_ver = tables['story_versions'][0]
+        print("story:%s"%story)
         
         self.story.setMetadata('authorId',author['id'])
         self.story.setMetadata('author',author['display_name'])
@@ -151,7 +152,8 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('comments',story['comments_count'])
         self.story.setMetadata('views',story['views_count'])
         self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
-        self.story.setMetadata('dislikes',int(story['dislikes']))
+        if 'dislikes' in story:
+            self.story.setMetadata('dislikes',int(story['dislikes']))
 
         if story_ver['is_complete']:
             self.story.setMetadata('status', 'Completed')

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -41,10 +41,10 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
+        return "https://www.fictionpress.com/s/1234/1/ https://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"https?://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
 
 def getClass():
     return FictionPressComSiteAdapter

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -59,7 +59,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"
 
     def getSiteURLPattern(self):
-        return r"http://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
+        return r"https?://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
         
     def extractChapterUrlsAndMetadata(self):
         
@@ -85,7 +85,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             
             # Unfortunately, we still need to load the story index
             # page to parse the characters.  And chapters, now, too.
-            data = self._fetchUrl(self.url)
+            data = self.do_fix_blockquotes(self._fetchUrl(self.url))
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -101,7 +101,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         # if "/images/missing_story.png" in data:
         #     raise exceptions.StoryDoesNotExist(self.url)
         
-        if "This story has been marked as having adult content." in data:
+        if "This story has been marked as having adult content. Please click below to confirm you are of legal age to view adult material in your country." in data:
             raise exceptions.AdultCheckRequired(self.url)
         
         if self.password:
@@ -249,18 +249,20 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
 
-        #Sequel links and group links are each bundled into story_group_list containers.
-        #Rather than mess around examining the header text, which is outside the containers,
-        #one can tell the two link types apart by examining them directly.
+        ## Groups and sequels code from FaceDeer
         allGroupLists = soup.findAll('ul', {'id':'story_group_list'})
         for groupList in allGroupLists:
             for groupName in groupList.findAll('a', {'href':re.compile('^/group/')}):
                 self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) 
                 self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
-            for sequel in groupList.findAll('a', {'class':'story_link'}):
+
+        sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels")
+        if not sequelStoryHeader == None:
+            sequelContainer = sequelStoryHeader.parent.parent
+            for sequel in sequelContainer.findAll('a', {'class':'story_link'}):
                 self.story.addToList("sequelsUrl", 'http://'+self.host+sequel["href"]) 
                 self.story.addToList("sequels", stripHTML(sequel).replace(',', ';'))
-
+                
         #The link to the prequel is embedded in the description text, so erring
         #on the side of caution and wrapping this whole thing in a try block.
         #If anything goes wrong this probably wasn't a valid prequel link.
@@ -281,12 +283,21 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
             self.oldchapters = self.oldchapters[:self.newestChapterNum]
         return len(self.oldchapters)
-            
+
+    def do_fix_blockquotes(self,data):
+        if self.getConfig('fix_fimf_blockquotes'):
+            # <p class="double"><blockquote>
+            # </blockquote></p>
+            # include > in re groups so there's always something in the group.
+            data = re.sub(r'<p([^>]*>\s*)<blockquote([^>]*>)',r'<blockquote\2<p\1',data)
+            data = re.sub(r'</blockquote(>\s*)</p>',r'</p\1</blockquote>',data)
+        return data
         
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
+
+        data = self.do_fix_blockquotes(self._fetchUrl(url))
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -160,6 +160,11 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
             if 'Warnings' in label:
                 warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                 for warning in warnings:

fanficdownloader/adapters/adapter_literotica.py

@@ -42,16 +42,19 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         
         self.story.setMetadata('siteabbrev','litero')
 
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        # normalize to first chapter.  Not sure if they ever have more than 2 digits.
+        storyid = self.parsedUrl.path.split('/',)[2]
+        if re.match(r'-ch\d\d$',storyid):
+            storyid = storyid[:-2]+'01'
+        self.story.setMetadata('storyId',storyid)
         
         self.origurl = url
-        if "http://www.i." in self.origurl:
+        if "//www.i." in self.origurl:
             ## accept m(mobile)url, but use www.
-            self.origurl = self.origurl.replace("http://www.i.","http://www.")
+            self.origurl = self.origurl.replace("//www.i.","//www.")
 
         # normalized story URL.
-        self._setURL("http://"+self.getSiteDomain()\
+        self._setURL(url[:url.index('//')+2]+self.getSiteDomain()\
                          +"/s/"+self.story.getMetadata('storyId'))
 
         # The date format will vary from site to site.
@@ -69,10 +72,10 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
     @classmethod
     def getSiteExampleURLs(self):
         #return "http://www.literotica.com/s/story-title http://www.literotica.com/stories/showstory.php?id=1234 http://www.i.literotica.com/stories/showstory.php?id=1234"
-        return "http://www.literotica.com/s/story-title"
+        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title"
 
     def getSiteURLPattern(self):
-        return r"http://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+        return r"https?://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -97,20 +100,24 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         # author
         a = soup1.find("span", "b-story-user-y")
         self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'])
-        self.story.setMetadata('authorUrl', a.a['href'])
+        authorurl = a.a['href']
+        if authorurl.startswith('//'):
+            authorurl = self.parsedUrl.scheme+':'+authorurl
+        self.story.setMetadata('authorUrl', authorurl)
         self.story.setMetadata('author', a.text)
 
         # get the author page
         try:
-            dataAuth = self._fetchUrl(a.a['href'])
+            dataAuth = self._fetchUrl(authorurl)
             soupAuth = bs.BeautifulSoup(dataAuth)
         except urllib2.HTTPError, e:
             if e.code == 404:
-                raise exceptions.StoryDoesNotExist(a.a['href'])
+                raise exceptions.StoryDoesNotExist(authorurl)
             else:
                 raise e
 
-        storyLink = soupAuth.find('a', href=url1)
+        ## site has started using //domain.name/asdf urls remove https?: from front
+        storyLink = soupAuth.find('a', href=url1[url1.index(':')+1:])
 
         if storyLink is not None:
             # pull the published date from the author page
@@ -166,7 +173,10 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
             self.story.setMetadata('datePublished',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
             while row['class'] == 'sl':
                 # pages include full URLs.
-                self.chapterUrls.append((row.a.string,row.a['href']))
+                chapurl = row.a['href']
+                if chapurl.startswith('//'):
+                    chapurl = self.parsedUrl.scheme+':'+chapurl
+                self.chapterUrls.append((row.a.string,chapurl))
                 if not row.nextSibling:
                     break
                 row = row.nextSibling
@@ -203,7 +213,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         # get story text
         story1 = soup1.find('div', 'b-story-body-x').p
         story1.name='div'
-        story1.append('<br>')
+        story1.append('<br />')
         storytext = self.utf8FromSoup(url,story1)
 
         # find num pages
@@ -220,7 +230,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
                 [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
                 story2 = soup2.find('div', 'b-story-body-x').p
                 story2.name='div'
-                story2.append('<br>')
+                story2.append('<br />')
                 storytext += self.utf8FromSoup(url,story2)
             except urllib2.HTTPError, e:
                 if e.code == 404:

fanficdownloader/adapters/adapter_nocturnallightnet.py

@@ -0,0 +1,177 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return NocturnalLightNetAdapter
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class NocturnalLightNetAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'nln'
+    SITE_DOMAIN = 'nocturnal-light.net'
+    BASE_URL = 'http://' + SITE_DOMAIN + '/fanfiction/'
+    STORY_URL_TEMPLATE = BASE_URL + 'story/%s'
+    AUTHORS_URL_TEMPLATE = BASE_URL + 'authors/%s'
+
+    DATETIME_FORMAT = '%m-%d-%y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        url_tokens = self.parsedUrl.path.split('/')
+        story_id = url_tokens[url_tokens.index('story') + 1]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return NocturnalLightNetAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.STORY_URL_TEMPLATE[:-2]) + r'\d+.*$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if soup.title.string.strip() == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        # "storycontent" is found in a single-chapter story
+        author_anchor = soup.find('div', id=lambda id: id in ('main', 'storycontent')).h1.a
+        self.story.setMetadata('author', author_anchor.string)
+
+        url_tokens = author_anchor['href'].split('/')
+        author_id = url_tokens[url_tokens.index('authors')+1]
+        self.story.setMetadata('authorId', author_id)
+        self.story.setMetadata('authorUrl', self.AUTHORS_URL_TEMPLATE % author_id)
+
+        chapter_anchors = soup('a', href=lambda href: href and href.startswith('/fanfiction/story/'))
+        for chapter_anchor in chapter_anchors:
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((chapter_anchor.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+        for listbox in soup('div', {'class': 'listbox'}):
+            url_tokens = listbox.a['href'].split('/')
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if story_id == url_tokens[url_tokens.index('story')+1]:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = listbox.a.string
+        self.story.setMetadata('title', title)
+
+        # No chapter anchors found in the original story URL, so the story has
+        # only a single chapter.
+        if not chapter_anchors:
+            self.chapterUrls.append((title, self.url))
+
+        for b_tag in listbox('b'):
+            key = b_tag.string.strip(':')
+            try:
+                value = b_tag.nextSibling.string.replace('•', '').strip(': ')
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                for sibling in _yield_next_siblings(b_tag):
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(author_url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+
+                # Pop last break line tag
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Category':
+                for sibling in b_tag.findNextSiblings(['a', 'b']):
+                    if sibling.name == 'b':
+                        break
+
+                    self.story.addToList('category', sibling.string)
+
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Chapters':
+                self.story.setMetadata('numChapters', int(value))
+
+                # Also parse reviews number which lies right after the chapters
+                # section
+                reviews_anchor = b_tag.findNextSibling('a')
+                reviews = reviews_anchor.string.split(' ')[1].strip('()')
+                self.story.setMetadata('reviews', reviews)
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Date Added':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Last Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Read':
+                self.story.setMetadata('readings', value.split()[0])
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        return self.utf8FromSoup(url, soup.find('div', id='storytext'))

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -192,7 +192,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
+                while value and not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -163,12 +163,11 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
+                # limit date values, there's some extra chars.
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y"))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y"))
 
         try:
             # Find Series name from series URL.

fanficdownloader/adapters/adapter_spikeluvercom.py

@@ -0,0 +1,206 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return SpikeluverComAdapter
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class SpikeluverComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'slc'
+    SITE_DOMAIN = 'spikeluver.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/SpuffyRealm/'
+    LOGIN_URL = BASE_URL + 'user.php?action=login'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%d'
+    METADATA_URL_SUFFIX = '&index=1'
+    AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=5'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+    STORY_DOES_NOT_EXIST_ERROR_TEXT = 'That story does not exist on this archive.  You may search for it or return to the home page.'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id))
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return SpikeluverComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX)
+
+        errortext_div = soup.find('div', {'class': 'errortext'})
+        if errortext_div:
+            error_text = ''.join(errortext_div(text=True)).strip()
+            if error_text == self.STORY_DOES_NOT_EXIST_ERROR_TEXT:
+                raise exceptions.StoryDoesNotExist(self.url)
+
+        # No additional login is required, just check for adult
+        pagetitle_div = soup.find('div', id='pagetitle')
+        if pagetitle_div.a['href'].startswith('javascript:'):
+            if not(self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+        url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
+        soup = self._customized_fetch_url(url)
+
+        pagetitle_div = soup.find('div', id='pagetitle')
+        self.story.setMetadata('title', pagetitle_div.a.string.strip())
+
+        author_anchor = pagetitle_div.a.findNextSibling('a')
+        url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        components = urlparse.urlparse(url)
+        query_data = urlparse.parse_qs(components.query)
+
+        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', url)
+
+        sort_div = soup.find('div', id='sort')
+        self.story.setMetadata('reviews', sort_div('a')[1].string.strip())
+
+        listbox_tag = soup.find('div', {'class': 'listbox'})
+        for span_tag in listbox_tag('span'):
+            key = span_tag.string.strip(' :')
+            try:
+                value = span_tag.nextSibling.string.strip()
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                for sibling in _yield_next_siblings(span_tag):
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        # Encountered next label, break. Not as bad as other
+                        # e-fiction sites, let's hope this is enough for proper
+                        # parsing.
+                        if sibling.name == 'span' and sibling.get('class', None) == 'label':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(self.url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+
+                # Remove the preceding break line tag and other crud
+                contents.pop()
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Rated':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Categories':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+
+                    self.story.addToList('category', sibling.string.strip())
+
+            # Seems to be always "None" for some reason
+            elif key == 'Characters':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('characters', sibling.string.strip())
+
+            elif key == 'Genres':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+
+                    self.story.addToList('genre', sibling.string.strip())
+
+            elif key == 'Warnings':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('warnings', sibling.string.strip())
+
+            # Challenges
+
+            elif key == 'Series':
+                a = span_tag.findNextSibling('a')
+                if not a:
+                    continue
+                self.story.setMetadata('series', a.string.strip())
+                self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href']))
+
+            elif key == 'Chapters':
+                self.story.setMetadata('numChapters', int(value))
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Word count':
+                self.story.setMetadata('numWords', value)
+
+            elif key == 'Published':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+        for p_tag in listbox_tag.findNextSiblings('p'):
+            chapter_anchor = p_tag.find('a', href=lambda href: href and href.startswith('viewstory.php?sid='))
+            if not chapter_anchor:
+                continue
+
+            title = chapter_anchor.string.strip()
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((title, url))
+
+    def getChapterText(self, url):
+        url += self.AGE_CONSENT_URL_SUFFIX
+        soup = self._customized_fetch_url(url)
+        return self.utf8FromSoup(url, soup.find('div', id='story'))

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -62,7 +62,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('https://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','wwomb')
@@ -83,10 +83,10 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
     
     @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
+        return "https://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/")+"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
+        return r"https?"+re.escape("://"+self.getSiteDomain()+"/")+r"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -116,7 +116,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # Find authorid and URL from... author url.
         author = soup.find('div', {'id':"pagetitle"}).find('a')
         self.story.setMetadata('authorId',author['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/peja/cgi-bin/'+author['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/peja/cgi-bin/'+author['href'])
         self.story.setMetadata('author',author.string)
 		
         authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
@@ -131,7 +131,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         chapterselect=soup.find('select',{'name':'chapter'})
         if chapterselect:
             for ch in chapterselect.findAll('option'):
-                self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
+                self.chapterUrls.append((stripHTML(ch),'https://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
         else:
             self.chapterUrls.append((title,url))
 		
@@ -207,7 +207,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
             # http://www.squidge.org/peja/cgi-bin/series.php?seriesid=254
             a = titleblock.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/peja/cgi-bin/'+a['href']
+            series_url = 'https://'+self.host+'/peja/cgi-bin/'+a['href']
     
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -66,7 +66,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)"
+        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)?"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -171,7 +171,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
             a = asoup.findAll('td', {'class' : 'lc2'})
             for lc2 in a:
-                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                if lc2.find('a', href=re.compile(r'^/s/'+self.story.getMetadata('storyId'))):
                     i=1
                     break
                 if a[len(a)-1] == lc2:

fanficdownloader/adapters/adapter_test1.py

@@ -50,6 +50,7 @@ class TestSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
         idstr = self.story.getMetadata('storyId')
         idnum = int(idstr)
+        self.do_sleep()
 
         if idnum >= 1000:
             logger.warn("storyId:%s - Custom INI data will be used."%idstr)
@@ -296,6 +297,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
+        self.do_sleep()
         if self.story.getMetadata('storyId').startswith('670') or \
                 self.story.getMetadata('storyId').startswith('672'):
             time.sleep(1.0)

fanficdownloader/adapters/adapter_voracity2eficcom.py

@@ -0,0 +1,232 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return Voracity2EficComAdapter
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class Voracity2EficComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'voe'
+    SITE_DOMAIN = 'voracity2.e-fic.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/'
+    LOGIN_URL = BASE_URL + 'user.php?action=login'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%d'
+    METADATA_URL_SUFFIX = '&index=1'
+    AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=4'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+    REQUIRED_SKIN = 'Simple Elegance'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id))
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+        self.is_logged_in = False
+
+    def _login(self):
+        # Apparently self.password is only set when login fails, i.e.
+        # the FailedToLogin exception is raised, so the adapter gets new
+        # login data and tries again
+        if self.password:
+            password = self.password
+            username = self.username
+        else:
+            username = self.getConfig('username')
+            password = self.getConfig('password')
+
+        parameters = {
+            'penname': username,
+            'password': password,
+            'submit': 'Submit'}
+
+        class CustomizedFailedToLogin(exceptions.FailedToLogin):
+            def __init__(self, url, passwdonly=False):
+                # Use username variable from outer scope
+                exceptions.FailedToLogin.__init__(self, url, username, passwdonly)
+
+        soup = self._customized_fetch_url(self.LOGIN_URL, CustomizedFailedToLogin, parameters)
+        div = soup.find('div', id='useropts')
+        if not div:
+            raise CustomizedFailedToLogin(self.LOGIN_URL)
+
+        self.is_logged_in = True
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return Voracity2EficComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX)
+
+        # Check if the story is for "Registered Users Only", i.e. has adult
+        # content. Based on the "is_adult" attributes either login or raise an
+        # error.
+        errortext_div = soup.find('div', {'class': 'errortext'})
+        if errortext_div:
+            error_text = ''.join(errortext_div(text=True)).strip()
+            if error_text == 'Registered Users Only':
+                if not (self.is_adult or self.getConfig('is_adult')):
+                    raise exceptions.AdultCheckRequired(self.url)
+                self._login()
+            else:
+                # This case usually occurs when the story doesn't exist, but
+                # might potentially be something else, so just raise
+                # FailedToDownload exception with the found error text.
+                raise exceptions.FailedToDownload(error_text)
+
+        url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
+        soup = self._customized_fetch_url(url)
+
+        # If logged in and the skin doesn't match the required skin throw an
+        # error
+        if self.is_logged_in:
+            skin = soup.find('select', {'name': 'skin'}).find('option', selected=True)['value']
+            if skin != self.REQUIRED_SKIN:
+                raise exceptions.FailedToDownload('Required skin "%s" must be set in preferences' % self.REQUIRED_SKIN)
+
+        pagetitle_div = soup.find('div', id='pagetitle')
+        self.story.setMetadata('title', pagetitle_div.a.string)
+
+        author_anchor = pagetitle_div.a.findNextSibling('a')
+        url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        components = urlparse.urlparse(url)
+        query_data = urlparse.parse_qs(components.query)
+
+        self.story.setMetadata('author', author_anchor.string)
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', url)
+
+        sort_div = soup.find('div', id='sort')
+        self.story.setMetadata('reviews', sort_div('a')[1].string)
+
+        for b_tag in soup.find('div', {'class': 'listbox'})('b'):
+            key = b_tag.string.strip(' :')
+            try:
+                value = b_tag.nextSibling.string.strip()
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                for sibling in _yield_next_siblings(b_tag):
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        # Encountered next label, break. This method is the
+                        # safest and most reliable I could think of. Blame
+                        # e-fiction sites that allow their users to include
+                        # arbitrary markup into their summaries and the
+                        # horrible HTML markup.
+                        if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(self.url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+
+                # Remove the preceding break line tag and other crud
+                contents.pop()
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Category':
+                for sibling in b_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('category', sibling.string)
+
+            # Seems to be always "None" for some reason
+            elif key == 'Characters':
+                for sibling in b_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('characters', sibling.string)
+
+            elif key == 'Series':
+                a = b_tag.findNextSibling('a')
+                if not a:
+                    continue
+                self.story.setMetadata('series', a.string)
+                self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href']))
+
+            elif key == 'Chapter':
+                self.story.setMetadata('numChapters', int(value))
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Words':
+                self.story.setMetadata('numWords', value)
+
+            elif key == 'Read':
+                self.story.setMetadata('readings', value)
+
+            elif key == 'Published':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+        for b_tag in soup.find('div', id='output').findNextSiblings('b'):
+            chapter_anchor = b_tag.a
+            title = chapter_anchor.string
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((title, url))
+
+    def getChapterText(self, url):
+        url += self.AGE_CONSENT_URL_SUFFIX
+        soup = self._customized_fetch_url(url)
+        return self.utf8FromSoup(url, soup.find('div', id='story'))

fanficdownloader/adapters/base_adapter.py

@@ -76,6 +76,8 @@ class BaseSiteAdapter(Configurable):
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
+
+        self.override_sleep = None
         
         self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         ## Specific UA because too many sites are blocking the default python UA.
@@ -147,8 +149,7 @@ class BaseSiteAdapter(Configurable):
 
     # Assumes application/x-www-form-urlencoded.  parameters, headers are dict()s
     def _postUrl(self, url, parameters={}, headers={}):
-        if self.getConfig('slow_down_sleep_time'):
-            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        self.do_sleep()
 
         ## u2.Request assumes POST when data!=None.  Also assumes data
         ## is application/x-www-form-urlencoded.
@@ -159,24 +160,38 @@ class BaseSiteAdapter(Configurable):
         req = u2.Request(url,
                          data=urllib.urlencode(parameters),
                          headers=headers)
-        return self._decode(self.opener.open(req).read())
+        return self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())
 
     def _fetchUrlRaw(self, url, parameters=None):
         if parameters != None:
-            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters)).read()
+            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
         else:
-            return self.opener.open(url.replace(' ','%20')).read()
+            return self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
+
+    def set_sleep(self,val):
+        print("\n===========\n set sleep time %s\n==========="%val)
+        self.override_sleep = val
     
+    def do_sleep(self):
+        if self.override_sleep:
+            time.sleep(float(self.override_sleep))
+        elif self.getConfig('slow_down_sleep_time'):
+            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        
     # parameters is a dict()
     def _fetchUrl(self, url, parameters=None):
-        if self.getConfig('slow_down_sleep_time'):
-            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        self.do_sleep()
 
         excpt=None
         for sleeptime in [0, 0.5, 4, 9]:
             time.sleep(sleeptime)	
             try:
                 return self._decode(self._fetchUrlRaw(url,parameters))
+            except u2.HTTPError, he:
+                excpt=he
+                if he.code == 404:
+                    logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(he)))
+                    break # break out on 404
             except Exception, e:
                 excpt=e
                 logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
@@ -367,7 +382,7 @@ class BaseSiteAdapter(Configurable):
             
         # Don't want body tags in chapter html--writers add them.
         # This is primarily for epub updates.
-        retval = re.sub(r"</?body>\r?\n?","",retval)
+        retval = re.sub(r"</?body[^>]*>\r?\n?","",retval)
         
         if self.getConfig("replace_br_with_p"):
             # Apply heuristic processing to replace <br> paragraph

fanficdownloader/story.py

@@ -28,6 +28,9 @@ import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 from configurable import Configurable
 
+SPACE_REPLACE=u'\s'
+SPLIT_META=u'\,'
+
 # Create convert_image method depending on which graphics lib we can
 # load.  Preferred: calibre, PIL, none
 
@@ -221,6 +224,65 @@ langs = {
     "Devanagari":"hi",
     }
 
+class InExMatch:
+    keys = []
+    regex = None
+    match = None
+    negate = False
+
+    def  __init__(self,line):
+        if "=~" in line:
+            (self.keys,self.match) = line.split("=~")
+            self.match = self.match.replace(SPACE_REPLACE,' ')
+            self.regex = re.compile(self.match)
+        elif "!~" in line:
+            (self.keys,self.match) = line.split("!~")
+            self.match = self.match.replace(SPACE_REPLACE,' ')
+            self.regex = re.compile(self.match)
+            self.negate = True
+        elif "==" in line:
+            (self.keys,self.match) = line.split("==")
+            self.match = self.match.replace(SPACE_REPLACE,' ')
+        elif "!=" in line:
+            (self.keys,self.match) = line.split("!=")
+            self.match = self.match.replace(SPACE_REPLACE,' ')
+            self.negate = True
+        self.keys = map( lambda x: x.strip(), self.keys.split(",") )
+
+    # For conditional, only one key
+    def is_key(self,key):
+        return key == self.keys[0]
+
+    # For conditional, only one key
+    def key(self):
+        return self.keys[0]
+
+    def in_keys(self,key):
+        return key in self.keys
+
+    def is_match(self,value):
+        retval = False
+        if self.regex:
+            if self.regex.search(value):
+                retval = True
+            #print(">>>>>>>>>>>>>%s=~%s r: %s,%s=%s"%(self.match,value,self.negate,retval,self.negate != retval))
+        else:
+            retval = self.match == value
+            #print(">>>>>>>>>>>>>%s==%s r: %s,%s=%s"%(self.match,value,self.negate,retval, self.negate != retval))
+            
+        return self.negate != retval
+
+    def __str__(self):
+        if self.negate:
+            f='!'
+        else:
+            f='='
+        if self.regex:
+            s='~'
+        else:
+            s='='
+        return u'InExMatch(%s %s%s %s)'%(self.keys,f,s,self.match)
+
 class Story(Configurable):
 
     def __init__(self, configuration):
@@ -231,6 +293,7 @@ class Story(Configurable):
         except:
             self.metadata = {'version':'4.4'}
         self.replacements = []
+        self.in_ex_cludes = {}
         self.chapters = [] # chapters will be tuples of (title,html)
         self.imgurls = []
         self.imgtuples = []
@@ -241,8 +304,7 @@ class Story(Configurable):
         self.logfile=None # cheesy way to carry log file forward across update.
 
         ## Look for config parameter, split and add each to metadata field.
-        for (config,metadata) in [("extratags","extratags"),
-                                  ("extracategories","category"),
+        for (config,metadata) in [("extracategories","category"),
                                   ("extragenres","genre"),
                                   ("extracharacters","characters"),
                                   ("extraships","ships"),
@@ -252,6 +314,15 @@ class Story(Configurable):
 
         self.setReplace(self.getConfig('replace_metadata'))
 
+        in_ex_clude_list = ['include_metadata_pre','exclude_metadata_pre',
+                            'include_metadata_post','exclude_metadata_post']
+        for ie in in_ex_clude_list:
+            ies = self.getConfig(ie)
+            # print("%s %s"%(ie,ies))
+            if ies:
+                iel = []
+                self.in_ex_cludes[ie] = self.set_in_ex_clude(ies)
+
     def setMetadata(self, key, value, condremoveentities=True):
         ## still keeps < < and &
         if condremoveentities:
@@ -269,6 +340,56 @@ class Story(Configurable):
             self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
 
+        
+    ## metakey[,metakey]=~pattern
+    ## metakey[,metakey]==string
+    ## *for* part lines.  Effect only when trailing conditional key=~regexp matches
+    ## metakey[,metakey]=~pattern[&&metakey=~regexp]
+    ## metakey[,metakey]==string[&&metakey=~regexp]
+    ## metakey[,metakey]=~pattern[&&metakey==string]
+    ## metakey[,metakey]==string[&&metakey==string]
+    def set_in_ex_clude(self,setting):
+        dest = []
+        # print("set_in_ex_clude:"+setting)
+        for line in setting.splitlines():
+            if line:
+                (match,condmatch)=(None,None)
+                if "&&" in line:
+                    (line,conditional) = line.split("&&")
+                    condmatch = InExMatch(conditional)
+                match = InExMatch(line)
+                dest.append([match,condmatch])
+        return dest
+              
+    def do_in_ex_clude(self,which,value,key):
+        if value and which in self.in_ex_cludes:
+            include = 'include' in which
+            keyfound = False
+            found = False
+            for (match,condmatch) in self.in_ex_cludes[which]:
+                keyfndnow = False
+                if match.in_keys(key):
+                    # key in keys and either no conditional, or conditional matched
+                    if condmatch == None or condmatch.is_key(key):
+                        keyfndnow = True
+                    else:
+                        condval = self.getMetadata(condmatch.key())
+                        keyfndnow = condmatch.is_match(condval)
+                    keyfound |= keyfndnow
+                        # print("match:%s %s\ncondmatch:%s %s\n\tkeyfound:%s\n\tfound:%s"%(
+                        #         match,value,condmatch,condval,keyfound,found))
+                    if keyfndnow:
+                        found = isinstance(value,basestring) and match.is_match(value)
+                    if found:
+                        # print("match:%s %s\n\tkeyfndnow:%s\n\tfound:%s"%(
+                        #         match,value,keyfndnow,found))
+                        if not include:
+                            value = None
+                        break
+            if include and keyfound and not found:
+                value = None
+        return value
+        
 
     ## Two or three part lines.  Two part effect everything.
     ## Three part effect only those key(s) lists.
@@ -297,10 +418,13 @@ class Story(Configurable):
                 # A way to explicitly include spaces in the
                 # replacement string.  The .ini parser eats any
                 # trailing spaces.
-                replacement=replacement.replace('\s',' ')
+                replacement=replacement.replace(SPACE_REPLACE,' ')
                 self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
 
     def doReplacements(self,value,key):
+        value = self.do_in_ex_clude('include_metadata_pre',value,key)
+        value = self.do_in_ex_clude('exclude_metadata_pre',value,key)
+
         for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
             if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
@@ -311,7 +435,20 @@ class Story(Configurable):
                     doreplace = condval != None and condregexp.search(condval)
 
                 if doreplace:
-                    value = regexp.sub(replacement,value)
+                    # split into more than one list entry if list and
+                    # SPLIT_META present in replacement string.  Split
+                    # first, then regex sub.
+                    if self.isList(key) and SPLIT_META in replacement:
+                        repllist = replacement.split(SPLIT_META)
+                        for repl in repllist[1:]:
+                            self.addToList(key,regexp.sub(repl,value))
+                        value = regexp.sub(repllist[0],value)
+                    else:            
+                        value = regexp.sub(replacement,value)
+                    
+
+        value = self.do_in_ex_clude('include_metadata_post',value,key)
+        value = self.do_in_ex_clude('exclude_metadata_post',value,key)
         return value
 
     def getMetadataRaw(self,key):
@@ -326,7 +463,7 @@ class Story(Configurable):
             return value
 
         if self.isList(key):
-            join_string = self.getConfig("join_string_"+key,u", ").replace('\s',' ')
+            join_string = self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ')
             value = join_string.join(self.getList(key, removeallentities, doreplacements=True))
             if doreplacements:
                 value = self.doReplacements(value,key+"_LIST")
@@ -377,7 +514,7 @@ class Story(Configurable):
                     auth=removeAllEntities(auth)
 
                 htmllist.append(linkhtml%('author',aurl,auth))
-            join_string = self.getConfig("join_string_authorHTML",u", ").replace('\s',' ')
+            join_string = self.getConfig("join_string_authorHTML",u", ").replace(SPACE_REPLACE,' ')
             self.setMetadata('authorHTML',join_string.join(htmllist))
         else:
             self.setMetadata('authorHTML',linkhtml%('author',self.getMetadata('authorUrl', removeallentities, doreplacements),
@@ -409,7 +546,7 @@ class Story(Configurable):
                     v=removeAllEntities(v)
 
                 htmllist.append(linkhtml%(k,url,v))
-            join_string = self.getConfig("join_string_"+k+"HTML",u", ").replace('\s',' ')
+            join_string = self.getConfig("join_string_"+k+"HTML",u", ").replace(SPACE_REPLACE,' ')
             self.setMetadata(k+'HTML',join_string.join(htmllist))
 
         for k in self.getValidMetaList():
@@ -573,7 +710,13 @@ class Story(Configurable):
             imgurl = url
         else:
             parsedUrl = urlparse.urlparse(parenturl)
-            if url.startswith("/") :
+            if url.startswith("//") :
+                imgurl = urlparse.urlunparse(
+                    (parsedUrl.scheme,
+                     '',
+                     url,
+                     '','',''))
+            elif url.startswith("/") :
                 imgurl = urlparse.urlunparse(
                     (parsedUrl.scheme,
                      parsedUrl.netloc,

index.html

@@ -46,13 +46,6 @@
       {{yourfile}}
       <!--			</div> -->
 
-          <h3>fanfiction.net / fimfiction.net</h3>
-          <p>
-            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
-            are working again.  I'd ask that users limit the number of
-            stories they download from those sites, thanks.
-          </p>
-	  
       {% if authorized %}
       <form action="/fdown" method="post">
 	<div id='urlbox'>
@@ -62,12 +55,17 @@
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
 	  <h3>Changes:</h3>
+	  <p>
+            Now supporting over 100 different sites!  Thanks, cryzed, for pushing us over the top.
+	  </p>
 	  <p>
 	    <ul>
-              <li>Fix for possible divide-by-zero in replace_br_with_p.</li>
+              <li>New site: nocturnal-light.net -- Thanks, cryzed!</li>
+              <li>New site: fanfic.hu (Hungarian language) -- Thanks, cryzed!</li>
+              <li>New site: fanfiction.csodaidok.hu (Hungarian language) -- Thanks, cryzed!</li>
+              <li>Improvements &amp; fixes for recently added sites -- Thanks, cryzed!</li>
 	    </ul>
 	  </p>
-          
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -76,7 +74,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-90.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-5-03.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -154,6 +154,11 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
 
+## How long to wait for each HTTP connection to finish.  Longer times
+## are better for sites that are slow to respond.  Shorter times
+## prevent excessive wait when your network or the site is down.
+connect_timeout:60.0
+
 ## Use regular expressions to find and replace (or remove) metadata.
 ## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
 ## etc.  See http://docs.python.org/library/re.html (look for re.sub)
@@ -181,7 +186,32 @@ extratags: FanFiction
 # .*-Centered=>
 # characters=>Sam W\.=>Sam Witwicky&&category=>Transformers
 # characters=>Sam W\.=>Sam Winchester&&category=>Supernatural
- 
+
+## Include/Exclude metadata
+##
+## You can use the include/exclude metadata features to either limit
+## the values of particular metadata lists to specific values or to
+## exclude specific values.  Further, you can conditionally apply each
+## line depending on other metadata, use exact strings or regular
+## expressions(regex) to match values, and negate matches.
+##
+## The settings are:
+## include_metadata_pre
+## exclude_metadata_pre
+## include_metadata_post
+## exclude_metadata_post
+##
+## The form of each line is:
+## metakey[,metakey]==exactvalue
+## metakey[,metakey]=~regex
+## metakey[,metakey]==exactvalue&&conditionalkey==exactcondvalue
+## metakey[,metakey]=~regex&&conditionalkey==exactcondvalue
+## metakey[,metakey]==exactvalue&&conditionalkey=~condregex
+##
+## This is fairly complicated, so it's documented on its own wiki
+## page:
+## https://code.google.com/p/fanficdownloader/wiki/InExcludeMetadataFeature
+
 ## Some readers don't show horizontal rule (<hr />) tags correctly.
 ## This replaces them all with a centered '* * *'.  (Note centering
 ## doesn't work on some devices either.)
@@ -202,6 +232,9 @@ extratags: FanFiction
 ## Make sure to keep at least one space at the start of each line and
 ## to escape % to %%, if used.
 ## template => regexp to match => GC Setting to use.
+## To use this, make sure you go to the Generate Cover tab in FFDL
+## config and check 'Allow generate_cover_settings from personal.ini
+## to override'
 #generate_cover_settings:
 # ${category} => Buffy:? [tT]he Vampire Slayer => BuffyCover
 # ${category} => Star Trek => StarTrekCover
@@ -259,6 +292,10 @@ chapter_title_add_pattern:${index}. ${title}
 ## anthologies.
 anthology_title_pattern:${title} Anthology
 
+## Add tag(s) for anthology (series) books.  Set to empty to not add
+## any anthology tags.
+anthology_tags:Anthology
+
 ## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
 ## on '/', so use replace_metadata to change separator first if
 ## needed. Something like: ships=>[ ]*(/|&amp;|&)[ ]*=>/ You can use
@@ -537,6 +574,12 @@ include_in_freefromtags:freeformtags
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
+## AO3 chapters can include several different types of notes.  We've
+## traditional included them all in the chapter text, but this allows
+## you to customize which you include.  Copy this parameter to your
+## personal.ini and list the ones you don't want.
+#exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
+
 [ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -559,6 +602,29 @@ extracategories:The Sentinel
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[bloodshedverse.com]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:warnings,reviews
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
+## Strips links found in the story text
+## Specific to bloodshedverse.com
+strip_text_links:true
+
 [bloodties-fans.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -718,6 +784,38 @@ extracategories:Harry Potter
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
+[fanfiction.csodaidok.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-2,auto
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,challenge
+reviews_label:Reviews
+challenge_label:Challenge
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+[fanfic.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfiction.mugglenet.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -768,6 +866,14 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [fictionpad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -924,6 +1030,17 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[nocturnal-light.net]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:readings,reviews
+readings_label:Readings
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
 [occlumency.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1014,6 +1131,16 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[spikeluver.com]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:warnings,reviews
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
 [stargate-atlantis.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
@@ -1112,6 +1239,13 @@ awards_label:Awards
 
 cover_exclusion_regexp:art/.*Awards.jpg
 
+[voracity2.e-fic.com]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,readings
+reviews_label:Reviews
+readings_label:Readings
+
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1202,6 +1336,20 @@ user_agent:
 ## Change this to false to use them anyway.
 never_make_cover: true
 
+## fanfiction.net is blocking people more aggressively.  If you
+## download fewer stories less often you can likely get by with
+## reducing this sleep.
+slow_down_sleep_time:4
+
+## ffnet is sensitive to too many hits.  Users are sensitive to long
+## waits during the initial metadata collection in the foreground.
+## When used, these settings will speed up metadata downloads in the
+## foreground linearly.
+tweak_fg_sleep:true
+min_fg_sleep:1.0
+max_fg_sleep:4.0
+max_fg_sleep_at_downloads: 10
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
@@ -1210,13 +1358,20 @@ extra_valid_entries:reviews,favs,follows
 ## to be romantic pairings.
 ships_label:Pairings
 
-## Date formats used by FFDL.  Published and Update don't have time.
+## Date formats used by FFDL.  Published and Update don't usually have
+## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
 #dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
 datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
 dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 
+## ffnet used to have a tendency to send out update notices in email
+## before all their servers were showing the update on the first
+## chapter.  It generates another server request and doesn't seem to
+## be needed lately, so now default it to off.
+check_next_chapter:false
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1249,14 +1404,6 @@ extratags:
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
-[ficwad.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
 [www.fimfiction.net]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1275,6 +1422,11 @@ extra_valid_entries:reviews,favs,follows
 ## when updating to enforce accurate chapters.
 #do_update_hook:false
 
+## fimfiction.net is reported to misinterprete some BBCode with
+## blockquotes incorrectly.  This fixes those instances and defaults
+## to on, but can be switched off if it is found to cause problems.
+fix_fimf_blockquotes:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
 

status.html

@@ -51,12 +51,6 @@
 	      by {{ fic.author }} ({{ fic.format }})
 	  {% endif %}
 	  {% if fic.failure %}
-          <h3>fanfiction.net / fimfiction.net</h3>
-          <p>
-            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
-            are working again.  I'd ask that users limit the number of
-            stories they download from those sites, thanks.
-          </p>
 	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}
 	  {% if  not fic.completed and not fic.failure %}