Update CLI download zip.

Bump versions
Remove some outdated checks against calibre versions that aren't supported now.
2026-09-15 11:36:30 +08:00 · 2014-03-29 17:21:36 -05:00 · 2014-03-29 17:20:42 -05:00 · 2014-03-29 11:33:01 -05:00 · 2014-03-29 11:25:08 -05:00 · 2014-03-29 11:24:37 -05:00
33 changed files with 4226 additions and 1689 deletions
@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-86
+version: 4-4-97
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -42,8 +42,8 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
    description         = _('UI plugin to download FanFiction stories from various sites.')
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (1, 8, 02)
-    minimum_calibre_version = (0, 8, 57)
+    version             = (1, 8, 16)
+    minimum_calibre_version = (1, 13, 0)

    #: This field defines the GUI plugin class that contains all the code
    #: that actually does something. Its format is module_path:class_name
@@ -55,7 +55,7 @@ from calibre_plugins.fanfictiondownloader_plugin.dialogs \
            EditTextDialog, RejectUrlEntry)
    
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
-    import (getConfigSections, getNormalStoryURL)
+    import getConfigSections

 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
    import ( KeyboardConfigDialog, PrefsViewerDialog )
@@ -209,6 +209,7 @@ class ConfigWidget(QWidget):
        prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
        prefs['mark'] = self.basic_tab.mark.isChecked()
        prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
+        prefs['autoconvert'] = self.basic_tab.autoconvert.isChecked()
        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
        prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
@@ -248,6 +249,7 @@ class ConfigWidget(QWidget):
                #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
        prefs['gc_site_settings'] = gc_site_settings
        prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
+        prefs['gc_polish_cover'] = self.generatecover_tab.gc_polish_cover.isChecked()

        # Count Pages tab
        countpagesstats = []
@@ -422,6 +424,11 @@ class BasicTab(QWidget):
        self.showmarked.setChecked(prefs['showmarked'])
        self.l.addWidget(self.showmarked)

+        self.autoconvert = QCheckBox(_("Automatically Convert new/update books?"),self)
+        self.autoconvert.setToolTip(_("Automatically call calibre's Convert for new/update books.\nConverts to the current output format as chosen in calibre's\nPreferences->Behavior settings."))
+        self.autoconvert.setChecked(prefs['autoconvert'])
+        self.l.addWidget(self.autoconvert)
+
        gui_gb = groupbox = QGroupBox(_("GUI Options"))
        self.l = QVBoxLayout()
        groupbox.setLayout(self.l)
@@ -747,6 +754,11 @@ class GenerateCoverTab(QWidget):
        self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
        self.l.addWidget(self.allow_gc_from_ini)
            
+        self.gc_polish_cover = QCheckBox(_("Use calibre's Polish feature to inject/update the cover"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated cover into the ebook, EPUB only."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        self.l.addWidget(self.gc_polish_cover)
+            
 class CountPagesTab(QWidget):

    def __init__(self, parent_dialog, plugin_action):
@@ -32,6 +32,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.utils.config import prefs as calibre_prefs
 from calibre.utils.date import local_tz
 from calibre.library.comments import sanitize_comments_html
 from calibre.constants import config_dir as calibre_config_dir
@@ -660,13 +661,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        #print("update_dialog()")
        
        db = self.gui.current_db
-        book_list = map( self.make_book_id_only, id_list )
+        books = map( self.make_book_id_only, id_list )

-        for j, book in enumerate(book_list):
+        for j, book in enumerate(books):
            book['listorder'] = j
-            
+
        LoopProgressDialog(self.gui,
-                           book_list,
+                           books,
                           partial(self.populate_book_from_calibre_id, db=self.gui.current_db),
                           self.update_dialog_finish,
                           init_label=_("Collecting stories for update..."),
@@ -718,6 +719,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            url_list = split_text_to_urls(books)
            books = self.convert_urls_to_books(url_list)

+        ## for tweak_fg_sleep
+        options['ffnetcount']=len(filter(lambda x : x['site']=='www.fanfiction.net', books))
+
        options['version'] = self.version
        logger.debug(self.version)
        
@@ -815,6 +819,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        
        options['personal.ini'] = get_ffdl_personalini()
        adapter = get_ffdl_adapter(url,fileform)
+        # reduce foreground sleep time for ffnet when few books.
+        if 'ffnetcount' in options and \
+                adapter.getConfig('tweak_fg_sleep') and \
+                adapter.getSiteDomain() == 'www.fanfiction.net':
+            minslp = float(adapter.getConfig('min_fg_sleep'))
+            maxslp = float(adapter.getConfig('max_fg_sleep'))
+            dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+            m = (maxslp-minslp) / (dwnlds-1)
+            b = minslp - m
+            slp = min(maxslp,m*float(options['ffnetcount'])+b)
+            #print("m:%s b:%s = %s"%(m,b,slp))
+            adapter.set_sleep(slp)

        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other
@@ -841,7 +857,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        series = story.getMetadata('series')
        if not merge and series and prefs['checkforseriesurlid']:
            # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
+            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
            identicalbooks = db.search_getting_ids(searchstr, None)
            # print("searchstr:%s"%searchstr)
            # print("identicalbooks:%s"%identicalbooks)
@@ -925,13 +941,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                if len(identicalbooks) < 1:
                    # find dups
                    authlist = story.getList("author", removeallentities=True)
-                    if len(authlist) > 100 and calibre_version < (0, 8, 61):
-                        ## should be fixed from 0.8.61 on.  In the
-                        ## meantime, if it matches the title *and* first
-                        ## 100 authors, I'm prepared to assume it's a
-                        ## match.
-                        logger.debug("reduce author list to 100 only when calibre < 0.8.61")
-                        authlist = authlist[:100]
                    mi = MetaInformation(story.getMetadata("title", removeallentities=True),
                                         authlist)
                    identicalbooks = db.find_identical_books(mi)
@@ -1138,6 +1147,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                func, args=args,
                description=desc)
        
+        self.gui.jobs_pointer.start()
        self.gui.status_bar.show_message(_('Starting %d FanFictionDownLoads')%len(book_list),3000)

    def update_books_loop(self,book,db=None,
@@ -1232,6 +1242,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            cp_plugin = self.gui.iactions['Count Pages']
            cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])

+        if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
+            self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
+            self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
+
    def download_list_completed(self, job, options={},merge=False):
        if job.failed:
            self.gui.job_exception(job, dialog_title='Failed to Download Stories')
@@ -1335,7 +1349,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        bad_list = sorted(bad_list,key=lambda x : x['listorder'])
        
        self.gui.status_bar.show_message(_('Merging %s books.')%total_good)
-
        
        existingbook = None
        if 'mergebook' in options:
@@ -1393,7 +1406,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if prefs['mark'] or (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
            self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.')%len(book_list))
-            label = custom_columns[prefs['errorcol']]['label']
+            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
+                label = custom_columns[prefs['errorcol']]['label']
+            else:
+                label = None
            LoopProgressDialog(self.gui,
                               book_list,
                               partial(self.update_error_column_loop, db=self.gui.current_db, label=label),
@@ -1402,12 +1418,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                               win_title=_("Update calibre for BAD FanFiction stories"),
                               status_prefix=_("Updated"))

-    def update_error_column_loop(self,book,db=None,label='errorcol'):
-        if book['calibre_id']:
-            custom_columns = self.gui.library_view.model().custom_columns
-            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
-                logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
-                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+    def update_error_column_loop(self,book,db=None,label=None):
+        if book['calibre_id'] and label:
+            logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)

    def add_book_or_update_format(self,book,options,prefs,mi=None):
        db = self.gui.current_db
@@ -1436,8 +1450,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
            for fmt in fmts:
                if fmt != formmapping[options['fileform']]:
-                    logger.debug("remove f:"+fmt)
+                    logger.debug("deleteotherforms remove f:"+fmt)
                    db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+        elif prefs['autoconvert']:
+            ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+            ## the format is already there.
+            fmt = calibre_prefs['output_format']
+            # delete if there, but not if the format we just made.
+            if fmt != formmapping[options['fileform']] and \
+                    db.has_format(book_id,fmt,index_is_id=True):
+                logger.debug("autoconvert remove f:"+fmt)
+                db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                

        return book_id

@@ -1545,8 +1569,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )

                    if meta not in book['all_metadata']:
-                        logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
-                        continue
+                        # if double quoted, use as a literal value.
+                        if meta[0] == '"' and meta[-1] == '"':
+                            val = meta[1:-1]
+                            logger.debug("No metadata value for %s, setting custom column(%s) literally to %s."%(meta,custcol,val))
+                        else:
+                            logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
+                            continue
+                    else:
+                        val = book['all_metadata'][meta]    
                    
                    if custcol not in custom_columns:
                        continue
@@ -1558,12 +1589,19 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
                            if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
                                # re-split list, strip commas, convert to floats, sum up.
-                                val = sum([ float(x.replace(",","")) for x in book['all_metadata'][meta].split(", ") ])
+                                val = sum([ float(x.replace(",","")) for x in val.split(", ") ])
                            else:
-                                val = unicode(book['all_metadata'][meta]).replace(",","")
+                                val = unicode(val).replace(",","")
                        else:
-                            val = book['all_metadata'][meta]
+                            val = val
                        if val != '':
+                            if coldef['datatype'] == 'bool':
+                                if val.lower() in ('t','true','1','yes','y'):
+                                    val = True
+                                elif val.lower() in ('f','false','0','no','n'):
+                                    val = False
+                                else:
+                                    val = None # for tri-state 'booleans'. Yes/No/Null
                            db.set_custom(book_id, val, label=label, commit=False)

                    if flag == 'a':
@@ -1577,8 +1615,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        except:
                            pass

-                        if book['all_metadata'][meta]:
-                            vallist = [book['all_metadata'][meta]]
+                        if val:
+                            vallist = [val]
                            
                        db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)

@@ -1587,30 +1625,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if 'authorUrl' in book['all_metadata']:
            authurls = book['all_metadata']['authorUrl'].split(", ")
            authorlist = [ a.replace('&',';') for a in book['author'] ]
-            if hasattr(db, 'new_api'): # new_api starts in calibre 1.0.0
-                authorids = db.new_api.get_item_ids('authors',authorlist)
-                authordata = db.new_api.author_data(authorids.values())
-                # print("\n\nauthorids:%s"%authorids)
-                # print("authordata:%s"%authordata)
+            authorids = db.new_api.get_item_ids('authors',authorlist)
+            authordata = db.new_api.author_data(authorids.values())
+            # print("\n\nauthorids:%s"%authorids)
+            # print("authordata:%s"%authordata)

-                author_id_to_link_map = dict()
-                for i, author in enumerate(authorlist):
-                    author_id_to_link_map[authorids[author]] = authurls[i]
+            author_id_to_link_map = dict()
+            for i, author in enumerate(authorlist):
+                author_id_to_link_map[authorids[author]] = authurls[i]

-                # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
-                db.new_api.set_link_for_authors(author_id_to_link_map)
-            else:
-                # keep for pre-calibre 1.0.0
-                for i, auth in enumerate(authorlist):
-                    #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                    autid=db.get_author_id(auth)
-                    db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
+            # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
+            db.new_api.set_link_for_authors(author_id_to_link_map)
+            
        db.commit()

        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):

-            logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
+            #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
            
            # force a refresh if generating cover so complex composite
            # custom columns are current and correct
@@ -1653,6 +1684,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                realmi = db.get_metadata(book_id, index_is_id=True)
                gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)

+                if prefs['gc_polish_cover'] and \
+                        options['fileform'] == "epub":
+                    # set cover inside epub from calibre's polish feature
+                    from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+                    from calibre.utils.logging import Log
+                    from collections import namedtuple
+
+                    # Couldn't find a better way to get the cover path.
+                    cover_path = os.path.join(db.library_path, db.path(book_id, index_is_id=True), 'cover.jpg')
+                    data = {'cover':cover_path}
+                    #print("cover_path:%s"%cover_path)
+                    opts = ALL_OPTS.copy()
+                    opts.update(data)
+                    O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+                    opts = O(**opts)
+                
+                    log = Log(level=Log.DEBUG)
+                    outfile = db.format_abspath(book_id, formmapping[options['fileform']], index_is_id=True)
+                    #print("polish cover outfile:%s"%outfile)
+                    polish({outfile:outfile}, opts, log, logger.info)
+                
+
    def get_clean_reading_lists(self,lists):
        if lists == None or lists.strip() == "" :
            return []
@@ -1741,6 +1794,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        book['end'] = None
        book['comment'] = '' # note this is a comment on the d/l or update.
        book['url'] = ''
+        book['site'] = ''
        book['added'] = False
        return book
    
@@ -1817,13 +1871,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            book['status'] = _('Not Found')
        else:
            # get normalized url or None.
-            book['url'] = self.is_good_downloader_url(url)
-            if book['url'] == None:
+            urlsitetuple = adapters.getNormalStoryURLSite(url)
+            if urlsitetuple == None:
                book['url'] = url
                book['comment'] = _("URL is not a valid story URL.")
                book['good'] = False
                book['icon']='dialog_error.png'
                book['status'] = _('Bad URL')
+            else:
+                (book['url'],book['site'])=urlsitetuple
    
    def get_story_url(self, db, book_id=None, path=None):
        if book_id == None:
@@ -93,16 +93,13 @@ def do_download_worker(book_list, options,
        logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
        logger.info(job.details)

+
+
        if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            logger.info("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
-            logger.info("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\nSuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                      filter(lambda x: x['good'], book_list) ] ) ) )
+            logger.info("\nUnsuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
            break

    server.close()
@@ -34,6 +34,7 @@ default_prefs['suppressauthorsort'] = False
 default_prefs['suppresstitlesort'] = False
 default_prefs['mark'] = False
 default_prefs['showmarked'] = False
+default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'
@@ -56,6 +57,7 @@ default_prefs['addtolistsonread'] = False
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
+default_prefs['gc_polish_cover'] = False

 default_prefs['countpagesstats'] = []

@@ -182,6 +182,11 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5

+## How long to wait for each HTTP connection to finish.  Longer times
+## are better for sites that are slow to respond.  Shorter times
+## prevent excessive wait when your network or the site is down.
+connect_timeout:60.0
+
 ## For use only with stand-alone CLI version--run a command on the
 ## generated file after it's produced.  All of the titlepage_entries
 ## values are available, plus output_filename.
@@ -538,6 +543,13 @@ extratags: FanFiction,Testing,HTML
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## archiveofourown.org stories allow chapters to be added out of
+## order.  So the newest chapter may not be the last one.  FFDL update
+## doesn't like that.  If do_update_hook is uncommented and set true,
+## the adapter will discard all existing chapters from the newest one
+## on when updating to enforce accurate chapters.
+#do_update_hook:false
+
 ## AO3 adapter defines a few extra metadata entries.
 ## If there's ever more than 4 series, add series04,series04Url etc.
 extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
@@ -561,12 +573,13 @@ include_in_freefromtags:freeformtags
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories

-[ashwinder.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Severus Snape,Hermione Granger
-extraships:Severus Snape/Hermione Granger
+## AO3 chapters can include several different types of notes.  We've
+## traditional included them all in the chapter text, but this allows
+## you to customize which you include.  Copy this parameter to your
+## personal.ini and list the ones you don't want.
+#exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes

+[ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -574,6 +587,11 @@ extraships:Severus Snape/Hermione Granger
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Hermione Granger
+extraships:Severus Snape/Hermione Granger
+
 [asr3.slashzone.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Sentinel
@@ -584,9 +602,6 @@ extracategories:The Sentinel
 #is_adult:true

 [bloodties-fans.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Blood Ties
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -599,6 +614,9 @@ extracategories:Blood Ties
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Blood Ties
+
 [buffynfaith.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Buffy: The Vampire Slayer
@@ -609,9 +627,6 @@ extracategories:Buffy: The Vampire Slayer
 #is_adult:true

 [castlefans.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Castle
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -624,6 +639,9 @@ extracategories:Castle
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Castle
+
 [chaos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -636,10 +654,23 @@ cover_exclusion_regexp:/images/.*?ribbon.gif

 [dark-solace.org]
 ## Site dedicated to these categories/characters/ships
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Buffy, Spike
 extraships:Spike/Buffy

+[dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -652,24 +683,11 @@ extraships:Spike/Buffy
 ## personal.ini, not defaults.ini.
 #is_adult:true

-[dramione.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extracharacters:Draco Malfoy,Hermione Granger
 extraships:Draco Malfoy/Hermione Granger

-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
@@ -712,22 +730,19 @@ cliches_label:Character Cliches
 extracategories:Lord of the Rings

 [erosnsappho.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif

 [fanfiction.mugglenet.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -740,10 +755,10 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfic.potterheadsanonymous.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -756,16 +771,19 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfiction.portkey.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extraships:Harry Potter/Hermione Granger

-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [fanfiction.tenhawkpresents.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -810,18 +828,26 @@ dislikes_label:Dislikes
 ## Clear FanFiction from defaults, site is original fiction.
 extratags:

-extra_valid_entries:size,universe,codes
-#extra_titlepage_entries:size,universe,codes
+extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
+#extra_titlepage_entries:size,universeHTML,codes,notice

 size_label:Size
 universe_label:Universe
+universeUrl_label:Universe URL
+universeHTML_label:Universe
 codes_label:Codes
+notice_label:Notice
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:universe
+
+## storiesonline.net stories can be in a series or a universe, but not
+## both.  By default, universe will be populated in 'series' with
+## index=0
+universe_as_series: true

 [grangerenchanted.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Hermione Granger
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -834,6 +860,10 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Hermione Granger
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:read,reviews
@@ -870,18 +900,15 @@ extracharacters:Kirk,Spock
 extraships:Kirk/Spock

 [lumos.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[merlinfic.dtwins.co.uk]
 ## Site dedicated to these categories/characters/ships
-extracategories:Merlin
+extracategories:Harry Potter

+[merlinfic.dtwins.co.uk]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -894,6 +921,9 @@ extracategories:Merlin
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Merlin
+
 [national-library.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:West Wing
@@ -905,24 +935,32 @@ extracategories:NCIS
 [netraptor.org]

 [nfacommunity.com]
-## Site dedicated to these categories/characters/ships
-extracategories:NCIS
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
 [nha.magical-worlds.us]
 ## Site dedicated to these categories/characters/ships
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow

 [occlumency.sycophanthex.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extracharacters:Severus Snape

+[onedirectionfanfiction.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -930,10 +968,15 @@ extracharacters:Severus Snape
 #username:YourName
 #password:yourpassword

-[onedirectionfanfiction.com]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:One Direction

+[pommedesang.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -946,40 +989,27 @@ extracategories:One Direction
 ## personal.ini, not defaults.ini.
 #is_adult:true

-[pommedesang.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Anita Blake Vampire Hunter 

-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+[ponyfictionarchive.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[ponyfictionarchive.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic

+[pretendercentre.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[pretendercentre.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Pretender

-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [samdean.archive.nu]
 ## Site dedicated to these categories/characters/ships
 extracategories:Supernatural
@@ -987,9 +1017,6 @@ extracharacters:Sam,Dean
 extraships:Sam/Dean

 [scarhead.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1002,6 +1029,9 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1032,9 +1062,6 @@ extracharacters:Draco Malfoy,Harry Potter
 extraships:Harry Potter/Draco Malfoy

 [thehookupzone.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Criminal Minds
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1047,6 +1074,9 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Criminal Minds
+
 [themasque.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1061,9 +1091,6 @@ extracategories:Criminal Minds
 #is_adult:true

 [thequidditchpitch.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1076,19 +1103,19 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true

-[tokra.fandomnet.com]
 ## Site dedicated to these categories/characters/ships
-extracategories:Stargate: SG-1
+extracategories:Harry Potter

+[tokra.fandomnet.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[trekiverse.org]
 ## Site dedicated to these categories/characters/ships
-extracategories:Star Trek
+extracategories:Stargate: SG-1

+[trekiverse.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1101,26 +1128,24 @@ extracategories:Star Trek
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+
 extra_valid_entries:awards
 awards_label:Awards

 cover_exclusion_regexp:art/.*Awards.jpg

 [www.adastrafanfic.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Star Trek
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[www.dracoandginny.com]
 ## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Draco Malfoy,Ginny Weasley
-extraships:Draco Malfoy/Ginny Weasley
+extracategories:Star Trek

+[www.dracoandginny.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1133,14 +1158,16 @@ extraships:Draco Malfoy/Ginny Weasley
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Ginny Weasley
+extraships:Draco Malfoy/Ginny Weasley
+
 [www.thealphagate.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1

 [www.checkmated.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1148,6 +1175,9 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.destinysgateway.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1155,11 +1185,6 @@ extracategories:Harry Potter
 #is_adult:true

 [www.dokuga.com]
-## Site dedicated to these categories/characters/ships
-extracategories:InuYasha
-extracharacters:Sesshoumaru,Kagome
-extraships:Sesshoumaru/Kagome
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1167,6 +1192,11 @@ extraships:Sesshoumaru/Kagome
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:Sesshoumaru,Kagome
+extraships:Sesshoumaru/Kagome
+
 [www.dotmoon.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1196,6 +1226,11 @@ user_agent:
 ## Change this to false to use them anyway.
 never_make_cover: true

+## fanfiction.net is blocking people more aggressively.  If you
+## download fewer stories less often you can likely get by with
+## reducing this sleep.
+slow_down_sleep_time:4
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
@@ -1204,13 +1239,20 @@ extra_valid_entries:reviews,favs,follows
 ## to be romantic pairings.
 ships_label:Pairings

-## Date formats used by FFDL.  Published and Update don't have time.
+## Date formats used by FFDL.  Published and Update don't usually have
+## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
 #dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
 datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
 dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S

+## ffnet used to have a tendency to send out update notices in email
+## before all their servers were showing the update on the first
+## chapter.  It generates another server request and doesn't seem to
+## be needed lately, so now default it to off.
+check_next_chapter:false
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1222,14 +1264,14 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 [www.ficbook.net]

 [www.fictionalley.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## fictionally.org storyIds are not unique.  Combine with authorId.
 output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}

@@ -1255,6 +1297,23 @@ extra_valid_entries:reviews,favs,follows
 #password:yourpassword

 [www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fimfiction.net stories can be locked requiring individual
+## passwords.  If fail_on_password is set, the downloader will fail
+## when a password is required rather than prompting every time.
+#fail_on_password: false
+
+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic

@@ -1285,55 +1344,33 @@ keep_in_order_groupsUrl:true
 ## make entryHTML.
 make_linkhtml_entries:prequel,sequels,groups

+[www.harrypotterfanfiction.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-## fimfiction.net stories can be locked requiring individual
-## passwords.  If fail_on_password is set, the downloader will fail
-## when a password is required rather than prompting every time.
-#fail_on_password: false
-
-## fimfiction.net stories allow chapters to be added out of order.  So
-## the newest chapter may not be the last one.  FFDL update doesn't
-## like that.  If do_update_hook is uncommented and set true, the
-## adapter will discard all existing chapters from the newest one on
-## when updating to enforce accurate chapters.
-#do_update_hook:false
-
-[www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter

-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.henneth-annun.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Hobbit

 [www.hpfandom.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.hpfanficarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter

 [www.ik-eternal.net]
-## Site dedicated to these categories/characters/ships
-extracategories:InuYasha
-extracharacters:InuYasha,Kagome
-extraships:InuYasha/Kagome
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1346,30 +1383,35 @@ extraships:InuYasha/Kagome
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:InuYasha,Kagome
+extraships:InuYasha/Kagome
+
 [www.jlaunlimited.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:JLA

 [www.libraryofmoria.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Lord of the Rings
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [www.mediaminer.org]

 [www.midnightwhispers.ca]
-## Site dedicated to these categories/characters/ships
-extracategories:Queer as Folk
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
@@ -1384,10 +1426,6 @@ extracategories:CSI
 extraships:Nick Stokes/Greg Sanders

 [www.phoenixsong.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extraships:Harry Potter/Ginny Weasley
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1398,9 +1436,13 @@ extraships:Harry Potter/Ginny Weasley
 ## phoenixsong.net, oddly, can have high rated chapters (login
 ## required) in the middle of a lower rated story.  Use this to force
 ## FFDL to always login to phoenixsong.net so those stories download
-## correctly.
+## correctly.  If you have a login, this is recommended.
 #force_login:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Ginny Weasley
+
 [www.potionsandsnitches.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1423,17 +1465,15 @@ extracategories:Psych
 #is_adult:true

 [www.qaf-fic.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Queer as Folk
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 [www.restrictedsection.org]
-extracategories:Harry Potter
-extragenres:Erotica
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1441,20 +1481,21 @@ extragenres:Erotica
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extragenres:Erotica
+
 [www.scarvesandcoffee.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Glee
 extracharacters:Kurt Hummel,Blaine Anderson

-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.simplyundeniable.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1462,15 +1503,18 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword

-[www.sinful-desire.org]
 ## Site dedicated to these categories/characters/ships
-extracategories:Supernatural
+extracategories:Harry Potter

+[www.sinful-desire.org]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+
 [www.siye.co.uk]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1478,25 +1522,26 @@ extracharacters:Harry Potter,Ginny Weasley
 extraships:Harry Potter/Ginny Weasley

 [www.squidge.org/peja]
-# www.squidge.org/peja calls it Fandom <shrug>
+## www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
-# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+
+## Remove numWords -- www.squidge.org/peja word counts are inaccurate
 titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description

 [www.squidge.org/peja:txt]
-## Add URLs since there aren't links.
-# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+## Add URLs since there aren't links and remove numWords --
+## www.squidge.org/peja word counts are inaccurate
 titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description

 [www.storiesofarda.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Lord of the Rings
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [www.thepetulantpoetess.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1539,9 +1584,6 @@ collect_series: false
 extracategories:Twilight

 [www.twilighted.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Twilight
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1549,13 +1591,13 @@ extracategories:Twilight
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## twilighted.net (ab)uses series as personal reading lists.
 collect_series: false

 [www.twiwrite.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Twilight
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1563,20 +1605,23 @@ extracategories:Twilight
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## twiwrite.net (ab)uses series as personal reading lists.
 collect_series: false

 [www.walkingtheplank.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Severus Snape,Harry Potter
-extraships:Severus Snape/Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Harry Potter
+extraships:Severus Snape/Harry Potter
+
 [www.whofic.com]

 [www.wizardtales.net]
@@ -1598,14 +1643,14 @@ extracategories:X-Men Movie
 extracharacters:Wolverine,Rogue

 [www.wraithbait.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Stargate: Atlantis
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For
@@ -270,26 +270,24 @@ def main(argv,
           elif chaptercount == 0:
               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
           else:
-               if not options.metaonly:
+               # update now handled by pre-populating the old
+               # images and chapters in the adapter rather than
+               # merging epubs.
+               (url,
+                chaptercount,
+                adapter.oldchapters,
+                adapter.oldimgs,
+                adapter.oldcover,
+                adapter.calibrebookmark,
+                adapter.logfile) = get_update_data(output_filename)

-                   # update now handled by pre-populating the old
-                   # images and chapters in the adapter rather than
-                   # merging epubs.
-                   (url,
-                    chaptercount,
-                    adapter.oldchapters,
-                    adapter.oldimgs,
-                    adapter.oldcover,
-                    adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(output_filename)
+               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)

-                   print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+               if not (options.update and chaptercount == urlchaptercount) \
+                       and adapter.getConfig("do_update_hook"):
+                   chaptercount = adapter.hookForUpdates(chaptercount)

-                   if not (options.update and chaptercount == urlchaptercount) \
-                           and adapter.getConfig("do_update_hook"):
-                       chaptercount = adapter.hookForUpdates(chaptercount)
-
-                   writeStory(configuration,adapter,"epub")
+               writeStory(configuration,adapter,"epub")
                   
       else:
           # regular download
@@ -122,6 +122,7 @@ import adapter_scarheadnet
 import adapter_fictionpadcom
 import adapter_storiesonlinenet
 import adapter_trekiverseorg
+import adapter_literotica

 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -145,6 +146,13 @@ for x in imports():
            __domain_map[site]=cls

 def getNormalStoryURL(url):
+    r = getNormalStoryURLSite(url)
+    if r:
+        return r[0]
+    else:
+        return None
+
+def getNormalStoryURLSite(url):
    if not getNormalStoryURL.__dummyconfig:
        getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
    # pulling up an adapter is pretty low over-head.  If
@@ -152,10 +160,11 @@ def getNormalStoryURL(url):
    try:
        adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
        url = adapter.url
+        site = adapter.getSiteDomain()
        del adapter
-        return url
+        return (url,site)
    except:
-        return None;
+        return None

 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None
@@ -192,7 +201,7 @@ def getConfigSectionFor(url):

 def getClassFor(url):
    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
    if not fixedurl.startswith("http"):
        fixedurl = "http://%s"%url
    ## remove any trailing '#' locations.
@@ -77,14 +77,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'archiveofourown.org'

-    @classmethod
+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"

    def getSiteURLPattern(self):
        # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
        # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
+        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
        
    ## Login
    def needToLoginCheck(self, data):
@@ -177,19 +177,28 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                self.story.addToList('authorUrl','http://'+self.host+a['href'])
                self.story.addToList('author',a.text)

+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on AO3 it's
+        # possible for authors to insert new chapters out-of-order or
+        # change the dates of earlier ones by editing them--That WILL
+        # break epub update.
        # Find the chapters:
        chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
        self.story.setMetadata('numChapters',len(chapters))
        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        for x in range(0,len(chapters)):
-            # just in case there's tags, like <i> in chapter titles.
-            chapter=chapters[x]
-            if len(chapters)==1:
-                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
-            else:
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapters[0]['href']+addurl))
+        else:
+            for index, chapter in enumerate(chapters):
+                # strip just in case there's tags, like <i> in chapter titles.
                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
-
-
+                # (2013-09-21)
+                date = stripHTML(chapter.findNext('span'))[1:-1]
+                chapterDate = makeDate(date,self.dateformat)
+                if newestChapter == None or chapterDate > newestChapter:
+                    newestChapter = chapterDate
+                    self.newestChapterNum = index

        a = metasoup.find('blockquote',{'class':'userstuff'})
        if a != None:
@@ -299,6 +308,12 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                    self.setSeries(series_name, series_index)
                    self.story.setMetadata('seriesUrl',series_url)

+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
    # grab the text for an individual chapter.
    def getChapterText(self, url):
        logger.debug('Getting chapter text from: %s' % url)
@@ -306,25 +321,31 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
        data = self._fetchUrl(url)
        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-		
-        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-        if headnotes != None:
-            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+
+        exclude_notes=self.getConfigList('exclude_notes')
+
+        if 'authorheadnotes' not in exclude_notes:
+            headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
            if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(headnotes)
+                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+                if headnotes != None:
+                    chapter.append("<b>Author's Note:</b>")
+                    chapter.append(headnotes)
        
-        chapsumm = soup.find('div', {'id' : "summary"})
-        if chapsumm != None:
-            chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
-            chapter.append(chapsumm)
-        chapnotes = soup.find('div', {'id' : "notes"})
-        if chapnotes != None:
-            chapnotes = chapnotes.find('blockquote')
+        if 'chaptersummary' not in exclude_notes:
+            chapsumm = soup.find('div', {'id' : "summary"})
+            if chapsumm != None:
+                chapsumm = chapsumm.find('blockquote')
+                chapter.append("<b>Summary for the Chapter:</b>")
+                chapter.append(chapsumm)
+                
+        if 'chapterheadnotes' not in exclude_notes:
+            chapnotes = soup.find('div', {'id' : "notes"})
            if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapnotes)
+                chapnotes = chapnotes.find('blockquote')
+                if chapnotes != None:
+                    chapter.append("<b>Notes for the Chapter:</b>")
+                    chapter.append(chapnotes)
 		
        text = soup.find('div', {'class' : "userstuff module"})
        chtext = text.find('h3', {'class' : "landmark heading"})
@@ -332,17 +353,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
            chtext.extract()
        chapter.append(text)
 		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-        if chapfoot != None:
-            chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
-            chapter.append(chapfoot)
+        if 'chapterfootnotes' not in exclude_notes:
+            chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+            if chapfoot != None:
+                chapfoot = chapfoot.find('blockquote')
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapfoot)
 		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        if footnotes != None:
-            footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
-            chapter.append(footnotes)
+        if 'authorfootnotes' not in exclude_notes:
+            footnotes = soup.find('div', {'id' : "work_endnotes"})
+            if footnotes != None:
+                footnotes = footnotes.find('blockquote')
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(footnotes)
 			
        if None == soup:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
@@ -87,7 +87,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
    
    def setHeader(self):
        "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
-        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
        
    @classmethod
    def getSiteExampleURLs(self):
@@ -57,6 +57,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
            ## accept m(mobile)url, but use www.
            self.origurl = self.origurl.replace("https://m.","https://www.")

+        self.opener.addheaders.append(('Referer',self.origurl))
+
    @staticmethod
    def getSiteDomain():
        return 'www.fanfiction.net'
@@ -72,6 +74,13 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
    def getSiteURLPattern(self):
        return r"https?://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"

+    def _fetchUrl(self,url):
+        time.sleep(1.0) ## ffnet(and, I assume, fpcom) tends to fail
+                        ## more if hit too fast.  This is in
+                        ## additional to what ever the
+                        ## slow_down_sleep_time setting is.
+        return BaseSiteAdapter._fetchUrl(self,url)
+
    def extractChapterUrlsAndMetadata(self):

        # fetch the chapter.  From that we will get almost all the
@@ -98,28 +107,31 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        if "not found. Please check to see you are not using an outdated url." in data:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)

-        try:
-            # rather nasty way to check for a newer chapter.  ffnet has a
-            # tendency to send out update notices in email before all
-            # their servers are showing the update on the first chapter.
+        if self.getConfig('check_next_chapter'):
            try:
-                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
-            # get chapter part of url.
+                ## ffnet used to have a tendency to send out update
+                ## notices in email before all their servers were
+                ## showing the update on the first chapter.  It
+                ## generates another server request and doesn't seem
+                ## to be needed lately, so now default it to off.
+                try:
+                    chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+                # get chapter part of url.
+                except:
+                    chapcount = 1
+                chapter = url.split('/',)[5]
+                tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                                self.story.getMetadata('storyId'),
+                                                chapcount+1)
+                logger.debug('=Trying newer chapter: %s' % tryurl)
+                newdata = self._fetchUrl(tryurl)
+                if "not found. Please check to see you are not using an outdated url." \
+                        not in newdata:
+                    logger.debug('=======Found newer chapter: %s' % tryurl)
+                    soup = bs.BeautifulSoup(newdata)
            except:
-                chapcount = 1
-            chapter = url.split('/',)[5]
-            tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            chapcount+1)
-            logger.debug('=Trying newer chapter: %s' % tryurl)
-            newdata = self._fetchUrl(tryurl)
-            if "not found. Please check to see you are not using an outdated url." \
-                    not in newdata:
-                logger.debug('=======Found newer chapter: %s' % tryurl)
-                soup = bs.BeautifulSoup(newdata)
-        except:
-            pass
-        
+                pass
+            
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"^/u/\d+"))
        self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -157,7 +169,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                
                
            
-        a = soup.find('a', href='http://www.fictionratings.com/')
+        a = soup.find('a', href=re.compile(r'https?://www\.fictionratings\.com/'))
        rating = a.string
        if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
            rating = rating[8:]
@@ -275,7 +287,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        return

    def getChapterText(self, url):
-        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(4.0) ## ffnet(and, I assume, fpcom) tends to fail
                        ## more if hit too fast.  This is in
                        ## additional to what ever the
                        ## slow_down_sleep_time setting is.
@@ -41,10 +41,10 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):

    @classmethod
    def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
+        return "https://www.fictionpress.com/s/1234/1/ https://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"

    def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"https?://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"

 def getClass():
    return FictionPressComSiteAdapter
@@ -101,7 +101,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        # if "/images/missing_story.png" in data:
        #     raise exceptions.StoryDoesNotExist(self.url)
        
-        if "This story has been marked as having adult content." in data:
+        if "This story has been marked as having adult content. Please click below to confirm you are of legal age to view adult material in your country." in data:
            raise exceptions.AdultCheckRequired(self.url)
        
        if self.password:
@@ -249,18 +249,20 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                    value = unicode(value)
                self.story.setMetadata(metakey, value)

-        #Sequel links and group links are each bundled into story_group_list containers.
-        #Rather than mess around examining the header text, which is outside the containers,
-        #one can tell the two link types apart by examining them directly.
+        ## Groups and sequels code from FaceDeer
        allGroupLists = soup.findAll('ul', {'id':'story_group_list'})
        for groupList in allGroupLists:
            for groupName in groupList.findAll('a', {'href':re.compile('^/group/')}):
                self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) 
                self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
-            for sequel in groupList.findAll('a', {'class':'story_link'}):
+
+        sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels")
+        if not sequelStoryHeader == None:
+            sequelContainer = sequelStoryHeader.parent.parent
+            for sequel in sequelContainer.findAll('a', {'class':'story_link'}):
                self.story.addToList("sequelsUrl", 'http://'+self.host+sequel["href"]) 
                self.story.addToList("sequels", stripHTML(sequel).replace(',', ';'))
-
+                
        #The link to the prequel is embedded in the description text, so erring
        #on the side of caution and wrapping this whole thing in a try block.
        #If anything goes wrong this probably wasn't a valid prequel link.
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-

-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,11 +37,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)

-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
        self.password = ""
        self.is_adult=False
@@ -51,7 +46,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
        if 'storyInfo' in self.story.getMetadata('storyId'):
            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
        
-        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
        
@@ -128,23 +128,22 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
        # <span class="label">Rated:</span> NC-17<br /> etc
        labels = soup.findAll('span',{'class':'label'})
        for labelspan in labels:
-            value = labelspan.nextSibling
+            val = labelspan.nextSibling
+            value = unicode('')
+            while val and not defaultGetattr(val,'class') == 'label':
+                value += unicode(val)
+                val = val.nextSibling
            label = labelspan.string
+            #print("label:%s\nvalue:%s"%(label,value))

            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,value)

            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
+                self.story.setMetadata('rating', stripHTML(value))

            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
+                self.story.setMetadata('numWords', stripHTML(value))

            if 'Categories' in label:
                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
@@ -161,13 +160,18 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                for genre in genres:
                    self.story.addToList('genre',genre.string)

+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
            if 'Warnings' in label:
                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                for warning in warnings:
                    self.story.addToList('warnings',warning.string)

            if 'Completed' in label:
-                if 'Yes' in value:
+                if 'Yes' in stripHTML(value):
                    self.story.setMetadata('status', 'Completed')
                else:
                    self.story.setMetadata('status', 'In-Progress')
@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+class LiteroticaSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        
+        self.story.setMetadata('siteabbrev','litero')
+
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        
+        self.origurl = url
+        if "http://www.i." in self.origurl:
+            ## accept m(mobile)url, but use www.
+            self.origurl = self.origurl.replace("http://www.i.","http://www.")
+
+        # normalized story URL.
+        self._setURL("http://"+self.getSiteDomain()\
+                         +"/s/"+self.story.getMetadata('storyId'))
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = '%m/%d/%y'
+        
+    @staticmethod
+    def getSiteDomain():
+        return 'www.literotica.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.literotica.com', 'www.i.literotica.com']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        #return "http://www.literotica.com/s/story-title http://www.literotica.com/stories/showstory.php?id=1234 http://www.i.literotica.com/stories/showstory.php?id=1234"
+        return "http://www.literotica.com/s/story-title"
+
+    def getSiteURLPattern(self):
+        return r"http://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+        
+        url1 = self.origurl
+        logger.debug("first page URL: "+url1)
+
+        try:
+            data1 = self._fetchUrl(url1)
+            soup1 = bs.BeautifulSoup(data1)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url1)
+            else:
+                raise e
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # author
+        a = soup1.find("span", "b-story-user-y")
+        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'])
+        self.story.setMetadata('authorUrl', a.a['href'])
+        self.story.setMetadata('author', a.text)
+
+        # get the author page
+        try:
+            dataAuth = self._fetchUrl(a.a['href'])
+            soupAuth = bs.BeautifulSoup(dataAuth)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(a.a['href'])
+            else:
+                raise e
+
+        storyLink = soupAuth.find('a', href=url1)
+
+        if storyLink is not None:
+            # pull the published date from the author page
+            # default values from single link.  Updated below if multiple chapter.
+            date = storyLink.parent.parent.findAll('td')[-1].text
+            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
+
+        # find num of pages
+        # find a "3 Pages:" string on the page and parse it
+        pgs = soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0]
+
+        # If there are multiple pages, find and request the last page
+        if "1" != pgs:
+            logger.debug("last page number: "+pgs)
+            try:
+                data2 = self._fetchUrl(url1, {'page': pgs})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    # TODO: Probably should reformat this
+                    raise exceptions.StoryDoesNotExist(url1, {'page': pgs})
+                else:
+                    raise e
+        else:
+            #If we're already on the last page, copy the soup
+            soup2 = soup1
+
+        # parse out the list of chapters
+        chaps = soup2.find('div', id='b-series')
+        if chaps:  # may be one post only
+            #self.chapterUrls = [(ch.a.text, ch.a['href']) for ch in chaps.findAll('li')]
+
+            # if there are chapters, lets pull them and title from the
+            # author page because *this* chapter is omitted from the
+            # list on the last page.
+            row = storyLink.parent.parent.previousSibling
+            while row['class'] != 'ser-ttl':
+                row = row.previousSibling
+
+            seriesTitle = stripHTML(row)
+            if seriesTitle:
+                # this regex is deliberately greedy. We want to get the biggest match before a ':'
+                self.story.setMetadata('title', re.match('(.*):[^:]*$', seriesTitle).group(1))
+            else:
+                self.story.setMetadata('title', soup1.h1.string)
+
+            # now chapter list.  Assumed oldest to newest.
+            self.chapterUrls = []
+            row = row.nextSibling
+                
+            self.story.setMetadata('datePublished',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
+            while row['class'] == 'sl':
+                # pages include full URLs.
+                self.chapterUrls.append((row.a.string,row.a['href']))
+                if not row.nextSibling:
+                    break
+                row = row.nextSibling
+
+            row = row.previousSibling
+            self.story.setMetadata('dateUpdated',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
+                
+        else:  # if one post only
+            self.chapterUrls = [(soup1.h1.string, url1)]
+            self.story.setMetadata('title', soup1.h1.string)
+
+        # normalize on first chapter URL.
+        self._setURL(self.chapterUrls[0][1])
+
+        # reset storyId to first chapter.
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+
+        self.story.setMetadata('category', soup1.find('div', 'b-breadcrumbs').findAll('a')[1].string)
+        self.story.setMetadata('description', soup1.find('meta', {'name': 'description'})['content'])
+
+        return
+
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        time.sleep(0.5)
+        data1 = self._fetchUrl(url)
+        soup1 = bs.BeautifulSoup(data1)
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        story1 = soup1.find('div', 'b-story-body-x').p
+        story1.name='div'
+        story1.append('<br>')
+        storytext = self.utf8FromSoup(url,story1)
+
+        # find num pages
+        pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0])
+        logger.debug("pages: "+str(pgs))
+
+        # get all the pages
+        for i in xrange(2, pgs+1):
+            try:
+                logger.debug("fetching page "+str(i))
+                time.sleep(0.5)
+                data2 = self._fetchUrl(url, {'page': i})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+                story2 = soup2.find('div', 'b-story-body-x').p
+                story2.name='div'
+                story2.append('<br>')
+                storytext += self.utf8FromSoup(url,story2)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        return storytext
+
+
+def getClass():
+    return LiteroticaSiteAdapter
+
+
+
@@ -163,12 +163,11 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                    self.story.setMetadata('status', 'In-Progress')

            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
+                # limit date values, there's some extra chars.
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y"))
            
            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y"))

        try:
            # Find Series name from series URL.
@@ -37,11 +37,6 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)

-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
        self.password = ""
        self.is_adult=False
@@ -147,6 +142,10 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
        a = soup.find('h1')
        self.story.setMetadata('title',stripHTML(a))
        
+        notice = soup.find('div', {'class' : 'notice'})
+        if notice:
+            self.story.setMetadata('notice',unicode(notice))
+
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"/a/\w+"))
        self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -185,23 +184,74 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
        self.story.setMetadata('size', lc2.findNext('td', {'class' : 'num'}).text)
        
        lc4 = lc2.findNext('td', {'class' : 'lc4'})        
-        
+        desc = lc4.contents[0]
+
        try:
            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
-            i = a.parent.text.split('(')[1].split(')')[0]
-            self.setSeries(stripHTML(a), i)
-            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
+            if a:
+                # if there's a number after the series name, series_contents is a two element list:
+                # [<a href="...">Title</a>, u' (2)']
+                series_contents = a.parent.contents
+                i = 0 if len(series_contents) == 1 else series_contents[1].strip(' ()')
+                seriesUrl = 'http://'+self.host+a['href']
+                self.story.setMetadata('seriesUrl',seriesUrl)
+                series_name = stripHTML(a)
+                logger.debug("Series name= %s" % series_name)
+                series_soup = bs.BeautifulSoup(self._fetchUrl(seriesUrl))
+                if series_soup:
+                    logger.debug("Retrieving Series - looking for name")
+                    series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0]
+                    logger.debug("Series name: '{0}'".format(series_name))
+                self.setSeries(series_name, i)
+                desc = lc4.contents[2]
+                # Check if series is in a universe
+                universes_soup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')  + "&type=uni")) 
+#                logger.debug("Universe page=", universes_soup)
+                if universes_soup:
+                    universes = universes_soup.findAll('div', {'class' : 'ser-box'})
+                    logger.debug("Number of Universes: %d" % len(universes))
+                    for universe in universes:
+                        logger.debug("universe.find('a')={0}".format(universe.find('a')))
+                        # The universe id is in an "a" tag that has an id but nothing else. It is the first tag.
+                        # The id is prefixed with the letter "u".
+                        universe_id = universe.find('a')['id'][1:]
+                        logger.debug("universe_id='%s'" % universe_id)
+                        universe_name = universe.find('div', {'class' : 'ser-name'}).text.partition(' ')[2]
+                        logger.debug("universe_name='%s'" % universe_name)
+                        # If there is link to the story, we have the right universe
+                        story_a = universe.find('a', {'href' : '/s/'+self.story.getMetadata('storyId')})
+                        if story_a:
+                            logger.debug("Story is in a series that is in a universe! The universe is '%s'" % universe_name)
+                            self.story.setMetadata("universe", universe_name)
+                            self.story.setMetadata('universeUrl','http://'+self.host+ '/library/universe.php?id=' + universe_id)
+                            break
        except:
            pass
        try:
            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
            if a:
                self.story.setMetadata("universe",stripHTML(a))
+                desc = lc4.contents[2]
+                # Assumed only one universe, but it does have a URL--use universeHTML
+                universe_name = stripHTML(a)
+                universeUrl = 'http://'+self.host+a['href']
+                logger.debug("Retrieving Universe - about to get page")
+                universe_soup = bs.BeautifulSoup(self._fetchUrl(universeUrl))
+                logger.debug("Retrieving Universe - have page")
+                if universe_soup:
+                    logger.debug("Retrieving Universe - looking for name")
+                    universe_name = universe_soup.find('span', {'id' : 'ptitle'}).text.partition(' &mdash;')[0]
+                    logger.debug("Universes name: '{0}'".format(universe_name))
+
+                self.story.setMetadata('universeUrl',universeUrl)
+                logger.debug("Setting universe name: '{0}'".format(universe_name))
+                self.story.setMetadata('universe',universe_name)
+                if self.getConfig("universe_as_series"):
+                    self.setSeries(universe_name, 0)
+                    self.story.setMetadata('seriesUrl',universeUrl)
        except:
            pass
-            

-        desc = lc4.contents[0]
        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
            
        for b in lc4.findAll('b'):
@@ -255,8 +305,18 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
        # some big chapters are split over several pages
        pager = div.find('span', {'class' : 'pager'})
        if pager != None:
+            a = pager.previousSibling
+            while a != None:
+                logger.debug("before pager: {0}".format(a))
+                b = a.previousSibling
+                a.extract()
+                a = b
+
            urls=pager.findAll('a')
            urls=urls[:len(urls)-1]
+            pager.extract()
+            div.contents = div.contents[2:]
+#            logger.debug(div)
                        
            for ur in urls:
                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
@@ -264,38 +324,76 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
        
                div1 = soup.find('div', {'id' : 'story'})
                
-                # appending next section
-                last=div.findAll('p')
-                next=div1.find('span', {'class' : 'conTag'}).nextSibling
-            
-                last[len(last)-1]=last[len(last)-1].append(next)
-                div.append(div1)
-            
-        # removing all the left-over stuff    
-        for a in div.findAll('span'):
-            a.extract() 
-            
-        for a in div.findAll('h1'):
-            a.extract()
-        for a in div.findAll('h2'):
-            a.extract()
-        for a in div.findAll('h3'):
-            a.extract()
-        for a in div.findAll('h4'):
-            a.extract()
-        for a in div.findAll('br'):
-            a.extract()
-        for a in div.findAll('div', {'class' : 'date'}):
-            a.extract()
-            
-        a = div.find('form')
-        if a != None:
-            b = a.nextSibling
-            while b != None:
+                # Find the "Continues" marker on the current page and remove everything after that. 
+                continues = div.find('span', {'class' : 'conTag'})
+                if continues != None:
+                    while continues != None:
+#                        logger.debug("removing end: {0}".format(continues))
+                        b = continues.nextSibling
+                        continues.extract()
+                        continues = b
+
+                # Find the "Continued" marker and delete everything before that
+                continued = div1.find('span', {'class' : 'conTag'})
+                if continued != None:
+                    a = continued.previousSibling
+                    while a != None:
+#                        logger.debug("before conTag: {0}".format(a))
+                        b = a.previousSibling
+                        a.extract()
+                        a = b
+                # Remove the pager from the end if this is the last page
+                endPager = div1.find('span', {'class' : 'pager'})
+                if endPager != None:
+                    b = endPager.nextSibling
+                    while endPager != None:
+                        logger.debug("removing end: {0}".format(endPager))
+                        b = endPager.nextSibling
+                        endPager.extract()
+                        endPager = b
+                    div1.contents = div1.contents[:len(div1) - 2]
+#                logger.debug("after removing pager: {0}".format(div1))
+                for tag in div1.contents[2:]:
+                    div.append(tag)
+
+        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus 
+        # everything before them can be discarded. 
+        postedDates = div.findAll('div', {'class' : 'date'})
+        if postedDates:
+            a = postedDates[0].previousSibling
+            while a != None:
+#                logger.debug("before dates: {0}".format(a))
+                b = a.previousSibling
                a.extract()
-                a=b
-                b=b.nextSibling
-        
+                a = b
+            for a in div.findAll('div', {'class' : 'date'}):
+                a.extract()
+
+        # For single chapter stories, there is a copyright statement. Remove this and everything
+        # before it.
+        copy = div.find('h4', {'class': 'copy'})
+        while copy != None:
+#            logger.debug("before copyright: {0}".format(copy))
+            b = copy.previousSibling
+            copy.extract()
+            copy = b
+
+        # For a story or the last chapter, remove voting form and the in library box
+        a = div.find('div', {'id' : 'vote-form'})
+        if a != None:
+            a.extract()
+        a = div.find('div', {'id' : 'b-man-div'})
+        if a != None:
+            a.extract()
+
+        # Kill the "The End" header and everything after it.
+        a = div.find(['h2', 'h3'], {'class' : 'end'})
+        logger.debug("Chapter end= '{0}'".format(a))
+        while a != None:
+            b = a.nextSibling
+            a.extract()
+            a=b
+

        if None == div:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
@@ -50,6 +50,7 @@ class TestSiteAdapter(BaseSiteAdapter):
    def extractChapterUrlsAndMetadata(self):
        idstr = self.story.getMetadata('storyId')
        idnum = int(idstr)
+        self.do_sleep()

        if idnum >= 1000:
            logger.warn("storyId:%s - Custom INI data will be used."%idstr)
@@ -296,6 +297,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"

    def getChapterText(self, url):
        logger.debug('Getting chapter text from: %s' % url)
+        self.do_sleep()
        if self.story.getMetadata('storyId').startswith('670') or \
                self.story.getMetadata('storyId').startswith('672'):
            time.sleep(1.0)
@@ -65,12 +65,16 @@ class TrekiverseOrgAdapter(BaseSiteAdapter):
        # The site domain.
        return 'trekiverse.org'

+    @classmethod
+    def getAcceptDomains(cls):
+        return ['trekiverse.org','efiction.trekiverse.org']
+
    @classmethod
    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://efiction."+self.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
-        return r'http://trekiverse\.org/efiction/viewstory\.php\?sid=\d+'
+        return r'(http://trekiverse\.org/efiction/viewstory\.php\?sid=\d+|http://efiction\.trekiverse\.org/viewstory\.php\?sid=\d+)'
       
    ## Login seems to be reasonably standard across eFiction sites.
    def needToLoginCheck(self, data):
@@ -76,6 +76,8 @@ class BaseSiteAdapter(Configurable):
        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
        self.password = ""
        self.is_adult=False
+
+        self.override_sleep = None
        
        self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
        ## Specific UA because too many sites are blocking the default python UA.
@@ -147,8 +149,7 @@ class BaseSiteAdapter(Configurable):

    # Assumes application/x-www-form-urlencoded.  parameters, headers are dict()s
    def _postUrl(self, url, parameters={}, headers={}):
-        if self.getConfig('slow_down_sleep_time'):
-            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        self.do_sleep()

        ## u2.Request assumes POST when data!=None.  Also assumes data
        ## is application/x-www-form-urlencoded.
@@ -159,24 +160,38 @@ class BaseSiteAdapter(Configurable):
        req = u2.Request(url,
                         data=urllib.urlencode(parameters),
                         headers=headers)
-        return self._decode(self.opener.open(req).read())
+        return self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())

    def _fetchUrlRaw(self, url, parameters=None):
        if parameters != None:
-            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters)).read()
+            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
        else:
-            return self.opener.open(url.replace(' ','%20')).read()
+            return self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
+
+    def set_sleep(self,val):
+        print("\n===========\n set sleep time %s\n==========="%val)
+        self.override_sleep = val
    
+    def do_sleep(self):
+        if self.override_sleep:
+            time.sleep(float(self.override_sleep))
+        elif self.getConfig('slow_down_sleep_time'):
+            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        
    # parameters is a dict()
    def _fetchUrl(self, url, parameters=None):
-        if self.getConfig('slow_down_sleep_time'):
-            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        self.do_sleep()

        excpt=None
        for sleeptime in [0, 0.5, 4, 9]:
            time.sleep(sleeptime)	
            try:
                return self._decode(self._fetchUrlRaw(url,parameters))
+            except u2.HTTPError, he:
+                if he.code == 404:
+                    logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(he)))
+                    excpt=he
+                    break # break out on 404
            except Exception, e:
                excpt=e
                logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
@@ -367,7 +382,7 @@ class BaseSiteAdapter(Configurable):
            
        # Don't want body tags in chapter html--writers add them.
        # This is primarily for epub updates.
-        retval = re.sub(r"</?body>\r?\n?","",retval)
+        retval = re.sub(r"</?body[^>]*>\r?\n?","",retval)
        
        if self.getConfig("replace_br_with_p"):
            # Apply heuristic processing to replace <br> paragraph
@@ -137,32 +137,35 @@ def replace_br_with_p(body):
            contentLinesSum += lineLen
            if lineLen > longestLineLength:
                longestLineLength = lineLen
+                
+    if contentLines == 0:
+        contentLines = 1

    averageLineLength = contentLinesSum/contentLines

    logger.debug(u'---')
-    logger.debug(u'Lines.............: ' + str(len(lines)))
-    logger.debug(u'contentLines......: ' + str(contentLines))
-    logger.debug(u'contentLinesSum...: ' + str(contentLinesSum))
-    logger.debug(u'longestLineLength.: ' + str(longestLineLength))
-    logger.debug(u'averageLineLength.: ' + str(averageLineLength))
+    logger.debug(u'Lines.............: ' + unicode(len(lines)))
+    logger.debug(u'contentLines......: ' + unicode(contentLines))
+    logger.debug(u'contentLinesSum...: ' + unicode(contentLinesSum))
+    logger.debug(u'longestLineLength.: ' + unicode(longestLineLength))
+    logger.debug(u'averageLineLength.: ' + unicode(averageLineLength))

    if breaksMaxIndex == len(breaksCount)-1 and breaksMax < 2:
        breaksMaxIndex = 0
        breaksMax = breaksCount[0]

    logger.debug(u'---')
-    logger.debug(u'breaks 1: ' + str(breaksCount[0]))
-    logger.debug(u'breaks 2: ' + str(breaksCount[1]))
-    logger.debug(u'breaks 3: ' + str(breaksCount[2]))
-    logger.debug(u'breaks 4: ' + str(breaksCount[3]))
-    logger.debug(u'breaks 5: ' + str(breaksCount[4]))
-    logger.debug(u'breaks 6: ' + str(breaksCount[5]))
-    logger.debug(u'breaks 7: ' + str(breaksCount[6]))
-    logger.debug(u'breaks 8: ' + str(breaksCount[7]))
+    logger.debug(u'breaks 1: ' + unicode(breaksCount[0]))
+    logger.debug(u'breaks 2: ' + unicode(breaksCount[1]))
+    logger.debug(u'breaks 3: ' + unicode(breaksCount[2]))
+    logger.debug(u'breaks 4: ' + unicode(breaksCount[3]))
+    logger.debug(u'breaks 5: ' + unicode(breaksCount[4]))
+    logger.debug(u'breaks 6: ' + unicode(breaksCount[5]))
+    logger.debug(u'breaks 7: ' + unicode(breaksCount[6]))
+    logger.debug(u'breaks 8: ' + unicode(breaksCount[7]))
    logger.debug(u'----')
-    logger.debug(u'max found: ' + str(breaksMax))
-    logger.debug(u'max Index: ' + str(breaksMaxIndex))
+    logger.debug(u'max found: ' + unicode(breaksMax))
+    logger.debug(u'max Index: ' + unicode(breaksMaxIndex))
    logger.debug(u'----')

    if breaksMaxIndex > 0 and breaksCount[0] > breaksMax and averageLineLength < 90:
@@ -173,13 +176,13 @@ def replace_br_with_p(body):
    for i in range(len(breaksCount)):
        # if i > 0 or breaksMaxIndex == 0:
        if i <= breaksMaxIndex:
-            logger.debug(str(i) + u' <= breaksMaxIndex (' + str(breaksMaxIndex) + u')')
+            logger.debug(unicode(i) + u' <= breaksMaxIndex (' + unicode(breaksMaxIndex) + u')')
            body = breaksRegexp[i].sub(r'\1</p>\n<p>\3', body)
        elif i == breaksMaxIndex+1:
-            logger.debug(str(i) + u' == breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            logger.debug(unicode(i) + u' == breaksMaxIndex+1 (' + unicode(breaksMaxIndex+1) + u')')
            body = breaksRegexp[i].sub(r'\1</p>\n<p><br/></p>\n<p>\3', body)
        else:
-            logger.debug(str(i) + u' > breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            logger.debug(unicode(i) + u' > breaksMaxIndex+1 (' + unicode(breaksMaxIndex+1) + u')')
            body = breaksRegexp[i].sub(r'\1</p>\n<hr />\n<p>\3', body)

    body = breaksRegexp[8].sub(r'</p>\n<hr />\n<p>', body)
@@ -230,7 +233,7 @@ def replace_br_with_p(body):
    return tag_sanitizer(body)

 def is_valid_block(block):
-    return str(block).find('<') == 0 and str(block).find('<!') != 0
+    return unicode(block).find('<') == 0 and unicode(block).find('<!') != 0

 def soup_up_div(body):
    blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'p', 'pre', 'table', 'ul']
@@ -247,8 +250,8 @@ def soup_up_div(body):
    lastElement = 1 # 1 = block, 2 = nested, 3 = invalid

    for i in soup.contents[0]:
-        if str(i).strip().__len__() > 0:
-            s = str(i)
+        if unicode(i).strip().__len__() > 0:
+            s = unicode(i)
            if  type(i) == bs.Tag:
                if  i.name in blockTags:
                    if lastElement > 1:
@@ -308,7 +311,7 @@ def tag_sanitizer(html):
        is_closed = is_closed_tag(rTag[0]) or is_comment_tag(rTag[0])

        # is_comment = is_comment_tag(rTag[0])
-        # logger.debug(u'%s >  isEnd: %s >  isClosed: %s >  isComment: %s'%(name, str(is_end), str(is_closed), str(is_comment)))
+        # logger.debug(u'%s >  isEnd: %s >  isClosed: %s >  isComment: %s'%(name, unicode(is_end), unicode(is_closed), unicode(is_comment)))
        # logger.debug(u'> %s%s\n'%(rTag[0], rTag[1]))

        if name in blockTags:
@@ -389,9 +389,14 @@ class Story(Configurable):
        elif self.getMetadataRaw('series') != None:
            self.setMetadata('seriesHTML',self.getMetadataRaw('series'))

+        # logger.debug("make_linkhtml_entries:%s"%self.getConfig('make_linkhtml_entries'))
        for k in self.getConfigList('make_linkhtml_entries'):
            # Assuming list, because it has to be site specific and
-            # they are all lists.
+            # they are all lists.  Bail if kUrl list not the same
+            # length.
+            # logger.debug("\nk:%s\nlist:%s\nlistURL:%s"%(k,self.getList(k),self.getList(k+'Url')))
+            if len(self.getList(k+'Url')) != len(self.getList(k)):
+                continue
            htmllist=[]
            for i, v in enumerate(self.getList(k)):
                url = self.getList(k+'Url')[i]
@@ -403,7 +408,7 @@ class Story(Configurable):
                    url=removeAllEntities(url)
                    v=removeAllEntities(v)

-                htmllist.append(linkhtml%('author',url,v))
+                htmllist.append(linkhtml%(k,url,v))
            join_string = self.getConfig("join_string_"+k+"HTML",u", ").replace('\s',' ')
            self.setMetadata(k+'HTML',join_string.join(htmllist))

@@ -568,7 +573,13 @@ class Story(Configurable):
            imgurl = url
        else:
            parsedUrl = urlparse.urlparse(parenturl)
-            if url.startswith("/") :
+            if url.startswith("//") :
+                imgurl = urlparse.urlunparse(
+                    (parsedUrl.scheme,
+                     '',
+                     url,
+                     '','',''))
+            elif url.startswith("/") :
                imgurl = urlparse.urlunparse(
                    (parsedUrl.scheme,
                     parsedUrl.netloc,
@@ -48,13 +48,9 @@

          <h3>fanfiction.net / fimfiction.net</h3>
          <p>
-            Fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-          <p>It appears that FimFiction.net is also blocking access from Google
-            App Engine now.
+            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
+            are working again.  I'd ask that users limit the number of
+            stories they download from those sites, thanks.
          </p>
 	  
      {% if authorized %}
@@ -68,12 +64,9 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>New Site: trekiverse.org/efiction/</li>
-              <li>Fix for higher rated stories with not logged in on tthfanfic.org.</li>
-              <li>Add prequels/sequels &amp; fix groups site specific metadata for fimfiction.net.</li>
+              <li>Add Pairing for hpfanficarchive.com</li>
 	    </ul>
 	  </p>
-          
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -82,7 +75,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-85.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-96.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -154,6 +154,11 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5

+## How long to wait for each HTTP connection to finish.  Longer times
+## are better for sites that are slow to respond.  Shorter times
+## prevent excessive wait when your network or the site is down.
+connect_timeout:60.0
+
 ## Use regular expressions to find and replace (or remove) metadata.
 ## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
 ## etc.  See http://docs.python.org/library/re.html (look for re.sub)
@@ -507,6 +512,13 @@ extratags: FanFiction,Testing,HTML
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## archiveofourown.org stories allow chapters to be added out of
+## order.  So the newest chapter may not be the last one.  FFDL update
+## doesn't like that.  If do_update_hook is uncommented and set true,
+## the adapter will discard all existing chapters from the newest one
+## on when updating to enforce accurate chapters.
+#do_update_hook:false
+
 ## AO3 adapter defines a few extra metadata entries.
 ## If there's ever more than 4 series, add series04,series04Url etc.
 extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
@@ -530,12 +542,13 @@ include_in_freefromtags:freeformtags
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories

-[ashwinder.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Severus Snape,Hermione Granger
-extraships:Severus Snape/Hermione Granger
+## AO3 chapters can include several different types of notes.  We've
+## traditional included them all in the chapter text, but this allows
+## you to customize which you include.  Copy this parameter to your
+## personal.ini and list the ones you don't want.
+#exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes

+[ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -543,6 +556,11 @@ extraships:Severus Snape/Hermione Granger
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Hermione Granger
+extraships:Severus Snape/Hermione Granger
+
 [asr3.slashzone.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Sentinel
@@ -553,9 +571,6 @@ extracategories:The Sentinel
 #is_adult:true

 [bloodties-fans.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Blood Ties
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -568,6 +583,9 @@ extracategories:Blood Ties
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Blood Ties
+
 [buffynfaith.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Buffy: The Vampire Slayer
@@ -578,9 +596,6 @@ extracategories:Buffy: The Vampire Slayer
 #is_adult:true

 [castlefans.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Castle
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -593,6 +608,9 @@ extracategories:Castle
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Castle
+
 [chaos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -605,10 +623,23 @@ cover_exclusion_regexp:/images/.*?ribbon.gif

 [dark-solace.org]
 ## Site dedicated to these categories/characters/ships
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Buffy, Spike
 extraships:Spike/Buffy

+[dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -621,24 +652,11 @@ extraships:Spike/Buffy
 ## personal.ini, not defaults.ini.
 #is_adult:true

-[dramione.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extracharacters:Draco Malfoy,Hermione Granger
 extraships:Draco Malfoy/Hermione Granger

-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
@@ -684,35 +702,34 @@ cliches_label:Character Cliches
 ## 'mode'.  'r' to Replace any existing values, 'a' to Add to existing
 ## value (use with tag-like columns), and 'n' for setting on New books
 ## only.  (Default is 'r'.)
+## Literal strings can be set into custom columns using double quotes.
 ## Each metadata=>column mapping must be on a separate line and each
 ## needs to have one space at the start of each line.

 #custom_columns_settings:
-# cliches=>#acolumn,r
+# cliches=>#acolumn
 # themes=>#bcolumn,a
 # timeline=>#ccolumn,n
+# "FanFiction"=>#collection

 [efiction.esteliel.de]
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings

 [erosnsappho.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif

 [fanfiction.mugglenet.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -725,10 +742,10 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfic.potterheadsanonymous.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -741,16 +758,19 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfiction.portkey.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extraships:Harry Potter/Hermione Granger

-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [fanfiction.tenhawkpresents.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -795,18 +815,26 @@ dislikes_label:Dislikes
 ## Clear FanFiction from defaults, site is original fiction.
 extratags:

-extra_valid_entries:size,universe,codes
-#extra_titlepage_entries:size,universe,codes
+extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
+#extra_titlepage_entries:size,universeHTML,codes,notice

 size_label:Size
 universe_label:Universe
+universeUrl_label:Universe URL
+universeHTML_label:Universe
 codes_label:Codes
+notice_label:Notice
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:universe
+
+## storiesonline.net stories can be in a series or a universe, but not
+## both.  By default, universe will be populated in 'series' with
+## index=0
+universe_as_series: true

 [grangerenchanted.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Hermione Granger
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -819,6 +847,10 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Hermione Granger
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:read,reviews
@@ -855,18 +887,15 @@ extracharacters:Kirk,Spock
 extraships:Kirk/Spock

 [lumos.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[merlinfic.dtwins.co.uk]
 ## Site dedicated to these categories/characters/ships
-extracategories:Merlin
+extracategories:Harry Potter

+[merlinfic.dtwins.co.uk]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -879,6 +908,9 @@ extracategories:Merlin
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Merlin
+
 [national-library.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:West Wing
@@ -890,24 +922,32 @@ extracategories:NCIS
 [netraptor.org]

 [nfacommunity.com]
-## Site dedicated to these categories/characters/ships
-extracategories:NCIS
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
 [nha.magical-worlds.us]
 ## Site dedicated to these categories/characters/ships
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow

 [occlumency.sycophanthex.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extracharacters:Severus Snape

+[onedirectionfanfiction.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -915,10 +955,15 @@ extracharacters:Severus Snape
 #username:YourName
 #password:yourpassword

-[onedirectionfanfiction.com]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:One Direction

+[pommedesang.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -931,40 +976,27 @@ extracategories:One Direction
 ## personal.ini, not defaults.ini.
 #is_adult:true

-[pommedesang.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Anita Blake Vampire Hunter 

-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+[ponyfictionarchive.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[ponyfictionarchive.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic

+[pretendercentre.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[pretendercentre.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Pretender

-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [samdean.archive.nu]
 ## Site dedicated to these categories/characters/ships
 extracategories:Supernatural
@@ -972,9 +1004,6 @@ extracharacters:Sam,Dean
 extraships:Sam/Dean

 [scarhead.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -987,6 +1016,9 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1017,9 +1049,6 @@ extracharacters:Draco Malfoy,Harry Potter
 extraships:Harry Potter/Draco Malfoy

 [thehookupzone.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Criminal Minds
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1032,6 +1061,9 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Criminal Minds
+
 [themasque.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1046,9 +1078,6 @@ extracategories:Criminal Minds
 #is_adult:true

 [thequidditchpitch.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1061,19 +1090,19 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true

-[tokra.fandomnet.com]
 ## Site dedicated to these categories/characters/ships
-extracategories:Stargate: SG-1
+extracategories:Harry Potter

+[tokra.fandomnet.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[trekiverse.org]
 ## Site dedicated to these categories/characters/ships
-extracategories:Star Trek
+extracategories:Stargate: SG-1

+[trekiverse.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1086,26 +1115,24 @@ extracategories:Star Trek
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+
 extra_valid_entries:awards
 awards_label:Awards

 cover_exclusion_regexp:art/.*Awards.jpg

 [www.adastrafanfic.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Star Trek
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[www.dracoandginny.com]
 ## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Draco Malfoy,Ginny Weasley
-extraships:Draco Malfoy/Ginny Weasley
+extracategories:Star Trek

+[www.dracoandginny.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1118,14 +1145,16 @@ extraships:Draco Malfoy/Ginny Weasley
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Ginny Weasley
+extraships:Draco Malfoy/Ginny Weasley
+
 [www.thealphagate.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1

 [www.checkmated.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1133,6 +1162,9 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.destinysgateway.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1140,11 +1172,6 @@ extracategories:Harry Potter
 #is_adult:true

 [www.dokuga.com]
-## Site dedicated to these categories/characters/ships
-extracategories:InuYasha
-extracharacters:Sesshoumaru,Kagome
-extraships:Sesshoumaru/Kagome
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1152,6 +1179,11 @@ extraships:Sesshoumaru/Kagome
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:Sesshoumaru,Kagome
+extraships:Sesshoumaru/Kagome
+
 [www.dotmoon.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1181,6 +1213,20 @@ user_agent:
 ## Change this to false to use them anyway.
 never_make_cover: true

+## fanfiction.net is blocking people more aggressively.  If you
+## download fewer stories less often you can likely get by with
+## reducing this sleep.
+slow_down_sleep_time:4
+
+## ffnet is sensitive to too many hits.  Users are sensitive to long
+## waits during the initial metadata collection in the foreground.
+## When used, these settings will speed up metadata downloads in the
+## foreground linearly.
+tweak_fg_sleep:true
+min_fg_sleep:1.0
+max_fg_sleep:4.0
+max_fg_sleep_at_downloads: 10
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
@@ -1189,13 +1235,20 @@ extra_valid_entries:reviews,favs,follows
 ## to be romantic pairings.
 ships_label:Pairings

-## Date formats used by FFDL.  Published and Update don't have time.
+## Date formats used by FFDL.  Published and Update don't usually have
+## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
 #dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
 datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
 dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S

+## ffnet used to have a tendency to send out update notices in email
+## before all their servers were showing the update on the first
+## chapter.  It generates another server request and doesn't seem to
+## be needed lately, so now default it to off.
+check_next_chapter:false
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1207,14 +1260,14 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 [www.ficbook.net]

 [www.fictionalley.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## fictionalley.org doesn't have a status metadatum.  If uncommented,
 ## this will be used for status.
 #default_value_status:Unknown
@@ -1237,6 +1290,23 @@ extra_valid_entries:reviews,favs,follows
 #password:yourpassword

 [www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fimfiction.net stories can be locked requiring individual
+## passwords.  If fail_on_password is set, the downloader will fail
+## when a password is required rather than prompting every time.
+#fail_on_password: false
+
+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic

@@ -1267,55 +1337,33 @@ keep_in_order_groupsUrl:true
 ## make entryHTML.
 make_linkhtml_entries:prequel,sequels,groups

+[www.harrypotterfanfiction.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-## fimfiction.net stories can be locked requiring individual
-## passwords.  If fail_on_password is set, the downloader will fail
-## when a password is required rather than prompting every time.
-#fail_on_password: false
-
-## fimfiction.net stories allow chapters to be added out of order.  So
-## the newest chapter may not be the last one.  FFDL update doesn't
-## like that.  If do_update_hook is uncommented and set true, the
-## adapter will discard all existing chapters from the newest one on
-## when updating to enforce accurate chapters.
-#do_update_hook:false
-
-[www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter

-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.henneth-annun.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Hobbit

 [www.hpfandom.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.hpfanficarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter

 [www.ik-eternal.net]
-## Site dedicated to these categories/characters/ships
-extracategories:InuYasha
-extracharacters:InuYasha,Kagome
-extraships:InuYasha/Kagome
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1328,30 +1376,35 @@ extraships:InuYasha/Kagome
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:InuYasha,Kagome
+extraships:InuYasha/Kagome
+
 [www.jlaunlimited.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:JLA

 [www.libraryofmoria.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Lord of the Rings
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [www.mediaminer.org]

 [www.midnightwhispers.ca]
-## Site dedicated to these categories/characters/ships
-extracategories:Queer as Folk
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
@@ -1366,10 +1419,6 @@ extracategories:CSI
 extraships:Nick Stokes/Greg Sanders

 [www.phoenixsong.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extraships:Harry Potter/Ginny Weasley
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1380,9 +1429,13 @@ extraships:Harry Potter/Ginny Weasley
 ## phoenixsong.net, oddly, can have high rated chapters (login
 ## required) in the middle of a lower rated story.  Use this to force
 ## FFDL to always login to phoenixsong.net so those stories download
-## correctly.
+## correctly.  If you have a login, this is recommended.
 #force_login:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Ginny Weasley
+
 [www.potionsandsnitches.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1405,17 +1458,15 @@ extracategories:Psych
 #is_adult:true

 [www.qaf-fic.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Queer as Folk
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 [www.restrictedsection.org]
-extracategories:Harry Potter
-extragenres:Erotica
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1423,20 +1474,21 @@ extragenres:Erotica
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extragenres:Erotica
+
 [www.scarvesandcoffee.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Glee
 extracharacters:Kurt Hummel,Blaine Anderson

-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.simplyundeniable.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1444,15 +1496,18 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword

-[www.sinful-desire.org]
 ## Site dedicated to these categories/characters/ships
-extracategories:Supernatural
+extracategories:Harry Potter

+[www.sinful-desire.org]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+
 [www.siye.co.uk]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1460,25 +1515,26 @@ extracharacters:Harry Potter,Ginny Weasley
 extraships:Harry Potter/Ginny Weasley

 [www.squidge.org/peja]
-# www.squidge.org/peja calls it Fandom <shrug>
+## www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
-# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+
+## Remove numWords -- www.squidge.org/peja word counts are inaccurate
 titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description

 [www.squidge.org/peja:txt]
-## Add URLs since there aren't links.
-# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+## Add URLs since there aren't links and remove numWords --
+## www.squidge.org/peja word counts are inaccurate
 titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description

 [www.storiesofarda.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Lord of the Rings
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [www.thepetulantpoetess.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1521,9 +1577,6 @@ collect_series: false
 extracategories:Twilight

 [www.twilighted.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Twilight
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1531,13 +1584,13 @@ extracategories:Twilight
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## twilighted.net (ab)uses series as personal reading lists.
 collect_series: false

 [www.twiwrite.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Twilight
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1545,20 +1598,23 @@ extracategories:Twilight
 #username:YourName
 #password:yourpassword

+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## twiwrite.net (ab)uses series as personal reading lists.
 collect_series: false

 [www.walkingtheplank.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Severus Snape,Harry Potter
-extraships:Severus Snape/Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Harry Potter
+extraships:Severus Snape/Harry Potter
+
 [www.whofic.com]

 [www.wizardtales.net]
@@ -1580,14 +1636,14 @@ extracategories:X-Men Movie
 extracharacters:Wolverine,Rogue

 [www.wraithbait.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Stargate: Atlantis
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For
@@ -53,13 +53,9 @@
 	  {% if fic.failure %}
          <h3>fanfiction.net / fimfiction.net</h3>
          <p>
-            FYI, fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-          <p>It appears that FimFiction.net is also blocking access from Google
-            App Engine now.
+            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
+            are working again.  I'd ask that users limit the number of
+            stories they download from those sites, thanks.
          </p>
 	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}
Author	SHA1	Message	Date
Jim Miller	bf509bb6ee	Update CLI download zip.	2014-03-29 17:21:36 -05:00
Jim Miller	8b4f1953fe	Bump versions	2014-03-29 17:20:42 -05:00
Jim Miller	cf7e913e60	Remove some outdated checks against calibre versions that aren't supported now.	2014-03-29 11:33:01 -05:00
Jim Miller	402366a3e5	Allow update and metaonly at the same time on CLI.	2014-03-29 11:25:08 -05:00
Jim Miller	bc0ef37fe6	Add Pairing to hpfanficarchivecom.	2014-03-29 11:24:37 -05:00
Jim Miller	3b77a6b1c1	Added tag calibre-plugin-1.8.15 for changeset f760e487f789	2014-03-16 15:28:49 -05:00
Jim Miller	f603762261	Added tag FanFictionDownLoader-4.4.96 for changeset f760e487f789	2014-03-16 15:28:31 -05:00
Jim Miller	094f864252	Update CLI download zip.	2014-03-16 15:28:06 -05:00
Jim Miller	34e67ebb26	Bump versions.	2014-03-16 15:27:55 -05:00
Jim Miller	7dae1140a7	Add exclude_notes option for AO3 to exclude the various notes.	2014-03-14 14:04:15 -05:00
Jim Miller	c4b1810887	Fix fictionpress.com for https.	2014-03-10 11:09:44 -05:00
Jim Miller	e9d3ef2b9a	Added tag FanFictionDownLoader-4.4.95 for changeset f98aa559dac3	2014-03-09 21:11:16 -05:00
Jim Miller	04895411b1	Added tag calibre-plugin-1.8.14 for changeset f98aa559dac3	2014-03-09 21:11:01 -05:00
Jim Miller	e8706b19fd	Update CLI download zip.	2014-03-09 21:10:41 -05:00
Jim Miller	dc86c90b9e	Bump versions.	2014-03-09 21:09:47 -05:00
Jim Miller	b90399d674	On auto-convert, delete target format else convert doesn't run.	2014-03-08 21:18:14 -06:00
Jim Miller	7c13b5aed9	Remove some dead code.	2014-03-08 15:45:16 -06:00
Jim Miller	8d87564158	Fix for AO3 https urls.	2014-03-08 15:45:03 -06:00
Jim Miller	1663f13950	Fix for seriesUrl check.	2014-03-08 15:44:51 -06:00
Jim Miller	10aedfbce6	Configurable connection timeout as per FaceDeer's recommendation.	2014-03-01 17:00:07 -06:00
Jim Miller	d5dd328a05	Change Job Log output so URLs are more copy/pasteable.	2014-03-01 16:59:30 -06:00
Jim Miller	d08bb7ac21	Fix for potionsandsnitches.net, extra text after date.	2014-03-01 16:59:02 -06:00
Jim Miller	6687601074	Added tag FanFictionDownLoader-4.4.94 for changeset 1873e173ef01	2014-02-28 17:10:35 -06:00
Jim Miller	4847a2b1b4	Added tag calibre-plugin-1.8.13 for changeset 1873e173ef01	2014-02-28 17:10:18 -06:00
Jim Miller	4718955ede	Update CLI download zip.	2014-02-28 17:10:05 -06:00
Jim Miller	d60ad9892d	Fix for KeyError:'site' error and https with ffnet. Also don't retry 404 errors and bump versions.	2014-02-28 17:08:07 -06:00
Jim Miller	35b5eb3dee	Added tag FanFictionDownLoader-4.4.93 for changeset f603b2ffd630	2014-02-24 21:11:11 -06:00
Jim Miller	f9d8e2f3db	Added tag calibre-plugin-1.8.12 for changeset f603b2ffd630	2014-02-24 21:10:58 -06:00
Jim Miller	f291625ae3	Update CLI download zip.	2014-02-24 21:10:46 -06:00
Jim Miller	6ef28cee6a	Add tweak_fg_sleep feature set for ffnet, make ffnet chapter look ahead optional	2014-02-24 21:08:56 -06:00
Jim Miller	fc3431098d	Add slow_down_sleep_time to test1 adapter for... testing.	2014-02-21 22:59:03 -06:00
Jim Miller	8980eaf21b	Set boolean custom columns from personal.ini when t/true/yes/y/1 vs f/false/no/n/0.	2014-02-21 19:07:32 -06:00
Jim Miller	c8a89fa8b2	More Fimf fixes from FaceDeer.	2014-02-21 19:05:37 -06:00
Jim Miller	683ad6182d	Fix for calibre converted <body ...> tags.	2014-02-21 19:04:40 -06:00
Jim Miller	3267064576	Added tag FanFictionDownLoader-4.4.92 for changeset 43b0cca5b451	2014-02-15 09:45:07 -06:00
Jim Miller	ab128c417c	Added tag calibre-plugin-1.8.11 for changeset e3c25dc63ece	2014-02-15 09:44:35 -06:00
Jim Miller	dbc9b5e250	Update CLI download zip.	2014-02-15 09:44:19 -06:00
Jim Miller	fffdd89254	Update translations to include new strings (not yet translated).	2014-02-15 09:42:02 -06:00
Jim Miller	cbb17aeba1	Bump versions.	2014-02-15 09:39:27 -06:00
Jim Miller	7433d2494f	Increase default slow_down_sleep_time for ffnet.	2014-02-15 09:07:52 -06:00
Jim Miller	d0fa72a002	Don't autoconvert on Update Calibre Only.	2014-02-11 10:48:48 -06:00
Jim Miller	7412acd007	Fix image download for ffnet, make buffynfaith.net use User-agent.	2014-02-10 13:27:33 -06:00
Jim Miller	196eff7f60	Add option to call Polish on GC to inject cover in epub.	2014-02-10 13:26:58 -06:00
Jim Miller	8f06a374cb	Moved tag FanFictionDownLoader-4.4.91 to changeset 40b20c6d208b (from changeset 135173f84a14)	2014-02-05 19:07:23 -06:00
Jim Miller	4f5738108f	Update CLI download zip.	2014-02-05 19:07:08 -06:00
Jim Miller	6f22a3c701	Added tag calibre-plugin-1.8.10 for changeset 135173f84a14	2014-02-05 18:58:20 -06:00
Jim Miller	589076eed9	Added tag FanFictionDownLoader-4.4.91 for changeset 135173f84a14	2014-02-05 18:58:06 -06:00
Jim Miller	ee07ac8919	Bump versions.	2014-02-05 18:57:41 -06:00
Jim Miller	dc5def5cc8	Change to storiesonline.net by davidfor to address overzealous culling in text.	2014-02-05 18:52:07 -06:00
Jim Miller	23ad8700bb	Quick fix for storiesonline.net & merging heuristics change.	2014-02-01 18:54:31 -06:00
asbjorn grandt	250121e137	Fix divide by zero error on chapters with no content.	2014-02-02 00:55:34 +01:00
Jim Miller	56a639508e	Translations(fr&de) for auto-convert feature strings. nbsp for French.	2014-01-28 13:14:07 -06:00
Jim Miller	d398b62c1a	Add auto-convert on download/update feature. Still need translations.	2014-01-26 11:57:49 -06:00
Jim Miller	bff3cfbd25	Fix for typo in French that breaks FFDL.	2014-01-22 17:04:51 -06:00
Jim Miller	34558b1653	Added tag calibre-plugin-1.8.09 for changeset ca8fa4c502c5	2014-01-22 10:08:33 -06:00
Jim Miller	0ce37974e2	calibre/python doesn't like 0 leading version numers anymore? Bump to 1.8.9	2014-01-22 10:07:09 -06:00
Jim Miller	929cf36e79	Added tag calibre-plugin-1.8.08 for changeset ba5441b903ae	2014-01-21 20:43:18 -06:00
Jim Miller	b86c15d5a6	Bump calibre version only.	2014-01-21 20:43:06 -06:00
Jim Miller	df622dfbf0	Allow literal strings in custom_columns_settings to set columns.	2014-01-21 20:42:04 -06:00
Jim Miller	2c16b5e148	Added tag FanFictionDownLoader-4.4.90 for changeset f1ff0a1eed93	2014-01-13 19:09:35 -06:00
Jim Miller	86ce293f78	Update CLI download 4.4.90	2014-01-13 19:09:02 -06:00
Jim Miller	63e8fc8bb2	Added tag calibre-plugin-1.8.07 for changeset 5aa783520099	2014-01-13 19:05:50 -06:00
Jim Miller	38ab66907b	Bump versions.	2014-01-13 19:05:36 -06:00
Jim Miller	1dedde2f9d	Fix for hpfanficarchive.com changes.	2014-01-13 18:42:36 -06:00
Jim Miller	fc5dd20853	Updated French translation.	2014-01-08 17:05:07 -06:00
Jim Miller	4e9fe9f86f	First version of French translation.	2014-01-08 14:56:09 -06:00
Jim Miller	7325be98f5	davidfor's changes for storiesonline.net universe/series.	2014-01-05 18:30:58 -06:00
Jim Miller	e2545a0da8	Encoding, series & universe for storiesonline.net.	2014-01-05 10:06:03 -06:00
Jim Miller	5cd3dba4a6	Default utf8 first for finestories.com.	2014-01-04 12:11:47 -06:00
Jim Miller	bc2420d143	Fix for 'update' with no updates and 'mark' enabled.	2014-01-04 12:11:13 -06:00
Jim Miller	099e8a62ca	Added tag FanFictionDownLoader-4.4.89 for changeset d8f5d11b860f	2013-12-23 17:49:48 -05:00
Jim Miller	7f9e22e1b9	Update index and CLI download zip.	2013-12-23 17:49:28 -05:00
Jim Miller	a46532bc72	Added tag calibre-plugin-1.8.06 for changeset dd7269e25be8	2013-12-23 17:46:44 -05:00
Jim Miller	f7429140f8	Bump versions.	2013-12-23 17:45:19 -05:00
Jim Miller	7872b25034	Minor tweak to ffnet fix in case they change it back, sleep on ffnet meta lookup.	2013-12-23 17:45:08 -05:00
Jim Miller	3d468a4dc5	New site: literotica.com, tweaks to htmlheuristics for it.	2013-12-23 17:43:32 -05:00
asbjorn grandt	413525a881	url for fictionratings changed to https, breaking the soup find.	2013-12-23 08:14:10 +01:00
Jim Miller	c066a809c7	Added tag FanFictionDownLoader-4.4.88 for changeset 7ed8deaec5d2	2013-12-12 18:10:51 -06:00
Jim Miller	443b2b5ef8	Save CLI download version 4.4.88	2013-12-12 18:10:35 -06:00
Jim Miller	3e7f05ba44	Save CLI download version 4.4.87	2013-12-12 18:09:22 -06:00
Jim Miller	25e65da239	Added tag calibre-plugin-1.8.05 for changeset 8254e19f1ab2	2013-12-12 18:03:35 -06:00
Jim Miller	c5f211f49f	Moved tag calibre-plugin-1.8.04 to changeset 6e301aeb7f4b (from changeset e45187f4215e)	2013-12-12 18:03:26 -06:00
Jim Miller	20fff58813	Moved tag calibre-plugin-1.8.03 to changeset ea92e962bcaa (from changeset 6e301aeb7f4b)	2013-12-12 18:03:00 -06:00
Jim Miller	8fe250747c	Added tag calibre-plugin-1.8.04 for changeset e45187f4215e	2013-12-12 17:59:56 -06:00
Jim Miller	4d41c28f3c	Fix a bug with AO3 single chapter stories introduced with do_update_hook.	2013-12-12 17:59:41 -06:00
Jim Miller	f8d1b21007	Moved tag FanFictionDownLoader-4.4.87 to changeset 8d78c6f57151 (from changeset 6e301aeb7f4b)	2013-12-12 13:35:55 -06:00
Jim Miller	72861e6cd5	Save CLI Zip file.	2013-12-12 13:35:30 -06:00
Jim Miller	bd9dba5813	Added tag FanFictionDownLoader-4.4.87 for changeset 6e301aeb7f4b	2013-12-12 13:33:46 -06:00
Jim Miller	1478df572e	Added tag calibre-plugin-1.8.03 for changeset 6e301aeb7f4b	2013-12-12 13:33:30 -06:00
Jim Miller	68466af944	Bump versions.	2013-12-12 13:33:13 -06:00
Jim Miller	7c9c84256c	Graceful fail for 'k'HTML when no 'k'Url, bump min calibre version.	2013-12-11 12:14:55 -06:00
Jim Miller	fcbfe96c64	Add do_update_hook to AO3 for updated chapters in the middle, trekiverse alt URL, reorder defaults.ini files to put user/pass is_adult first in all sections.	2013-12-11 12:13:22 -06:00
Jim Miller	e47748dbbc	Added tag calibre-plugin-1.8.02 for changeset a277cb866377	2013-12-05 13:16:45 -06:00
Jim Miller	91a6c668d1	Added tag FanFictionDownLoader-4.4.86 for changeset a277cb866377	2013-12-05 13:16:22 -06:00