Bump versions.

Fix for change to AO3 author links.
Force overall cover for Anthologies if any sub-books have covers to avoid Calibre Polish issues.
2026-09-13 12:11:20 +08:00 · 2016-08-24 13:17:35 -05:00 · 2016-08-24 09:57:11 -05:00 · 2016-08-20 14:05:36 -05:00 · 2016-08-20 12:08:19 -05:00 · 2016-08-20 00:10:18 -05:00
41 changed files with 5283 additions and 3697 deletions
@@ -48,7 +48,7 @@ class FanFicFareBase(InterfaceActionBase):
    description         = _('UI plugin to download FanFiction stories from various sites.')
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (2, 3, 4)
+    version             = (2, 3, 6)
    minimum_calibre_version = (1, 48, 0)

    #: This field defines the GUI plugin class that contains all the code
@@ -328,6 +328,8 @@ class ConfigWidget(QWidget):
        prefs['std_cols_newonly'] = colsnewonly

        prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
+        prefs['includecomments'] =self.std_columns_tab.includecomments.isChecked()
+        prefs['anth_comments_newonly'] =self.std_columns_tab.anth_comments_newonly.isChecked()

        # Custom Columns tab
        # error column
@@ -1438,6 +1440,17 @@ class StandardColumnsTab(QWidget):
        self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
        self.set_author_url.setChecked(prefs['set_author_url'])
        self.l.addWidget(self.set_author_url)
+
+        self.includecomments = QCheckBox(_("Include Books' Comments in Anthology Comments?"),self)
+        self.includecomments.setToolTip(_('''Include all the merged books' comments in the new book's comments.
+Default is a list of included titles only.'''))
+        self.includecomments.setChecked(prefs['includecomments'])
+        self.l.addWidget(self.includecomments)
+
+        self.anth_comments_newonly = QCheckBox(_("Set Anthology Comments only for new books"),self)
+        self.anth_comments_newonly.setToolTip(_("Comments will only be set for New Anthologies, not updates.\nThat way comments you set manually are retained."))
+        self.anth_comments_newonly.setChecked(prefs['anth_comments_newonly'])
+        self.l.addWidget(self.anth_comments_newonly)
        
        self.l.insertStretch(-1)

@@ -622,6 +622,7 @@ class _LoopProgressDialog(QProgressDialog):
        self.status_prefix = status_prefix
        self.i = 0
        self.start_time = datetime.now()
+        self.first = True

        # can't import at file load.
        from calibre_plugins.fanficfare_plugin.prefs import prefs
@@ -648,8 +649,14 @@ class _LoopProgressDialog(QProgressDialog):

    def do_loop(self):

-        if self.i == 0:
-            self.setValue(0)
+        if self.first:
+            ## Windows 10 doesn't want to show the prog dialog content
+            ## until after the timer's been called again.  Something to
+            ## do with cooperative multi threading maybe?
+            ## So this just trips the timer loop an extra time at the start.
+            self.first = False
+            QTimer.singleShot(0, self.do_loop)
+            return

        book = self.book_list[self.i]
        try:
@@ -83,7 +83,7 @@ from calibre_plugins.fanficfare_plugin.fanficfare import (

 from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import (
    get_dcsource, get_dcsource_chaptercount, get_story_url_from_html,
-    reset_orig_chapters_epub)
+    reset_orig_chapters_epub, get_cover_data)

 from calibre_plugins.fanficfare_plugin.fanficfare.geturls import (
    get_urls_from_page, get_urls_from_html,get_urls_from_text,
@@ -116,6 +116,13 @@ formmapping = {
    'txt':'TXT'
    }

+imagetypes = {
+    'image/jpeg':'jpg',
+    'image/png':'png',
+    'image/gif':'gif',
+    'image/svg+xml':'svg',
+    }
+
 PLUGIN_ICONS = ['images/icon.png']

 class FanFicFarePlugin(InterfaceAction):
@@ -1733,6 +1740,7 @@ class FanFicFarePlugin(InterfaceAction):
                                      show_copy_button=False)

    def do_download_merge_update(self, payload):
+        db = self.gui.current_db

        (good_list,bad_list,options) = payload
        total_good = len(good_list)
@@ -1758,20 +1766,62 @@ class FanFicFarePlugin(InterfaceAction):

        #print("mergebook:\n%s"%mergebook)

-        if mergebook['good']: # there shouldn't be any !'good' books at this point.
-            # if still 'good', make a temp file to write the output to.
-            tmp = PersistentTemporaryFile(suffix='.'+options['fileform'],
-                                          dir=options['tdir'])
-            logger.debug("title:"+mergebook['title'])
-            logger.debug("outfile:"+tmp.name)
-            mergebook['outfile'] = tmp.name
+        # make a temp file to write the output to.
+        tmp = PersistentTemporaryFile(suffix='.'+options['fileform'],
+                                      dir=options['tdir'])
+        # logger.debug("title:"+mergebook['title'])
+        logger.debug("outfile:"+tmp.name)
+        mergebook['outfile'] = tmp.name

+        ## Calibre's Polish heuristics for covers can cause problems
+        ## if a merged anthology book has sub-book covers, but not a
+        ## proper main cover.  So, now if there are any covers, we
+        ## will force a main cover.
+
+        ## start with None.  If no subbook covers, don't force one
+        ## here.  User can configure FFF to always create/polish a
+        ## cover if they want.  This is about when we force it.
+        coverpath = None
+        coverimgtype = None
+
+        ## first, look for covers inside the subbooks.  Stop at the
+        ## first one, which will be used if there isn't a pre-existing
+        ## calibre cover.
+        if not coverpath:
+            for book in good_list:
+                coverdata = get_cover_data(book['outfile'])
+                if coverdata: # found a cover.
+                    (coverimgtype,coverimgdata) = coverdata[4:6]
+                    logger.debug('coverimgtype:%s [%s]'%(coverimgtype,imagetypes[coverimgtype]))
+                    tmpcover = PersistentTemporaryFile(suffix='.'+imagetypes[coverimgtype],
+                                                       dir=options['tdir'])
+                    tmpcover.write(coverimgdata)
+                    tmpcover.flush()
+                    tmpcover.close()
+                    coverpath = tmpcover.name
+                    break
+        # logger.debug('coverpath:%s'%coverpath)
+
+        ## if updating an existing book and there is at least one
+        ## subbook cover:
+        if coverpath and mergebook['calibre_id']:
+            # Couldn't find a better way to get the cover path.
+            calcoverpath = os.path.join(db.library_path,
+                                     db.path(mergebook['calibre_id'], index_is_id=True),
+                                     'cover.jpg')
+            ## if there's an existing cover, use it.  Calibre will set
+            ## it for us during lots of different actions anyway.
+            if os.path.exists(calcoverpath):
+                coverpath = calcoverpath
+
+        # logger.debug('coverpath:%s'%coverpath)
        self.get_epubmerge_plugin().do_merge(tmp.name,
                                             [ x['outfile'] for x in good_list ],
                                             tags=mergebook['tags'],
                                             titleopt=mergebook['title'],
                                             keepmetadatafiles=True,
-                                             source=mergebook['url'])
+                                             source=mergebook['url'],
+                                             coverjpgpath=coverpath)

        options['collision']=OVERWRITEALWAYS
        self.update_books_loop(mergebook,self.gui.current_db,options)
@@ -1891,7 +1941,8 @@ class FanFicFarePlugin(InterfaceAction):
            #print("mi.tags:%s"%mi.tags)

        if book['all_metadata']['langcode']:
-            mi.languages=[book['all_metadata']['langcode']]
+            # split due to anthologies.  Gives list of one for non-anth.
+            mi.languages=book['all_metadata']['langcode'].split(', ')
        else:
            # Set language english, but only if not already set.
            if not oldmi.languages:
@@ -1958,14 +2009,17 @@ class FanFicFarePlugin(InterfaceAction):
        configuration = None
        if prefs['allow_custcol_from_ini']:
            configuration = get_fff_config(book['url'],options['fileform'])
-            # meta => custcol[,a|n|r]
+            # meta => custcol[,a|n|r|n_anthaver,r_anthaver]
            # cliches=>\#acolumn,r
            for line in configuration.getConfig('custom_columns_settings').splitlines():
                if "=>" in line:
                    (meta,custcol) = map( lambda x: x.strip(), line.split("=>") )
                    flag='r'
+                    anthaver=False
                    if "," in custcol:
                        (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )
+                        anthaver = 'anthaver' in flag
+                        flag=flag[0] # first char only.

                    if meta not in book['all_metadata']:
                        # if double quoted, use as a literal value.
@@ -1987,8 +2041,15 @@ class FanFicFarePlugin(InterfaceAction):
                    if flag == 'r' or (flag == 'n' and book['added']):
                        if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
                            if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
-                                # re-split list, strip commas, convert to floats, sum up.
-                                val = sum([ float(x.replace(",","")) for x in val.split(", ") ])
+                                # re-split list, strip commas, convert to floats
+                                items = [ float(x.replace(",","")) for x in val.split(", ") ]
+                                if anthaver:
+                                    if items:
+                                        val = sum(items) / float(len(items))
+                                    else:
+                                        val = 0
+                                else:
+                                    val = sum(items)
                            else:
                                val = unicode(val).replace(",","")
                        else:
@@ -2400,7 +2461,9 @@ class FanFicFarePlugin(InterfaceAction):
                #print("book series:%s"%serieslist[-1])

            if b['publisher']:
-                if 'publisher' not in book:
+                if not book['publisher']:
+                    ## not set in all_metadata because it's not one of
+                    ## the permitted metadata--use site instead.
                    book['publisher']=b['publisher']
                elif book['publisher']!=b['publisher']:
                    book['publisher']=None # if any are different, don't use.
@@ -2458,18 +2521,33 @@ class FanFicFarePlugin(InterfaceAction):
                        book['anthology_meta_list'][k]=True

        logger.debug("book['url']:%s"%book['url'])
+
+        book['comments'] = _("Anthology containing:")+"\n\n"
+        if len(book['author']) > 1:
+            mkbooktitle = lambda x : _("%s by %s") % (x['title'],' & '.join(x['author']))
+        else:
+            mkbooktitle = lambda x : x['title']
+
+        if prefs['includecomments']:
+            def mkbookcomments(x):
+                if x['comments']:
+                    return '<b>%s</b>\n\n%s'%(mkbooktitle(x),x['comments'])
+                else:
+                    return '<b>%s</b>\n'%mkbooktitle(x)
+
+            book['comments'] += ('<div class="mergedbook">' +
+                            '<hr></div><div class="mergedbook">'.join([ mkbookcomments(x) for x in book_list]) +
+                            '</div>')
+        else:
+            book['comments'] += '\n'.join( [ mkbooktitle(x) for x in book_list ] )
+
        configuration = get_fff_config(book['url'],fileform)
        if existingbook:
            book['title'] = deftitle = existingbook['title']
-            book['comments'] = existingbook['comments']
+            if prefs['anth_comments_newonly']:
+                book['comments'] = existingbook['comments']
        else:
            book['title'] = deftitle = book_list[0]['title']
-            if len(book['author']) > 1:
-                book['comments'] = _("Anthology containing:")+"\n" + \
-                    "\n".join([ _("%s by %s")%(b['title'],', '.join(b['author'])) for b in book_list ])
-            else:
-                book['comments'] = _("Anthology containing:")+"\n" + \
-                    "\n".join([ b['title'] for b in book_list ])
            # book['all_metadata']['description']

            # if all same series, use series for name.  But only if all and not previous named
@@ -25,7 +25,7 @@
 ## titlepage_entries: category,genre, status,dateUpdated,rating
 ## [epub]
 ## # overrides defaults & site section
-## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated
 ## [www.whofic.com:epub]
 ## # overrides defaults, site section & format section
 ## titlepage_entries: category,genre, status,datePublished
@@ -510,6 +510,20 @@ first_post_title:First Post
 ## if threadmarks are used.  Can result in a duplicated chapter.
 always_include_first_post:false

+## In normal operation, when updating an existing epub, old chapters
+## will be reused as-is.  Normally, that works fine, but forum stories
+## sometimes have an index post as the first 'chapter', and the
+## version in the first chapter gets out of sync.
+##
+## If always_reload_first_chapter:true, then the first chapter will
+## always be downloaded again (ie, reloaded).  It will NOT be maked
+## '(new)' (see mark_new_chapters).  Because it is reloaded, manual
+## edits made to the first chapter will be lost.
+##
+## While intended for base_xenforoforum sites, this setting can be
+## applied to other sites.
+always_reload_first_chapter:false
+
 ## In normal operation, forumtags will only be populated when
 ## threadmarks are used for chapters (see minimum_threadmarks above).
 ## When always_use_forumtags:true, always populate forumtags.
@@ -585,6 +599,11 @@ include_logpage: false
 ## end up with Completed stories that have just one logpage entry.
 #include_logpage: smart

+## By default, logpage is placed before the story chapters.  This
+## setting, if true, will place the logpage after the chapters
+## instead.
+logpage_at_end: false
+
 ## items to include in the log page Empty metadata entries, or those
 ## that haven't changed since the last update, will *not* appear, even
 ## if in the list.  You can include extra text or HTML that will be
@@ -748,6 +767,19 @@ extratags: FanFiction,Testing,Text
 [test1.com:html]
 extratags: FanFiction,Testing,HTML

+[adult-fanfiction.org]
+extra_valid_entries:eroticatags,disclaimer
+eroticatags_label:Erotica Tags
+disclaimer_label:Disclaimer
+extra_titlepage_entries:eroticatags,disclaimer
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [archive.skyehawke.com]

 [archiveofourown.org]
@@ -1008,15 +1040,24 @@ cliches_label:Character Cliches
 ## 'mode'.  'r' to Replace any existing values, 'a' to Add to existing
 ## value (use with tag-like columns), and 'n' for setting on New books
 ## only.  (Default is 'r'.)
+
 ## Literal strings can be set into custom columns using double quotes.
 ## Each metadata=>column mapping must be on a separate line and each
 ## needs to have one space at the start of each line.

+## 'r_anthaver' and 'n_anthaver' can be used to indicate the same as
+## 'r' and 'n' for normal downloads, but to average the metadata for
+## the differents story in an anthology before setting in integer and
+## float type custom columns.  This can be useful for a averrating
+## column, for example.  Default is to sum the values of all stories,
+## and numChapters and numWords are always summed.
+
 #custom_columns_settings:
 # cliches=>#acolumn
 # themes=>#bcolumn,a
 # timeline=>#ccolumn,n
 # "FanFiction"=>#collection
+# averrating=>#averrating,r_anthaver

 [efiction.esteliel.de]
 ## Site dedicated to these categories/characters/ships
@@ -1136,14 +1177,6 @@ romance_label: Romance
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[ficwad.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
 [fictionmania.tv]
 ## website encoding(s) In theory, each website reports the character
 ## encoding they use for each page.  In practice, some sites report it
@@ -1200,6 +1233,14 @@ views_label:Views
 likes_label:Likes
 dislikes_label:Dislikes

+[ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [finestories.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1241,12 +1282,35 @@ universe_as_series: true
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/css/bir.png

+[forum.questionablequesting.com]
+## see [base_xenforoforum]
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [forums.spacebattles.com]
 ## see [base_xenforoforum]

 [forums.sufficientvelocity.com]
 ## see [base_xenforoforum]

+[harem.lucifael.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
@@ -1359,6 +1423,19 @@ extra_titlepage_entries: eroticatags
 ## Site dedicated to these categories/characters/ships
 extracategories:Merlin

+[mujaji.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [national-library.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:West Wing
@@ -1460,15 +1537,16 @@ extracategories:My Little Pony: Friendship is Magic
 ## Site dedicated to these categories/characters/ships
 extracategories:The Pretender

-[forum.questionablequesting.com]
-## see [base_xenforoforum]
+[quotev.com]
+extra_valid_entries:pages,readers,reads,favorites,searchtags,comments
+pages_label:Pages
+readers_label:Readers
+reads_label:Reads
+favorites_label:Favorites
+searchtags_label:Search Tags
+comments_label:Comments

-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
+include_in_category:category,searchtags

 [samandjack.net]
 ## Some sites require login (or login for some rated stories) The
@@ -1702,6 +1780,12 @@ extraships:Draco Malfoy/Ginny Weasley
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1

+[www.deepinmysoul.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.destinysgateway.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -2056,6 +2140,8 @@ extracategories:Psych
 extracategories:Queer as Folk

 [quotev.com]
+user_agent:
+slow_down_sleep_time:2
 extra_valid_entries:pages,readers,reads,favorites,searchtags,comments
 pages_label:Pages
 readers_label:Readers
@@ -2132,23 +2218,9 @@ extracategories:Lord of the Rings
 #username:YourName
 #password:yourpassword

-[www.twcslibrary.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
-## twcslibrary.net (ab)uses series as personal reading lists.
-collect_series: false
-
 [www.tthfanfic.org]
+user_agent:
+slow_down_sleep_time:2
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -2183,6 +2255,22 @@ pairingcat_to_characters_ships:true
 ## instead be added to characters Buffy, Spike and ships Buffy/Spike
 romancecat_to_characters_ships:true

+[www.twcslibrary.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## twcslibrary.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.twilightarchives.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Twilight
@@ -166,6 +166,8 @@ default_prefs['allow_custcol_from_ini'] = True

 default_prefs['std_cols_newonly'] = {}
 default_prefs['set_author_url'] = True
+default_prefs['includecomments'] = False
+default_prefs['anth_comments_newonly'] = True

 default_prefs['imapserver'] = ''
 default_prefs['imapuser'] = ''
@@ -138,6 +138,11 @@ import adapter_buffygilescom
 import adapter_andromedawebcom
 import adapter_artemisfowlcom
 import adapter_naiceanilmenet
+import adapter_deepinmysoulnet
+import adapter_haremlucifaelcom
+import adapter_kiarepositorymujajinet
+import adapter_fanfictionlucifaelcom
+import adapter_adultfanfictionorg

 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -0,0 +1,375 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+###   Written by GComyn
+################################################################################
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import sys
+
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+################################################################################
+
+def getClass():
+    return AdultFanFictionOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class AdultFanFictionOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        logger.debug("AdultFanFictionOrgAdapter.__init__ - url='{0}'".format(url))
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                            # Most sites that claim to be
+                            # iso-8859-1 (and some that claim to be
+                            # utf8) are really windows-1252.
+
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+
+        # get storyId from url
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+
+        #Setting the 'Zone' for each "Site"
+        self.zone = self.parsedUrl.netloc.split('.')[0]
+
+        # normalized story URL. (checking self.zone against list
+        # removed--it was redundant w/getAcceptDomains and
+        # getSiteURLPattern both)
+        self._setURL('http://' + self.zone + '.' + self.getBaseDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
+
+        # Each adapter needs to have a unique site abbreviation.
+        #self.story.setMetadata('siteabbrev',self.getSiteAbbrev())
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev',self.zone+'aff')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+        
+
+    ##This method will be moved to the sub-adapters
+    # @classmethod
+    # def getSiteAbbrev(self):
+    #     return self.zone+'aff'
+
+    ## Added because adult-fanfiction.org does send you to
+    ## www.adult-fanfiction.org when you go to it and it also moves
+    ## the site & examples down the web service front page so the
+    ## first screen isn't dominated by 'adult' links.
+    def getBaseDomain(self):
+        return 'adult-fanfiction.org'
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.adult-fanfiction.org'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        # mobile.fimifction.com isn't actually a valid domain, but we can still get the story id from URLs anyway
+        return ['anime.adult-fanfiction.org',
+                'anime2.adult-fanfiction.org',
+                'bleach.adult-fanfiction.org',
+                'books.adult-fanfiction.org',
+                'buffy.adult-fanfiction.org',
+                'cartoon.adult-fanfiction.org',
+                'celeb.adult-fanfiction.org',
+                'comics.adult-fanfiction.org',
+                'ff.adult-fanfiction.org',
+                'games.adult-fanfiction.org',
+                'hp.adult-fanfiction.org',
+                'inu.adult-fanfiction.org',
+                'lotr.adult-fanfiction.org',
+                'manga.adult-fanfiction.org',
+                'movies.adult-fanfiction.org',
+                'naruto.adult-fanfiction.org',
+                'ne.adult-fanfiction.org',
+                'original.adult-fanfiction.org',
+                'tv.adult-fanfiction.org',
+                'xmen.adult-fanfiction.org',
+                'ygo.adult-fanfiction.org',
+                'yuyu.adult-fanfiction.org']
+
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return ("http://anime.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://anime2.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://bleach.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://books.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://buffy.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://cartoon.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://celeb.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://comics.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://ff.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://games.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://hp.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://inu.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://lotr.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://manga.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://movies.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://naruto.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://ne.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://original.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://tv.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://xmen.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://ygo.adult-fanfiction.org/story.php?no=123456789 "
+              + "http://yuyu.adult-fanfiction.org/story.php?no=123456789")
+
+    def getSiteURLPattern(self):
+        return r'http?://(anime|anime2|bleach|books|buffy|cartoon|celeb|comics|ff|games|hp|inu|lotr|manga|movies|naruto|ne|original|tv|xmen|ygo|yuyu)\.adult-fanfiction\.org/story\.php\?no=\d+$'
+
+    ##This is not working right now, so I'm commenting it out, but leaving it for future testing
+    ## Login seems to be reasonably standard across eFiction sites.
+    #def needToLoginCheck(self, data):
+        ##This adapter will always require a login
+    #    return True
+
+#    <form name="login" method="post" action="">
+#      <div class="top">E-mail: <span id="sprytextfield1">
+#        <input name="email" type="text" id="email" size="20" maxlength="255" />
+#        <span class="textfieldRequiredMsg">Email is required.</span><span class="textfieldInvalidFormatMsg">Invalid E-mail.</span></span></div>
+#      <div class="top">Password: <span id="sprytextfield2">
+#        <input name="pass1" type="password" id="pass1" size="20" maxlength="32" />
+#        <span class="textfieldRequiredMsg">password is required.</span><span class="textfieldMinCharsMsg">Minimum 8 characters8.</span><span class="textfieldMaxCharsMsg">Exceeded 32 characters.</span></span></div>
+#      <div class="top"><br /> <input name="loginsubmittop" type="hidden" id="loginsubmit" value="TRUE" />
+#        <input type="submit" value="Login" />
+#      </div>
+#    </form>
+
+
+    ##This is not working right now, so I'm commenting it out, but leaving it for future testing
+    #def performLogin(self, url, soup):
+    #    params = {}
+
+    #    if self.password:
+    #        params['email'] = self.username
+    #        params['pass1'] = self.password
+    #    else:
+    #        params['email'] = self.getConfig("username")
+    #        params['pass1'] = self.getConfig("password")
+    #    params['submit'] = 'Login'
+
+    #    # copy all hidden input tags to pick up appropriate tokens.
+    #    for tag in soup.findAll('input',{'type':'hidden'}):
+    #        params[tag['name']] = tag['value']
+
+    #    logger.debug("Will now login to URL {0} as {1} with password: {2}".format(url, params['email'],params['pass1']))
+
+    #    d = self._postUrl(url, params, usecache=False)
+    #    d = self._fetchUrl(url, params, usecache=False)
+    #    soup = self.make_soup(d)
+        
+        #if not (soup.find('form', {'name' : 'login'}) == None):
+        #    logger.info("Failed to login to URL %s as %s" % (url, params['email']))
+        #    raise exceptions.FailedToLogin(url,params['email'])
+        #    return False
+        #else:
+    #    return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def doExtractChapterUrlsAndMetadata(self, get_cover=True):
+
+        ## You need to have your is_adult set to true to get this story
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code in 404:
+                raise exceptions.StoryDoesNotExist("Code: 404. %s"%self.url)
+            elif e.code == 410:
+                raise exceptions.StoryDoesNotExist("Code: 410. %s"%self.url)
+            elif e.code == 401:
+                self.needToLogin = True
+                data = ''
+            else:
+                raise e
+
+        if "The dragons running the back end of the site can not seem to find the story you are looking for." in data:
+            raise exceptions.StoryDoesNotExist(self.zone+'.'+self.getBaseDomain() 
+                                               +" says: The dragons running the back end of the site can not seem to find the story you are looking for.")
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = self.make_soup(data)
+        
+        ##This is not working right now, so I'm commenting it out, but leaving it for future testing
+        #self.performLogin(url, soup)
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        ## Some of the titles have a backslash on the story page, but not on the Author's page
+        ## So I am removing it from the title, so it can be found on the Author's page further in the code.
+        ## Also, some titles may have extra spaces '  ', and the search on the Author's page removes them,
+        ## so I have to here as well. I used multiple replaces to make sure, since I did the same below.
+        a = soup.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a).replace('\\','').replace('  ',' ').replace('  ',' ').replace('  ',' ').strip())
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"profile.php\?no=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl',a['href'])
+        self.story.setMetadata('author',stripHTML(a))
+
+        # Find the chapters:
+        chapters = soup.find('div',{'id':'snav'})
+        for i, chapter in enumerate(chapters.findAll('a')):
+            self.chapterUrls.append((stripHTML(chapter),self.url+'&chapter='+str(i+1)))
+        
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+
+        ##The story page does not give much Metadata, so we go to the Author's page
+        
+        ##Get the first Author page to see if there are multiple pages. 
+        ##AFF doesn't care if the page number is larger than the actual pages, 
+        ##it will continue to show the last page even if the variable is larger than the actual page
+        author_Url = self.story.getMetadata('authorUrl')+'&view=story&zone='+self.zone+'&page=1'
+
+        ##I'm resetting the author page to the zone for this story
+        self.story.setMetadata('authorUrl',author_Url)
+
+        logger.debug('Getting the author page: {0}'.format(author_Url))
+        try:
+            adata = self._fetchUrl(author_Url)
+        except urllib2.HTTPError, e:
+            if e.code in 404:
+                raise exceptions.StoryDoesNotExist("Author Page: Code: 404. %s"%author_Url)
+            elif e.code == 410:
+                raise exceptions.StoryDoesNotExist("Author Page: Code: 410. %s"%author_Url)
+            else:
+                raise e
+
+        if "The member you are looking for does not exist." in adata:
+            raise exceptions.StoryDoesNotExist(self.zone+'.'+self.getBaseDomain() +" says: The member you are looking for does not exist.")
+
+        asoup = self.make_soup(adata)
+
+        ##Getting the number of pages
+        pages=asoup.find('div',{'class' : 'pagination'}).findAll('li')[-1].find('a')
+        if not pages == None:
+            pages = pages['href'].split('=')[-1]
+        else:
+            pages = 0
+        logger.info(pages)
+        ##If there is only 1 page of stories, check it to get the Metadata, 
+        if pages == 0:
+            a = asoup.findAll('li')
+            for lc2 in a:
+                if lc2.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$")):
+                    break
+        ## otherwise go through the pages 
+        else:
+            page=1
+            i=0
+            while i == 0:
+                ##We already have the first page, so if this is the first time through, skip getting the page
+                if page != 1:
+                    author_Url = self.story.getMetadata('authorUrl')+'&view=story&zone='+self.zone+'&page='+str(page)
+                    logger.debug('Getting the author page: {0}'.format(author_Url))
+                    try:
+                        adata = self._fetchUrl(author_Url)
+                    except urllib2.HTTPError, e:
+                        if e.code in 404:
+                            raise exceptions.StoryDoesNotExist("Author Page: Code: 404. %s"%author_Url)
+                        elif e.code == 410:
+                            raise exceptions.StoryDoesNotExist("Author Page: Code: 410. %s"%author_Url)
+                        else:
+                            raise e
+                    ##This will probably never be needed, since AFF doesn't seem to care what number you put as 
+                    ## the page number, it will default to the last page, even if you use 1000, for an author
+                    ## that only hase 5 pages of stories, but I'm keeping it in to appease Saint Justin Case (just in case).
+                    if "The member you are looking for does not exist." in adata:
+                        raise exceptions.StoryDoesNotExist(self.zone+'.'+self.getBaseDomain() +" says: The member you are looking for does not exist.")
+    
+                asoup = self.make_soup(adata)
+    
+                a = asoup.findAll('li')
+                for lc2 in a:
+                    if lc2.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$")):
+                        i=1
+                        break
+                page = page + 1
+                if page > pages:
+                    break
+
+        ##Split the Metadata up into a list
+        ##We have to change the soup type to a string, then remove the newlines, and double spaces, 
+        ##then changes the <br/> to '-:-', which seperates the different elemeents.
+        ##Then we strip the HTML elements from the string.
+        ##There is also a double <br/>, so we have to fix that, then remove the leading and trailing '-:-'.
+        ##They are always in the same order.
+        liMetadata = stripHTML(str(lc2).replace('\n','').replace('\r','').replace('\t',' ').replace('  ',' ').replace('  ',' ').replace('  ',' ').replace(r'<br/>','-:-'))
+        liMetadata = liMetadata.replace(r'-:--:-','-:-').strip('-:-').strip('-:-')
+
+        for i, value in enumerate(liMetadata.split('-:-')):
+            ##The item 6 is the reviews... We are disregarding them.
+            ##The item 7 is the 'Dragon Prints'... not sure what they are, so disregarding them.
+            ##The 0 item is the title
+            if i == 0:
+                if value <> self.story.getMetadata('title'):
+                    raise exceptions.StoryDoesNotExist('Did not find story in author story list: {0}'.format(author_Url))
+            elif i == 1:
+                ##Get the description
+                self.story.setMetadata('description',stripHTML(value.strip()))
+            elif i == 2:
+                ##The Get the Category
+                self.story.setMetadata('category',value.replace(r'&gt;',r'>').replace(r'Located :',r'').strip())
+            elif i == 3:
+                ##Get the Erotic Tags
+                value = stripHTML(value.replace(r'Content Tags :',r'')).strip()
+                for code in re.split(r'\s',value):
+                    self.story.addToList('eroticatags',code)
+            elif i == 4:
+                ##Get the Posted Date
+                value = value.replace(r'Posted :',r'').strip()
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            elif i == 5:
+                ##Get the 'Updated' Edited date
+                ##AFF has the time for the Updated date, and we only want the date,
+                ##so we take the first 10 characters only
+                value = value.replace(r'Edited :',r'').strip()[0:10]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        #Since each chapter is on 1 page, we don't need to do anything special, just get the content of the page.
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = self.make_soup(self._fetchUrl(url))
+        chaptertag = soup.find('div',{'class' : 'pagination'}).parent.findNext('td')
+
+        if None == chaptertag:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,chaptertag)
@@ -187,7 +187,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        else:
            for a in alist:
                self.story.addToList('authorId',a['href'].split('/')[-1])
-                self.story.addToList('authorUrl',a['href'])
+                self.story.addToList('authorUrl','http://'+self.host+a['href'])
                self.story.addToList('author',a.text)

        byline = metasoup.find('h3',{'class':'byline'})
@@ -0,0 +1,300 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DeepInMySoulNetAdapter   ## XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DeepInMySoulNetAdapter(BaseSiteAdapter):  # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+
+
+        # normalized story URL.
+        # XXX Most sites don't have the /fiction part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/fiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dimsn')  ## XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.deepinmysoul.net'  # XXX
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/fiction/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+
+        d = self._fetchUrl(loginUrl, params)
+
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+
+        # fiction/viewstory.php?sid=1882&amp;warning=4
+        # fiction/viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        m = re.search(r"'fiction/viewstory.php\?sid=29(&amp;warning=4)'",data)
+        m = re.search(r"'fiction/viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = self.make_soup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagecontent'})
+
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+
+        self.story.setMetadata('title',stripHTML(a))
+
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl))
+
+            self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while 'label' not in defaultGetattr(value,'class'):
+                    svalue += unicode(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"fiction/viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = self.make_soup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^fiction/viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('fiction/viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = self.make_soup(self._fetchUrl(url))
+
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,div)
+
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+from base_efiction_adapter import BaseEfictionAdapter
+
+class FanfictionLucifaelComAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'fanfiction.lucifael.com'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'luci'
+
+    @classmethod
+    def getDateFormat(self):
+        return "%d/%m/%Y"
+
+def getClass():
+    return FanfictionLucifaelComAdapter
@@ -170,9 +170,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        elif 'Crossover' in categories[0]['href']:
            caturl = "https://%s%s"%(self.getSiteDomain(),categories[0]['href'])
            catsoup = self.make_soup(self._fetchUrl(caturl))
+            found = False
            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/.+?/\d+/")):
                self.story.addToList('category',stripHTML(a))
-            else:
+                found = True
+            if not found:
                # Fall back.  I ran across a story with a Crossver
                # category link to a broken page once.
                # http://www.fanfiction.net/s/2622060/1/
@@ -181,8 +183,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                for c in stripHTML(categories[0]).replace(" Crossover","").split(' + '):
                    self.story.addToList('category',c)

-
-
        a = soup.find('a', href=re.compile(r'https?://www\.fictionratings\.com/'))
        rating = a.string
        if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
@@ -206,6 +206,12 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        #     b.extract()
        metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
        #logger.debug("metatext:(%s)"%metatext)
+        
+        if 'Status: Complete' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
        metalist = metatext.split(" - ")
        #logger.debug("metalist:(%s)"%metalist)

@@ -240,36 +246,44 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
            self.story.setMetadata('dateUpdated',datetime.fromtimestamp(float(dates[0]['data-xutime'])))
        self.story.setMetadata('datePublished',datetime.fromtimestamp(float(dates[-1]['data-xutime'])))

-        donechars = False
+        # Meta key titles and the metadata they go into, if any.
+        metakeys = {
+            # These are already handled separately.
+            'Chapters':False,
+            'Status':False,
+            'id':False,
+            'Updated':False,
+            'Published':False,
+            'Reviews':'reviews',
+            'Favs':'favs',
+            'Follows':'follows',
+            'Words':'numWords',
+            }
+
+        chars_ships_list=[]
        while len(metalist) > 0:
-            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Updated:') or metalist[0].startswith('Published:'):
-                pass
-            elif  metalist[0].startswith('Reviews'):
-                self.story.setMetadata('reviews',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Favs:'):
-                self.story.setMetadata('favs',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Follows:'):
-                self.story.setMetadata('follows',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Words'):
-                self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
-            elif not donechars:
-                # with 'pairing' support, pairings are bracketed w/o comma after
-                # [Caspian X, Lucy Pevensie] Edmund Pevensie, Peter Pevensie
-                self.story.extendList('characters',metalist[0].replace('[','').replace(']',',').split(','))
+            m = metalist.pop(0)
+            if ':' in m:
+                key = m.split(':')[0].strip()
+                if key in metakeys:
+                    if metakeys[key]:
+                        self.story.setMetadata(metakeys[key],m.split(':')[1].strip())
+                    continue
+            # no ':' or not found in metakeys
+            chars_ships_list.append(m)

-                l = metalist[0]
-                while '[' in l:
-                    self.story.addToList('ships',l[l.index('[')+1:l.index(']')].replace(', ','/'))
-                    l = l[l.index(']')+1:]
-
-                donechars = True
-            metalist=metalist[1:]
-
-        if 'Status: Complete' in metatext:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
+        # all because sometimes chars can have ' - ' in them.
+        chars_ships_text = (' - ').join(chars_ships_list)
+        # print("chars_ships_text:%s"%chars_ships_text)
+        # with 'pairing' support, pairings are bracketed w/o comma after
+        # [Caspian X, Lucy Pevensie] Edmund Pevensie, Peter Pevensie
+        self.story.extendList('characters',chars_ships_text.replace('[','').replace(']',',').split(','))

+        l = chars_ships_text
+        while '[' in l:
+            self.story.addToList('ships',l[l.index('[')+1:l.index(']')].replace(', ','/'))
+            l = l[l.index(']')+1:]                    
+        
        if get_cover:
            # Try the larger image first.
            cover_url = ""
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import re

 from base_xenforoforum_adapter import BaseXenForoForumAdapter

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+from base_efiction_adapter import BaseEfictionAdapter
+
+class HaremLucifaelComAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'harem.lucifael.com'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'seraglio'
+
+    @classmethod
+    def getDateFormat(self):
+        return "%d/%m/%Y"
+
+def getClass():
+    return HaremLucifaelComAdapter
@@ -0,0 +1,300 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return KiaRepositoryMujajiNetAdapter   ## XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class KiaRepositoryMujajiNetAdapter(BaseSiteAdapter):  # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+
+
+        # normalized story URL.
+        # XXX Most sites don't have the /fiction part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/repository/viewstory.php?sid='+self.story.getMetadata('storyId'))
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','kia')  ## XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"   ## XXX
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'mujaji.net'  # XXX
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/repository/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/repository/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/repository/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+
+        d = self._fetchUrl(loginUrl, params)
+
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+
+        # fiction/viewstory.php?sid=1882&amp;warning=4
+        # fiction/viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        m = re.search(r"'repository/viewstory.php\?sid=29(&amp;warning=4)'",data)
+        m = re.search(r"'repository/viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = self.make_soup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+
+        self.story.setMetadata('title',stripHTML(a))
+
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/repository/'+chapter['href']+addurl))
+
+            self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while 'label' not in defaultGetattr(value,'class'):
+                    svalue += unicode(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"repository/viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = self.make_soup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^repository/viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('repository/viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = self.make_soup(self._fetchUrl(url))
+
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,div)
+
@@ -144,6 +144,9 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
            else:
                raise e

+        if "This submission is awaiting moderator's approval" in data1:
+            raise exceptions.StoryDoesNotExist("This submission is awaiting moderator's approval. %s"%self.url)
+
        # author
        a = soup1.find("span", "b-story-user-y")
        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'][0])
@@ -459,7 +459,22 @@ class Chapter(object):
                .strip(u'| \n')
        except AttributeError:
            raise ParsingError(u'Failed to locate date.')
-        date = makeDate(dateText, '%d.%m.%Y')
+
+        # The site uses Europe/Moscow (MSK, UTC+0300) server time.
+        def todayInMoscow():
+            now = datetime.datetime.now() + datetime.timedelta(hours=3)
+            today = datetime.datetime(now.year, now.month, now.day)
+            return today
+
+        def parseDateText(text):
+            if text == u'Вчера':
+                return todayInMoscow() - datetime.timedelta(days=1)
+            elif text == u'Сегодня':
+                return todayInMoscow()
+            else:
+                return makeDate(text, '%d.%m.%Y')
+
+        date = parseDateText(dateText)
        return date

    def _getInfoBarElement(self):
@@ -112,6 +112,11 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
        # need(or easier) to pull other metadata from the author's list page.
        authsoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl')))

+        # remove author profile incase they've put the story URL in their bio.
+        profile = authsoup.find('div',{'id':'profile'})
+        if profile: # in case it changes.
+            profile.extract()
+
        ## Title
        titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
        self.story.setMetadata('title',stripHTML(titlea))
@@ -132,9 +132,9 @@ class BaseSiteAdapter(Configurable):

    def set_cookiejar(self,cj):
        self.cookiejar = cj
+        saveheaders = self.opener.addheaders
        self.opener = u2.build_opener(u2.HTTPCookieProcessor(self.cookiejar),GZipProcessor())
-        self.opener.addheaders = [('User-Agent', self.getConfig('user_agent')),
-                                  ('X-Clacks-Overhead','GNU Terry Pratchett')]
+        self.opener.addheaders = saveheaders
        
    def load_cookiejar(self,filename):
        '''
@@ -393,6 +393,13 @@ class BaseSiteAdapter(Configurable):
                        data = self.getChapterText(url)
                        # if had to fetch and has existing chapters
                        newchap = bool(self.oldchapters or self.oldchaptersmap)
+
+                    if index == 0 and self.getConfig('always_reload_first_chapter'):
+                        data = self.getChapterText(url)
+                        # first chapter is rarely marked new
+                        # anyway--only if it's replaced during an
+                        # update.
+                        newchap = False
                        
                    self.story.addChapter(url,
                                          removeEntities(title),
@@ -85,7 +85,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):

    @classmethod
    def getSiteURLPattern(self):
-        return r"http://(www\.)?%s%s/%s\?sid=(?P<storyId>\d+)" % (self.getSiteDomain(), self.getPathToArchive(), self.getViewStoryPhpName())
+        return r"https?://(www\.)?%s%s/%s\?sid=(?P<storyId>\d+)" % (self.getSiteDomain(), self.getPathToArchive(), self.getViewStoryPhpName())

    @classmethod
    def getEncoding(cls):
@@ -196,8 +196,8 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
            soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above)

        if threadmark_chaps or self.getConfig('always_use_forumtags'):
-            ## only  use  tags if  threadmarks  for  chapters or
-            for tag in topsoup.findAll('a',{'class':'tag'}):
+            ## only use tags if threadmarks for chapters or always_use_forumtags is on.
+            for tag in topsoup.findAll('a',{'class':'tag'}) + topsoup.findAll('span',{'class':'prefix'}):
                tstr = stripHTML(tag)
                if self.getConfig('capitalize_forumtags'):
                    tstr = tstr.title()
@@ -229,7 +229,7 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
                if ( url.startswith(self.getURLPrefix()) or
                     url.startswith('http://'+self.getSiteDomain()) or
                     url.startswith('https://'+self.getSiteDomain()) ) and \
-                     ( '/posts/' in url or '/threads/' in url or 'showpost.php' in url):
+                     ( '/posts/' in url or '/threads/' in url or 'showpost.php' in url or 'goto/post' in url):

                    # brute force way to deal with SB's http->https change when hardcoded http urls.
                    url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix())
@@ -237,6 +237,9 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
                    # http://forums.spacebattles.com/showpost.php?p=4755532&postcount=9
                    url = re.sub(r'showpost\.php\?p=([0-9]+)(&postcount=[0-9]+)?',r'/posts/\1/',url)

+                    # http://forums.spacebattles.com/goto/post?id=15222406#post-15222406
+                    url = re.sub(r'/goto/post\?id=([0-9]+)(#post-[0-9]+)?',r'/posts/\1/',url)
+
                    url = re.sub(r'(^[\'"]+|[\'"]+$)','',url) # strip leading or trailing '" from incorrect quoting.
                    url = re.sub(r'like$','',url) # strip 'like' if incorrect 'like' link instead of proper post URL.

@@ -176,6 +176,7 @@ def get_valid_set_options():
                            None,boollist),
               
               'include_logpage':(None,['epub'],boollist+['smart']),
+               'logpage_at_end':(None,['epub'],boollist),
               
               'windows_eol':(None,['txt'],boollist),
               
@@ -188,6 +189,7 @@ def get_valid_set_options():
               'minimum_threadmarks':(base_xenforo_list,None,None),
               'first_post_title':(base_xenforo_list,None,None),
               'always_include_first_post':(base_xenforo_list,None,boollist),
+               'always_reload_first_chapter':(base_xenforo_list,None,boollist),
               }

    return dict(valdict)
@@ -282,6 +284,7 @@ def get_valid_keywords():
                 'image_max_size',
                 'include_images',
                 'include_logpage',
+                 'logpage_at_end',
                 'include_subject_tags',
                 'include_titlepage',
                 'include_tocpage',
@@ -25,7 +25,7 @@
 ## titlepage_entries: category,genre, status,dateUpdated,rating
 ## [epub]
 ## # overrides defaults & site section
-## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated
 ## [www.whofic.com:epub]
 ## # overrides defaults, site section & format section
 ## titlepage_entries: category,genre, status,datePublished
@@ -509,6 +509,20 @@ first_post_title:First Post
 ## if threadmarks are used.  Can result in a duplicated chapter.
 always_include_first_post:false

+## In normal operation, when updating an existing epub, old chapters
+## will be reused as-is.  Normally, that works fine, but forum stories
+## sometimes have an index post as the first 'chapter', and the
+## version in the first chapter gets out of sync.
+##
+## If always_reload_first_chapter:true, then the first chapter will
+## always be downloaded again (ie, reloaded).  It will NOT be maked
+## '(new)' (see mark_new_chapters).  Because it is reloaded, manual
+## edits made to the first chapter will be lost.
+##
+## While intended for base_xenforoforum sites, this setting can be
+## applied to other sites.
+always_reload_first_chapter:false
+
 ## In normal operation, forumtags will only be populated when
 ## threadmarks are used for chapters (see minimum_threadmarks above).
 ## When always_use_forumtags:true, always populate forumtags.
@@ -590,6 +604,11 @@ include_logpage: false
 ## end up with Completed stories that have just one logpage entry.
 #include_logpage: smart

+## By default, logpage is placed before the story chapters.  This
+## setting, if true, will place the logpage after the chapters
+## instead.
+logpage_at_end: false
+
 ## items to include in the log page Empty metadata entries, or those
 ## that haven't changed since the last update, will *not* appear, even
 ## if in the list.  You can include extra text or HTML that will be
@@ -754,6 +773,19 @@ extratags: FanFiction,Testing,Text
 [test1.com:html]
 extratags: FanFiction,Testing,HTML

+[adult-fanfiction.org]
+extra_valid_entries:eroticatags,disclaimer
+eroticatags_label:Erotica Tags
+disclaimer_label:Disclaimer
+extra_titlepage_entries:eroticatags,disclaimer
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [archive.skyehawke.com]

 [archiveofourown.org]
@@ -1124,14 +1156,6 @@ romance_label: Romance
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[ficwad.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
 [fictionmania.tv]
 ## website encoding(s) In theory, each website reports the character
 ## encoding they use for each page.  In practice, some sites report it
@@ -1188,6 +1212,14 @@ views_label:Views
 likes_label:Likes
 dislikes_label:Dislikes

+[ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [finestories.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1229,12 +1261,35 @@ universe_as_series: true
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/css/bir.png

+[forum.questionablequesting.com]
+## see [base_xenforoforum]
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [forums.spacebattles.com]
 ## see [base_xenforoforum]

 [forums.sufficientvelocity.com]
 ## see [base_xenforoforum]

+[harem.lucifael.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
@@ -1347,6 +1402,19 @@ extra_titlepage_entries: eroticatags
 ## Site dedicated to these categories/characters/ships
 extracategories:Merlin

+[mujaji.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [national-library.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:West Wing
@@ -1448,15 +1516,16 @@ extracategories:My Little Pony: Friendship is Magic
 ## Site dedicated to these categories/characters/ships
 extracategories:The Pretender

-[forum.questionablequesting.com]
-## see [base_xenforoforum]
+[quotev.com]
+extra_valid_entries:pages,readers,reads,favorites,searchtags,comments
+pages_label:Pages
+readers_label:Readers
+reads_label:Reads
+favorites_label:Favorites
+searchtags_label:Search Tags
+comments_label:Comments

-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
+include_in_category:category,searchtags

 [samandjack.net]
 ## Some sites require login (or login for some rated stories) The
@@ -1690,6 +1759,12 @@ extraships:Draco Malfoy/Ginny Weasley
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1

+[www.deepinmysoul.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.destinysgateway.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -2038,6 +2113,8 @@ extracategories:Psych
 extracategories:Queer as Folk

 [quotev.com]
+user_agent:
+slow_down_sleep_time:2
 extra_valid_entries:pages,readers,reads,favorites,searchtags,comments
 pages_label:Pages
 readers_label:Readers
@@ -2114,23 +2191,9 @@ extracategories:Lord of the Rings
 #username:YourName
 #password:yourpassword

-[www.twcslibrary.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
-## twcslibrary.net (ab)uses series as personal reading lists.
-collect_series: false
-
 [www.tthfanfic.org]
+user_agent:
+slow_down_sleep_time:2
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -2165,6 +2228,22 @@ pairingcat_to_characters_ships:true
 ## instead be added to characters Buffy, Spike and ships Buffy/Spike
 romancecat_to_characters_ships:true

+[www.twcslibrary.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## twcslibrary.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.twilightarchives.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Twilight
@@ -21,6 +21,10 @@ def get_dcsource(inputio):
 def get_dcsource_chaptercount(inputio):
    return get_update_data(inputio,getfilecount=True,getsoups=False)[:2] # (source,filecount)

+def get_cover_data(inputio):
+    # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
+    return get_update_data(inputio,getfilecount=True,getsoups=False)[4]
+
 def get_update_data(inputio,
                    getfilecount=True,
                    getsoups=True):
@@ -50,26 +54,32 @@ def get_update_data(inputio,
        if item.getAttribute("type") == "cover":
            # there is a cover (x)html file, save the soup for it.
            href=relpath+item.getAttribute("href")
-            oldcoverhtmlhref = href
-            oldcoverhtmldata = epub.read(href)
-            oldcoverhtmltype = "application/xhtml+xml"
-            for item in contentdom.getElementsByTagName("item"):
-                if( relpath+item.getAttribute("href") == oldcoverhtmlhref ):
-                    oldcoverhtmltype = item.getAttribute("media-type")
-                    break
-            soup = bs.BeautifulSoup(oldcoverhtmldata.decode("utf-8"),"html5lib")
            src = None
-            # first img or image tag.
-            imgs = soup.findAll('img')
-            if imgs:
-                src = get_path_part(href)+imgs[0]['src']
-            else:
-                imgs = soup.findAll('image')
+            try:
+                oldcoverhtmlhref = href
+                oldcoverhtmldata = epub.read(href)
+                oldcoverhtmltype = "application/xhtml+xml"
+                for item in contentdom.getElementsByTagName("item"):
+                    if( relpath+item.getAttribute("href") == oldcoverhtmlhref ):
+                        oldcoverhtmltype = item.getAttribute("media-type")
+                        break
+                soup = bs.BeautifulSoup(oldcoverhtmldata.decode("utf-8"),"html5lib")
+                # first img or image tag.
+                imgs = soup.findAll('img')
                if imgs:
-                    src=get_path_part(href)+imgs[0]['xlink:href']
+                    src = get_path_part(href)+imgs[0]['src']
+                else:
+                    imgs = soup.findAll('image')
+                    if imgs:
+                        src=get_path_part(href)+imgs[0]['xlink:href']
+
+                if not src:
+                    continue
+            except Exception as e:
+                ## Calibre's Polish Book corrupts sub-book covers.
+                logger.warn("Cover (x)html file %s not found"%href)
+                logger.warn("Exception: %s"%(unicode(e)))

-            if not src:
-                continue
            try:
                # remove all .. and the path part above it, if present.
                # Mostly for epubs edited by Sigil.
@@ -84,7 +94,7 @@ def get_update_data(inputio,
                oldcover = (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
            except Exception as e:
                logger.warn("Cover Image %s not found"%src)
-                logger.warn("Exception: %s"%(unicode(e)),exc_info=True)
+                logger.warn("Exception: %s"%(unicode(e)))

    filecount = 0
    soups = [] # list of xhmtl blocks
@@ -125,7 +125,10 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict

 def get_urls_from_text(data,configuration=None,normalize=False):
    urls = collections.OrderedDict()
-    data=unicode(data)
+    try:
+        data = unicode(data)
+    except UnicodeDecodeError:
+        data=data.decode('utf8') ## for when called outside calibre.

    if not configuration:
        configuration = Configuration(["test1.com"],"EPUB",lightweight=True)
@@ -229,7 +232,7 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
                if part.get_content_type() == 'text/html':
                    urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
            except Exception as e:
-                logger.error("Failed to read email content: %s"%e)
+                logger.error("Failed to read email content: %s"%e,exc_info=True)
        #logger.debug "urls:%s"%get_urls_from_text(get_first_text_block(email_message))

        if urllist and markread:
@@ -46,7 +46,7 @@ class EpubWriter(BaseStoryWriter):
        BaseStoryWriter.__init__(self, config, story)

        self.EPUB_CSS = string.Template('''${output_css}''')
-        
+
        self.EPUB_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
@@ -117,7 +117,7 @@ ${value}<br />
        self.EPUB_TOC_ENTRY = string.Template('''
 <a href="file${index}.xhtml">${chapter}</a><br />
 ''')
-                          
+
        self.EPUB_TOC_PAGE_END = string.Template('''
 </div>
 </body>
@@ -156,15 +156,15 @@ ${value}<br />
        self.EPUB_LOG_UPDATE_START = string.Template('''
 <p class='log_entry'>
 ''')
-                          
+
        self.EPUB_LOG_ENTRY = string.Template('''
 <b>${label}:</b> <span id="${id}">${value}</span>
 ''')
-                          
+
        self.EPUB_LOG_UPDATE_END = string.Template('''
 </p><hr />
 ''')
-                          
+
        self.EPUB_LOG_PAGE_END = string.Template('''
 </body>
 </html>
@@ -184,7 +184,7 @@ div { margin: 0pt; padding: 0pt; }
 <img src="${coverimg}" alt="cover"/>
 </div></body></html>
 ''')
-        
+
    def writeLogPage(self, out):
        """
        Write the log page, but only include entries that there's
@@ -201,7 +201,7 @@ div { margin: 0pt; padding: 0pt; }
            END = string.Template(self.getConfig("logpage_end"))
        else:
            END = self.EPUB_LOG_PAGE_END
-        
+
        # if there's a self.story.logfile, there's an existing log
        # to add to.
        if self.story.logfile:
@@ -234,7 +234,7 @@ div { margin: 0pt; padding: 0pt; }
                pass

        return values
-        
+
    def _makeLogEntry(self, oldvalues={}):
        if self.hasConfig("logpage_update_start"):
            START = string.Template(self.getConfig("logpage_update_start"))
@@ -275,16 +275,16 @@ div { margin: 0pt; padding: 0pt; }
                # mostly it makes it easy to tell when you get the
                # keyword wrong.
                retval = retval + entry
-                
+
        retval = retval + END.substitute(self.story.getAllMetadata())
-        
+
        if self.getConfig('replace_hr'):
            # replacing a self-closing tag with a container tag in the
            # soup is more difficult than it first appears.  So cheat.
            retval = re.sub("<hr[^>]*>","<div class='center'>* * *</div>",retval)
-            
+
        return retval
-                
+
    def writeStoryImpl(self, out):

        ## Python 2.5 ZipFile is rather more primative than later
@@ -305,7 +305,7 @@ div { margin: 0pt; padding: 0pt; }
        ## Re-open file for content.
        outputepub = ZipFile(zipio, 'a', compression=ZIP_DEFLATED)
        outputepub.debug=3
-        
+
        ## Create META-INF/container.xml file.  The only thing it does is
        ## point to content.opf
        containerdom = getDOMImplementation().createDocument(None, "container", None)
@@ -332,7 +332,7 @@ div { margin: 0pt; padding: 0pt; }
            self.getMetadata('site'),
            self.story.getList('authorId')[0],
            self.getMetadata('storyId'))
-        
+
        contentdom = getDOMImplementation().createDocument(None, "package", None)
        package = contentdom.documentElement
        package.setAttribute("version","2.0")
@@ -374,12 +374,12 @@ div { margin: 0pt; padding: 0pt; }
            metadata.appendChild(newTag(contentdom,"dc:date",
                                        attrs={"opf:event":"publication"},
                                        text=self.story.getMetadataRaw('datePublished').strftime("%Y-%m-%d")))
-        
+
        if self.story.getMetadataRaw('dateCreated'):
            metadata.appendChild(newTag(contentdom,"dc:date",
                                        attrs={"opf:event":"creation"},
                                        text=self.story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d")))
-        
+
        if self.story.getMetadataRaw('dateUpdated'):
            metadata.appendChild(newTag(contentdom,"dc:date",
                                        attrs={"opf:event":"modification"},
@@ -387,7 +387,7 @@ div { margin: 0pt; padding: 0pt; }
            metadata.appendChild(newTag(contentdom,"meta",
                                        attrs={"name":"calibre:timestamp",
                                               "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")}))
-        
+
        if self.getMetadata('description'):
            metadata.appendChild(newTag(contentdom,"dc:description",text=
                                        self.getMetadata('description')))
@@ -395,11 +395,11 @@ div { margin: 0pt; padding: 0pt; }
        for subject in self.story.getSubjectTags():
            metadata.appendChild(newTag(contentdom,"dc:subject",text=subject))

-                    
+
        if self.getMetadata('site'):
            metadata.appendChild(newTag(contentdom,"dc:publisher",
                                        text=self.getMetadata('site')))
-        
+
        if self.getMetadata('storyUrl'):
            metadata.appendChild(newTag(contentdom,"dc:identifier",
                                        attrs={"opf:scheme":"URL"},
@@ -415,7 +415,7 @@ div { margin: 0pt; padding: 0pt; }

        guide = None
        coverIO = None
-                
+
        coverimgid = "image0000"
        if not self.story.cover and self.story.oldcover:
            logger.debug("writer_epub: no new cover, has old cover, write image.")
@@ -427,7 +427,7 @@ div { margin: 0pt; padding: 0pt; }
             oldcoverimgdata) = self.story.oldcover
            outputepub.writestr(oldcoverhtmlhref,oldcoverhtmldata)
            outputepub.writestr(oldcoverimghref,oldcoverimgdata)
-            
+
            coverimgid = "image0"
            items.append((coverimgid,
                          oldcoverimghref,
@@ -441,8 +441,8 @@ div { margin: 0pt; padding: 0pt; }
            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
                                                                   "title":"Cover",
                                                                   "href":oldcoverhtmlhref}))
-            
-            
+
+

        if self.getConfig('include_images'):
            imgcount=0
@@ -459,7 +459,7 @@ div { margin: 0pt; padding: 0pt; }
                    # just the first image.
                    coverimgid = items[-1][0]

-        
+
        items.append(("style","OEBPS/stylesheet.css","text/css",None))

        if self.story.cover:
@@ -467,7 +467,7 @@ div { margin: 0pt; padding: 0pt; }
            # for it to work on Nook.
            items.append(("cover","OEBPS/cover.xhtml","application/xhtml+xml",None))
            itemrefs.append("cover")
-            # 
+            #
            # <meta name="cover" content="cover.jpg"/>
            metadata.appendChild(newTag(contentdom,"meta",{"content":coverimgid,
                                                           "name":"cover"}))
@@ -480,14 +480,14 @@ div { margin: 0pt; padding: 0pt; }
            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
                                                       "title":"Cover",
                                                       "href":"OEBPS/cover.xhtml"}))
-            
+
            if self.hasConfig("cover_content"):
                COVER = string.Template(self.getConfig("cover_content"))
            else:
                COVER = self.EPUB_COVER
            coverIO = StringIO.StringIO()
            coverIO.write(COVER.substitute(dict(self.story.getAllMetadata().items()+{'coverimg':self.story.cover}.items())))
-            
+
        if self.getConfig("include_titlepage"):
            items.append(("title_page","OEBPS/title_page.xhtml","application/xhtml+xml","Title Page"))
            itemrefs.append("title_page")
@@ -495,14 +495,13 @@ div { margin: 0pt; padding: 0pt; }
            items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents"))
            itemrefs.append("toc_page")

+        ## save where to insert logpage.
+        logpage_indices = (len(items),len(itemrefs))
+
        dologpage = ( self.getConfig("include_logpage") == "smart" and \
                          (self.story.logfile or self.story.getMetadataRaw("status") == "In-Progress") )  \
                     or self.getConfig("include_logpage") == "true"

-        if dologpage:
-            items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
-            itemrefs.append("log_page")
-            
        for index, chap in enumerate(self.story.getChapters(fortoc=True)):
            if chap.html:
                i=index+1
@@ -512,6 +511,13 @@ div { margin: 0pt; padding: 0pt; }
                              chap.title))
                itemrefs.append("file%04d"%i)

+        if dologpage:
+            if self.getConfig("logpage_at_end") == "true":
+                ## insert logpage after chapters.
+                logpage_indices = (len(items),len(itemrefs))
+            items.insert(logpage_indices[0],("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
+            itemrefs.insert(logpage_indices[1],"log_page")
+
        manifest = contentdom.createElement("manifest")
        package.appendChild(manifest)
        for item in items:
@@ -520,7 +526,7 @@ div { margin: 0pt; padding: 0pt; }
                                        attrs={'id':id,
                                               'href':href,
                                               'media-type':type}))
-        
+
        spine = newTag(contentdom,"spine",attrs={"toc":"ncx"})
        package.appendChild(spine)
        for itemref in itemrefs:
@@ -530,10 +536,10 @@ div { margin: 0pt; padding: 0pt; }
        # guide only exists if there's a cover.
        if guide:
            package.appendChild(guide)
-            
+
        # write content.opf to zip.
        contentxml = contentdom.toxml(encoding='utf-8')
-        
+
        # tweak for brain damaged Nook STR.  Nook insists on name before content.
        contentxml = contentxml.replace('<meta content="%s" name="cover"/>'%coverimgid,
                                        '<meta name="cover" content="%s"/>'%coverimgid)
@@ -557,11 +563,11 @@ div { margin: 0pt; padding: 0pt; }
                                attrs={"name":"dtb:totalPageCount", "content":"0"}))
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:maxPageNumber", "content":"0"}))
-        
+
        docTitle = tocncxdom.createElement("docTitle")
        docTitle.appendChild(newTag(tocncxdom,"text",text=self.getMetadata('title')))
        ncx.appendChild(docTitle)
-    
+
        tocnavMap = tocncxdom.createElement("navMap")
        ncx.appendChild(tocnavMap)

@@ -586,14 +592,14 @@ div { margin: 0pt; padding: 0pt; }
                navLabel.appendChild(newTag(tocncxdom,"text",text=stripHTML(title)))
                navPoint.appendChild(newTag(tocncxdom,"content",attrs={"src":href}))
                index=index+1
-        
+
        # write toc.ncx to zip file
        outputepub.writestr("toc.ncx",tocncxdom.toxml(encoding='utf-8'))
        tocncxdom.unlink()
        del tocncxdom

        # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata())) 
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata()))

        # write title page.
        if self.getConfig("titlepage_use_table"):
@@ -612,7 +618,7 @@ div { margin: 0pt; padding: 0pt; }
        if coverIO:
            outputepub.writestr("OEBPS/cover.xhtml",coverIO.getvalue())
            coverIO.close()
-            
+
        titlepageIO = StringIO.StringIO()
        self.writeTitlePage(out=titlepageIO,
                            START=TITLE_PAGE_START,
@@ -624,7 +630,7 @@ div { margin: 0pt; padding: 0pt; }
            outputepub.writestr("OEBPS/title_page.xhtml",titlepageIO.getvalue())
        titlepageIO.close()

-        # write toc page.  
+        # write toc page.
        tocpageIO = StringIO.StringIO()
        self.writeTOCPage(tocpageIO,
                          self.EPUB_TOC_PAGE_START,
@@ -645,12 +651,12 @@ div { margin: 0pt; padding: 0pt; }
            CHAPTER_START = string.Template(self.getConfig("chapter_start"))
        else:
            CHAPTER_START = self.EPUB_CHAPTER_START
-        
+
        if self.hasConfig('chapter_end'):
            CHAPTER_END = string.Template(self.getConfig("chapter_end"))
        else:
            CHAPTER_END = self.EPUB_CHAPTER_END
-        
+
        for index, chap in enumerate(self.story.getChapters()): # (url,title,html)
            if chap.html:
                #logger.debug('Writing chapter text for: %s' % chap.title)
@@ -23,7 +23,7 @@ setup(
    # Versions should comply with PEP440.  For a discussion on single-sourcing
    # the version across setup.py and the project code, see
    # https://packaging.python.org/en/latest/single_source_version.html
-    version="2.3.4",
+    version="2.3.6",

    description='A tool for downloading fanfiction to eBook formats',
    long_description=long_description,
@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanficfare
 application: fanficfare
-version: 2-3-04
+version: 2-3-06
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -34,13 +34,3 @@ handlers:

 - url: /.*
  script: main.app
-
-#builtins:
-#- datastore_admin: on
-
-libraries:
- name: django
-  version: "1.2"
-
- name: PIL
-  version: "1.1.7"
@@ -24,7 +24,7 @@
 	<div id='urlbox'>
 	  <div id='greeting'>
 	    <p>Hi, {{ nickname }}! This is FanFicFare, which makes reading stories from various websites
-	      much easier. </p>
+	      much easier by helping you download them to EBook files. </p>
 	  </div>

 	  <p>
@@ -35,7 +35,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>.  The
-	    <a href="http://2-3-03.fanficfare.appspot.com">previous version
+	    <a href="http://2-3-05.fanficfare.appspot.com">previous version
 	    </a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
@@ -75,7 +75,7 @@
      <div id='urlbox'>
 	<div id='greeting'>
 	  <p>
-	    This is a FanFicFare, which makes reading stories from various websites much easier. Before you
+	    This is a FanFicFare, which makes reading stories from various websites much easier by helping you download them to EBook files. Before you
 	    can start downloading fanfics, you need to login, so FanFicFare can remember your fanfics and store them.
 	  </p>
 	  <p><a href="{{ login_url }}">Login using Google account</a></p>
@@ -85,17 +85,17 @@
      
      <div id='typebox'>
 	  <p>
-	    <b>FanFicFare calibre Plugin</b>
+	    <b>FanFicFare Calibre Plugin</b>
 	    <br /><br />
 	    
 	    There's also a version of this downloader that runs inside
-	    the popular <a href="http://calibre-ebook.com/">calibre</a>
+	    the popular <a href="http://calibre-ebook.com/">Calibre</a>
 	    ebook management package as a plugin.
 	    
 	    <br /><br />
 	    
-	    Once you have calibre installed and running, inside
-	    calibre, you can go to 'Get plugins to enhance calibre' or
+	    Once you have Calibre installed and running, inside
+	    Calibre, you can go to 'Get plugins to enhance calibre' or
 	    'Get new plugins' and
 	    install <a href="http://www.mobileread.com/forums/showthread.php?t=259221">FanFicFare</a>.
 	    
@@ -30,20 +30,6 @@ import datetime
 import traceback
 from StringIO import StringIO

-## Just to shut up the appengine warning about "You are using the
-## default Django version (0.96). The default Django version will
-## change in an App Engine release in the near future. Please call
-## use_library() to explicitly select a Django version. For more
-## information see
-## http://code.google.com/appengine/docs/python/tools/libraries.html#Django"
-## Note that if you are using the SDK App Engine Launcher and hit an SDK
-## Console page first, you will get a django version mismatch error when you
-## to go hit one of the application pages.  Just change a file again, and
-## make sure to hit an app page before the SDK page to clear it.
-#os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
-#from google.appengine.dist import use_library
-#use_library('django', '1.2')
-
 from google.appengine.ext import db
 from google.appengine.api import taskqueue
 from google.appengine.api import users
Author	SHA1	Message	Date
Jim Miller	173fca8773	Bump versions.	2016-08-24 13:17:35 -05:00
Jim Miller	2ce514a38e	Fix for change to AO3 author links.	2016-08-24 09:57:11 -05:00
Jim Miller	d0e4999712	Force overall cover for Anthologies if any sub-books have covers to avoid Calibre Polish issues.	2016-08-20 14:05:36 -05:00
Jim Miller	5c8940d333	Fix ffnet category fall back.	2016-08-20 12:08:19 -05:00
Jim Miller	664684b04f	Add extra decode code for calling get_urls_from_imap() from outside calibre.	2016-08-20 00:10:18 -05:00
Jim Miller	75064b2267	Update translations.	2016-08-19 23:40:01 -05:00
Jim Miller	06d3fd9080	Fail more gracefully when anthology cover doesn't exist. Polish can remove them.	2016-08-18 19:33:22 -05:00
Jim Miller	c6cd9c57e2	Adding logpage_at_end INI option.	2016-08-17 12:14:14 -05:00
Jim Miller	1887cf2cb9	Fix for Publisher and Language metadata for Anthologies.	2016-08-14 11:22:32 -05:00
Jim Miller	2e6a1ef65d	Merge pull request #128 from PlushBeaver/masseffect2in-textual-dates Fix date parsing for masseffect2.in.	2016-08-13 21:06:32 -05:00
Dmitry Kozliuk	a0b276beb4	Fix date parsing for masseffect2.in. The site displays `Вчера' for yesterday and `Сегодня' for today now.	2016-08-14 01:39:14 +03:00
Jim Miller	1ac9e5d36c	Tweaks to adultfanfictionorg	2016-08-13 10:38:01 -05:00
Jim Miller	93712de4b7	Add options for controlling Anthology book comments on the Standard Columns config tab. (plugin only)	2016-08-12 21:40:55 -05:00
Jim Miller	e7941298b6	Add always_reload_first_chapter setting for base_xenforoforum stories with index chapter.	2016-08-11 12:18:30 -05:00
Jim Miller	65f286be99	Add always_reload_first_chapter setting for base_xenforoforum stories with index chapter.	2016-08-11 12:12:34 -05:00
Jim Miller	1030cf44af	Progress Bar - add extra timer loop for Win10 display.	2016-08-11 11:52:55 -05:00
Jim Miller	e9190b9a12	New sites from GComyn, including adult-fanfiction.org.	2016-08-10 14:09:02 -05:00
Jim Miller	dc63773fad	Update translations.	2016-08-09 13:27:30 -05:00
Jim Miller	710800f976	Make base_efiction code accept https story urls.	2016-08-09 09:44:27 -05:00
Jim Miller	3eca202567	Tweaks for tthfanfic.org, including a slow down.	2016-08-09 09:41:38 -05:00
Jim Miller	6d423e586f	For lierotica.com, throw a StoryDoesNotExist exception for 'This submission is awaiting moderator's approval'	2016-08-01 22:02:42 -05:00
Jim Miller	bda62aab9f	Tweaks for quotev.com, including a slow down.	2016-08-01 12:44:33 -05:00
Jim Miller	f6d65c334f	Bump versions.	2016-07-22 12:02:48 -05:00
Jim Miller	2c5371d95e	Update translations.	2016-07-22 11:59:42 -05:00
Jim Miller	6f4660763f	Save addheaders when setting cookiejar. For ffn referer.	2016-07-17 17:08:33 -05:00
Jim Miller	75f8f72266	Include 'prefix' tags in forumtags in base_xenforoforum.	2016-07-17 16:38:53 -05:00
Jim Miller	b1d689ba3e	Add n_anthaver and r_anthaver modes to custom_columns_settings for averaging metadata for anthologies before setting in integer and float calibre custom columns.	2016-07-12 14:57:10 -05:00
Jim Miller	39bb6e37f6	Add n_anthaver and r_anthaver modes to custom_columns_settings for averaging metadata for anthologies before setting in integer and float calibre custom columns.	2016-07-12 14:55:51 -05:00
Jim Miller	288f12afed	Add n_anthaver and r_anthaver modes to custom_columns_settings for averaging metadata for anthologies before setting in integer and float calibre custom columns.	2016-07-12 14:53:18 -05:00
Jim Miller	2a25aef7ac	Update translations.	2016-07-12 14:49:03 -05:00
Jim Miller	085fb47b08	Remove Django from app.yaml--old version going away.	2016-07-09 09:27:46 -05:00
Jim Miller	5fdcbab46a	Allow old goto/post chapter URLs in base_xenforoforum.	2016-07-05 23:22:25 -05:00
Jim Miller	2d83fa8f5d	Fix for SIYE when author puts story URL in bio.	2016-06-30 10:38:13 -05:00
Jim Miller	4a752e05e1	Change ffnet metadata colletion to allow for chars with (' - ') in them.	2016-06-26 11:34:36 -05:00