Fix for latest ffnet tweak.

Added tag fanfictionDownLoader-4.4.78 for changeset 3f7539804f25
Added tag calibre-plugin-1.7.49 for changeset 3f7539804f25
2026-09-16 11:47:12 +08:00 · 2013-10-23 19:31:28 -05:00 · 2013-10-19 11:24:42 -05:00 · 2013-10-19 11:24:28 -05:00 · 2013-10-19 11:24:18 -05:00 · 2013-10-18 15:01:26 -05:00
101 changed files with 896 additions and 338 deletions
@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-68
+version: 4-4-79
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -7,6 +7,16 @@ __license__   = 'GPL v3'
 __copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'

+import sys
+if sys.version_info >= (2, 7):
+    import logging
+    logger = logging.getLogger(__name__)
+    loghandler=logging.StreamHandler()
+    loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+    logger.addHandler(loghandler)
+    loghandler.setLevel(logging.DEBUG)
+    logger.setLevel(logging.DEBUG)
+
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase

@@ -26,7 +36,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
    description         = 'UI plugin to download FanFiction stories from various sites.'
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (1, 7, 35)
+    version             = (1, 7, 50)
    minimum_calibre_version = (0, 8, 57)

    #: This field defines the GUI plugin class that contains all the code
@@ -7,6 +7,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'

+import logging
+logger = logging.getLogger(__name__)
+
 import traceback, copy, threading
 from collections import OrderedDict

@@ -126,9 +129,15 @@ class ConfigWidget(QWidget):
        label.setOpenExternalLinks(True)
        self.l.addWidget(label)

-        tab_widget = QTabWidget(self)
-        self.l.addWidget(tab_widget)

+        self.scroll_area = QScrollArea(self)
+        self.scroll_area.setFrameShape(QScrollArea.NoFrame)
+        self.scroll_area.setWidgetResizable(True)
+        self.l.addWidget(self.scroll_area)
+
+        tab_widget = QTabWidget(self)
+        self.scroll_area.setWidget(tab_widget)
+        
        self.basic_tab = BasicTab(self, plugin_action)
        tab_widget.addTab(self.basic_tab, 'Basic')

@@ -169,6 +178,9 @@ class ConfigWidget(QWidget):
        prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
        prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
        prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
+        prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
+        prefs['mark'] = self.basic_tab.mark.isChecked()
        prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
@@ -177,6 +189,7 @@ class ConfigWidget(QWidget):
        prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
        prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
        prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
+        prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
        prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
        prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()

@@ -347,18 +360,38 @@ class BasicTab(QWidget):
        self.keeptags.setChecked(prefs['keeptags'])
        self.l.addWidget(self.keeptags)

+        self.suppressauthorsort = QCheckBox('Force Author into Author Sort?',self)
+        self.suppressauthorsort.setToolTip("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc.")
+        self.suppressauthorsort.setChecked(prefs['suppressauthorsort'])
+        self.l.addWidget(self.suppressauthorsort)
+
+        self.suppresstitlesort = QCheckBox('Force Title into Title Sort?',self)
+        self.suppresstitlesort.setToolTip("If checked, the title as given will be used for the Title Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'The Title' sort as 'Title, The', etc.")
+        self.suppresstitlesort.setChecked(prefs['suppresstitlesort'])
+        self.l.addWidget(self.suppresstitlesort)
+
        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
        self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
        self.l.addWidget(self.checkforseriesurlid)

+        self.checkforurlchange = QCheckBox("Check for changed Story URL?",self)
+        self.checkforurlchange.setToolTip("Warn you if an update will change the URL of an existing book.")
+        self.checkforurlchange.setChecked(prefs['checkforurlchange'])
+        self.l.addWidget(self.checkforurlchange)
+
        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
        self.l.addWidget(self.lookforurlinhtml)

-        self.showmarked = QCheckBox("Show added/updated books when finished?",self)
-        self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.mark = QCheckBox("Mark added/updated books when finished?",self)
+        self.mark.setToolTip("Mark added/updated books when finished.  Use with option below.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.mark.setChecked(prefs['mark'])
+        self.l.addWidget(self.mark)
+
+        self.showmarked = QCheckBox("Show Marked books when finished?",self)
+        self.showmarked.setToolTip("Show Marked added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
        self.showmarked.setChecked(prefs['showmarked'])
        self.l.addWidget(self.showmarked)

@@ -419,15 +452,17 @@ class BasicTab(QWidget):
        topl.addWidget(defs_gb)

        horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(cali_gb)
-        horz.addWidget(rej_gb)
-        
-        horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(gui_gb)
-        horz.addWidget(misc_gb)

+        horz.addWidget(cali_gb)
+
+        vert = QVBoxLayout()
+        vert.addWidget(gui_gb)
+        vert.addWidget(misc_gb)
+        vert.addWidget(rej_gb)
+        
+        horz.addLayout(vert)
+        
+        topl.addLayout(horz)
        topl.insertStretch(-1)
        
    def set_collisions(self):
@@ -503,9 +538,9 @@ class PersonalIniTab(QWidget):
        self.ini = QTextEdit(self)
        try:
            self.ini.setFont(QFont("Courier",
-                                   self.plugin_action.gui.font().pointSize()+1));
+                                   self.plugin_action.gui.font().pointSize()+1))
        except Exception as e:
-            print("Couldn't get font: %s"%e)
+            logger.error("Couldn't get font: %s"%e)
        self.ini.setLineWrapMode(QTextEdit.NoWrap)
        self.ini.setText(prefs['personal.ini'])
        self.l.addWidget(self.ini)
@@ -539,9 +574,9 @@ class ShowDefaultsIniDialog(QDialog):
        self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
        try:
            self.ini.setFont(QFont("Courier",
-                                   get_gui().font().pointSize()+1));
+                                   get_gui().font().pointSize()+1))
        except Exception as e:
-            print("Couldn't get font: %s"%e)
+            logger.error("Couldn't get font: %s"%e)
        self.ini.setLineWrapMode(QTextEdit.NoWrap)
        self.ini.setText(text)
        self.ini.setReadOnly(True)
@@ -7,9 +7,15 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'

+import logging
+logger = logging.getLogger(__name__)
+
 import traceback, re
 from functools import partial

+import logging
+logger = logging.getLogger(__name__)
+
 import urllib
 import email

@@ -164,7 +170,7 @@ class DroppableQTextEdit(QTextEdit):
        
    def canInsertFromMimeData(self, source):
        if source.hasUrls():
-            return True;
+            return True
        else:
            return QTextEdit.canInsertFromMimeData(self,source)

@@ -207,8 +213,8 @@ class AddNewDialog(SizePersistedDialog):
        self.groupbox.setCheckable(True)
        self.groupbox.setChecked(False)
        self.groupbox.setFlat(True)
-        print("style:%s"%self.groupbox.styleSheet())
-        self.groupbox.setStyleSheet(gpstyle);
+        #print("style:%s"%self.groupbox.styleSheet())
+        self.groupbox.setStyleSheet(gpstyle)

        self.gbf = QFrame()
        self.gbl = QVBoxLayout()
@@ -537,7 +543,7 @@ class LoopProgressDialog(QProgressDialog):
    def updateStatus(self):
        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
        self.setValue(self.i+1)
-        print(self.labelText())
+        #print(self.labelText())

    def do_loop(self):

@@ -558,7 +564,7 @@ class LoopProgressDialog(QProgressDialog):
        except Exception as e:
            book['good']=False
            book['comment']=unicode(e)
-            print("Exception: %s:%s"%(book,unicode(e)))
+            logger.error("Exception: %s:%s"%(book,unicode(e)))
            traceback.print_exc()
            
        self.updateStatus()
@@ -7,13 +7,17 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'

-import time, os, copy, threading, re, platform
+import logging
+logger = logging.getLogger(__name__)
+
+import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
 from datetime import datetime
 from string import Template
 import urllib
 import email
+import traceback

 from PyQt4.Qt import (QApplication, QMenu, QToolButton, QTimer)

@@ -23,7 +27,7 @@ from PyQt4.QtCore import QBuffer
 from calibre.constants import numeric_version as calibre_version

 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
-from calibre.ebooks.metadata import MetaInformation, authors_to_string
+from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
@@ -142,7 +146,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            file_path = os.path.join(calibre_config_dir,
                                     *("plugins/fanfictiondownloader_macmenuhack.txt".split('/')))
            file_path = os.path.abspath(file_path)
-            print("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
+            logger.debug("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
            self.macmenuhack = os.access(file_path, os.F_OK)
            return self.macmenuhack

@@ -386,7 +390,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        show_copy_button=False)

    def get_urls_from_page(self,url):
-        print("get_urls_from_page URL:%s"%url)
+        logger.debug("get_urls_from_page URL:%s"%url)
        if 'archiveofourown.org' in url:
            configuration = get_ffdl_config(url)
        else:
@@ -540,7 +544,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            return
            
        tdir = PersistentTemporaryDirectory(prefix='ffdl_anthology_')
-        print("tdir:\n%s"%tdir)
+        logger.debug("tdir:\n%s"%tdir)

        bookepubio = StringIO(db.format(book_id,'EPUB',index_is_id=True))
        
@@ -616,7 +620,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            '''%(len(urlmapfile),"</li><li>".join(urlmapfile.keys()))
            if not question_dialog(self.gui, 'Stories Removed',
                               text, show_copy_button=False):
-                print("Canceling anthology update due to removed stories.")
+                logger.debug("Canceling anthology update due to removed stories.")
                return

        # Now that we've 
@@ -628,7 +632,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            return

        if not id_list:
-            id_list = self.gui.library_view.get_selected_ids();
+            id_list = self.gui.library_view.get_selected_ids()
        
        if len(id_list) == 0:
            self.gui.status_bar.show_message(_('No Selected Books to Update'), 3000)
@@ -695,7 +699,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            books = self.convert_urls_to_books(url_list)

        options['version'] = self.version
-        print(self.version)
+        logger.debug(self.version)
        
        #print("prep_downloads:%s"%books)

@@ -730,7 +734,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        '''

        url = book['url']
-        print("url:%s"%url)
+        logger.debug("url:%s"%url)
        mi = None

        if not merge: # skip reject list when merging.
@@ -786,7 +790,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            try:
                adapter.getStoryMetadataOnly()
            except exceptions.FailedToLogin, f:
-                print("Login Failed, Need Username/Password.")
+                logger.warn("Login Failed, Need Username/Password.")
                userpass = UserPassDialog(self.gui,url,f)
                userpass.exec_() # exec_ will make it act modal
                if userpass.status:
@@ -868,14 +872,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            
            if book['calibre_id'] != None:
                # updating an existing book.  Update mode applies.
-                print("update existing id:%s"%book['calibre_id'])
+                logger.debug("update existing id:%s"%book['calibre_id'])
                book_id = book['calibre_id']
                # No handling needed: OVERWRITEALWAYS,CALIBREONLY
                
            # only care about collisions when not ADDNEW
            elif collision != ADDNEW:
                # 'new' book from URL.  collision handling applies.
-                print("from URL(%s)"%url)
+                logger.debug("from URL(%s)"%url)
    
                # try to find by identifier url or uri first.
                searchstr = 'identifiers:"~ur(i|l):=%s"'%url.replace(":","|")
@@ -888,16 +892,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        ## meantime, if it matches the title *and* first
                        ## 100 authors, I'm prepared to assume it's a
                        ## match.
-                        print("reduce author list to 100 only when calibre < 0.8.61")
+                        logger.debug("reduce author list to 100 only when calibre < 0.8.61")
                        authlist = authlist[:100]
                    mi = MetaInformation(story.getMetadata("title", removeallentities=True),
                                         authlist)
                    identicalbooks = db.find_identical_books(mi)
                    if len(identicalbooks) > 0:
-                        print("existing found by title/author(s)")
+                        logger.debug("existing found by title/author(s)")
    
                else:
-                    print("existing found by identifier URL")
+                    logger.debug("existing found by identifier URL")
                    
                if collision == SKIP and identicalbooks:
                    raise NotGoingToDownload("Skipping duplicate story.","list_remove.png")
@@ -918,7 +922,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
    
                if book_id and mi: # book_id and mi only set if matched by title/author.
                    liburl = self.get_story_url(db,book_id)
-                    if book['url'] != liburl:
+                    if book['url'] != liburl and prefs['checkforurlchange']:
                        if collision in (OVERWRITE,OVERWRITEALWAYS):
                            updat="overwrit"
                        else:
@@ -996,14 +1000,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                  suffix='.epub',
                                                  dir=options['tdir'])
                    db.copy_format_to(book_id,fileform,tmp,index_is_id=True)
-                    print("existing epub tmp:"+tmp.name)
+                    logger.debug("existing epub tmp:"+tmp.name)
                    book['epub_for_update'] = tmp.name

            if book_id != None and prefs['injectseries']:
                mi = db.get_metadata(book_id,index_is_id=True)
                if not book['series'] and mi.series != None:
                    book['calibre_series'] = (mi.series,mi.series_index)
-                    print("calibre_series:%s [%s]"%book['calibre_series'])
+                    #print("calibre_series:%s [%s]"%book['calibre_series'])
                
        if book['good']: # there shouldn't be any !'good' books at this point.
            # if still 'good', make a temp file to write the output to.
@@ -1013,8 +1017,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            tmp = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
                                          suffix='.'+options['fileform'],
                                          dir=options['tdir'])
-            print("title:"+book['title'])
-            print("outfile:"+tmp.name)
+            logger.debug("title:"+book['title'])
+            logger.debug("outfile:"+tmp.name)
            book['outfile'] = tmp.name            
                
        return
@@ -1092,7 +1096,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if book['calibre_id'] and prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
            label = custom_columns[prefs['errorcol']]['label']
            if not book['good']:
-                print("record/update error message column %s %s"%(book['title'],book['url']))
+                logger.debug("record/update error message column %s %s"%(book['title'],book['url']))
                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True) # book['comment']
            else:
                db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
@@ -1100,7 +1104,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if not book['good']:
            return # only update errorcol on error.
        
-        print("add/update %s %s"%(book['title'],book['url']))
+        logger.debug("add/update %s %s"%(book['title'],book['url']))
        mi = self.make_mi_from_book(book)
                
        if options['collision'] != CALIBREONLY:
@@ -1108,12 +1112,22 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

        if options['collision'] == CALIBREONLY or \
                ( (options['updatemeta'] or book['added']) and book['good'] ):
-            self.update_metadata(db, book['calibre_id'], book, mi, options)
+            try:
+                self.update_metadata(db, book['calibre_id'], book, mi, options)
+            except:
+                det_msg = "".join(traceback.format_exception(*sys.exc_info()))+"\nStory Details:\n%s"%pretty_book(book)
+                logger.error("Error Updating Metadata:\n%s"%det_msg)
+                error_dialog(self.gui,
+                             "Error Updating Metadata",
+                             "<p>An error has occurred while FFDL was updating calibre's metadata for <a href='%s'>%s</a>.</p>"%(book['url'],book['title'])+
+                             "The ebook has been updated, but the metadata has not.",
+                             det_msg=det_msg,
+                             show=True)

    def update_books_finish(self, book_list, options={}, showlist=True):
        '''Notify calibre about updated rows, update external plugins
        (Reading Lists & Count Pages) as configured'''
-        
+
        add_list = filter(lambda x : x['good'] and x['added'], book_list)
        add_ids = [ x['calibre_id'] for x in add_list ]
        update_list = filter(lambda x : x['good'] and not x['added'], book_list)
@@ -1141,7 +1155,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if self.gui.cover_flow:
            self.gui.cover_flow.dataChanged()

-        if showlist: # don't use with anthology
+        if showlist and prefs['mark']: # don't use with anthology
            db = self.gui.current_db
            marked_ids = dict()
            marked_text = "ffdl_success"
@@ -1159,10 +1173,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                self.gui.library_view.sort_by_named_field('marked', True)
                
        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
-            
-        print("all done, remove temp dir.")
        remove_dir(options['tdir'])
-
+        
        if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
            cp_plugin = self.gui.iactions['Count Pages']
            cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
@@ -1256,7 +1268,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        (good_list,bad_list,options) = payload
        total_good = len(good_list)

-        print("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['listorder']) for x in good_list ]))
+        logger.debug("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['listorder']) for x in good_list ]))

        good_list = sorted(good_list,key=lambda x : x['listorder'])
        bad_list = sorted(bad_list,key=lambda x : x['listorder'])
@@ -1282,8 +1294,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            # if still 'good', make a temp file to write the output to.
            tmp = PersistentTemporaryFile(suffix='.'+options['fileform'],
                                          dir=options['tdir'])
-            print("title:"+mergebook['title'])
-            print("outfile:"+tmp.name)
+            logger.debug("title:"+mergebook['title'])
+            logger.debug("outfile:"+tmp.name)
            mergebook['outfile'] = tmp.name

        self.get_epubmerge_plugin().do_merge(tmp.name,
@@ -1304,7 +1316,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

        self.gui.status_bar.show_message(_('FFDL Adding/Updating books.'))

-        if good_list or (bad_list and prefs['errorcol'] != '' and prefs['errorcol'] in self.gui.library_view.model().custom_columns):
+        if good_list or prefs['mark'] or (bad_list and prefs['errorcol'] != '' and prefs['errorcol'] in self.gui.library_view.model().custom_columns):
            LoopProgressDialog(self.gui,
                               good_list+bad_list,
                               partial(self.update_books_loop, options=options, db=self.gui.current_db),
@@ -1317,22 +1329,24 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        '''Update custom error column if configured.'''
        (empty_list,book_list,options)=payload
        custom_columns = self.gui.library_view.model().custom_columns
-        if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+        if prefs['mark'] or (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
            self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%len(book_list)))
            label = custom_columns[prefs['errorcol']]['label']
            LoopProgressDialog(self.gui,
                               book_list,
                               partial(self.update_error_column_loop, db=self.gui.current_db, label=label),
-                               partial(self.update_books_finish, options=options, showlist=False),
+                               partial(self.update_books_finish, options=options),
                               init_label="Updating calibre for BAD FanFiction stories...",
                               win_title="Update calibre for BAD FanFiction stories",
                               status_prefix="Updated")

    def update_error_column_loop(self,book,db=None,label='errorcol'):
        if book['calibre_id']:
-            print("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
-            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+            custom_columns = self.gui.library_view.model().custom_columns
+            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
+                logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)

    def add_book_or_update_format(self,book,options,prefs,mi=None):
        db = self.gui.current_db
@@ -1361,7 +1375,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
            for fmt in fmts:
                if fmt != formmapping[options['fileform']]:
-                    print("remove f:"+fmt)
+                    logger.debug("remove f:"+fmt)
                    db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False

        return book_id
@@ -1370,14 +1384,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        oldmi = db.get_metadata(book_id,index_is_id=True)
        if prefs['keeptags']:
            old_tags = db.get_tags(book_id)
+            #print("old_tags:%s"%old_tags)
+            #print("mi.tags:%s"%mi.tags)
            # remove old Completed/In-Progress only if there's a new one.
            if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
                old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags)
                # remove old Last Update tags if there are new ones.
            if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
                old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
+                
            # mi.tags needs to be list, but set kills dups.
-            mi.tags = list(set(list(old_tags)+mi.tags))
+            # this way also removes case-mismatched dups, keeping old_tags version.
+            foldedcase_tags = dict()
+            for t in list(mi.tags) + list(old_tags):
+                foldedcase_tags[t.lower()] = t
+
+            mi.tags = foldedcase_tags.values()   
+            #print("mi.tags:%s"%mi.tags)

        if book['all_metadata']['langcode']:
            mi.languages=[book['all_metadata']['langcode']]
@@ -1393,7 +1416,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                try:
                    db.set_cover(book_id, epubmi.cover_data[1])
                except:
-                    print("Failed to set_cover, skipping")
+                    logger.info("Failed to set_cover, skipping")

        # implement 'newonly' flags here by setting to the current
        # value again.
@@ -1406,10 +1429,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        try:
                            mi.__setattr__(col,oldmi.__getattribute__(col))
                        except AttributeError:
-                            print("AttributeError? %s"%col)
+                            logger.warn("AttributeError? %s"%col)
                            pass

        db.set_metadata(book_id,mi)
+        # mi.authors gets run through the string_to_authors and split on '&' ',' 'and' and 'with'
+        db.set_authors(book_id,book['author']) # author is a list.

        # do configured column updates here.
        #print("all_metadata: %s"%book['all_metadata'])
@@ -1419,25 +1444,26 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        for col, meta in prefs['custom_cols'].iteritems():
            #print("setting %s to %s"%(col,meta))
            if col not in custom_columns:
-                print("%s not an existing column, skipping."%col)
+                logger.debug("%s not an existing column, skipping."%col)
                continue
            coldef = custom_columns[col]
            if col in prefs['custom_cols_newonly'] and prefs['custom_cols_newonly'][col] and not book['added']:
-                print("Skipping custom column(%s) update, set to New Books Only"%coldef['name'])
+                logger.debug("Skipping custom column(%s) update, set to New Books Only"%coldef['name'])
                continue
            if not meta.startswith('status-') and meta not in book['all_metadata'] or \
                    meta.startswith('status-') and 'status' not in book['all_metadata']:
-                print("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))
+                logger.debug("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))
                continue
            if meta not in permitted_values[coldef['datatype']]:
-                print("%s not a valid column type for %s, skipping."%(col,meta))
+                logger.debug("%s not a valid column type for %s, skipping."%(col,meta))
                continue
            label = coldef['label']
            if coldef['datatype'] in ('enumeration','text','comments','datetime','series'):
-                db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
+                db.set_custom(book_id, book['all_metadata'][meta], label, commit=False)
            elif coldef['datatype'] in ('int','float'):
                num = unicode(book['all_metadata'][meta]).replace(",","")
-                db.set_custom(book_id, num, label=label, commit=False)
+                if num != '':
+                    db.set_custom(book_id, num, label=label, commit=False)
            elif coldef['datatype'] == 'bool' and meta.startswith('status-'):
                if meta == 'status-C':
                    val = book['all_metadata']['status'] == 'Completed'
@@ -1457,14 +1483,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                    if "," in custcol:
                        (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )

-                    print("meta:(%s) => custcol:(%s), flag(%s) "%(meta,custcol,flag))
-                    
                    if meta not in book['all_metadata']:
-                        print("No value for %s, skipping custom column(%s) update."%(meta,custcol))
+                        logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
                        continue
                    
                    if custcol not in custom_columns:
-                        print("No custom column(%s), skipping."%(custcol))
                        continue
                    else:
                        coldef = custom_columns[custcol]
@@ -1479,7 +1502,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                val = unicode(book['all_metadata'][meta]).replace(",","")
                        else:
                            val = book['all_metadata'][meta]
-                        db.set_custom(book_id, val, label=label, commit=False)
+                        if val != '':
+                            db.set_custom(book_id, val, label=label, commit=False)

                    if flag == 'a':
                        vallist = []
@@ -1501,12 +1525,26 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        # Moved down so author's already in the DB.
        if 'authorUrl' in book['all_metadata']:
            authurls = book['all_metadata']['authorUrl'].split(", ")
-            for i, auth in enumerate(book['author']):
-                #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                autid=db.get_author_id(auth)
-                db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
+            authorlist = [ a.replace('&',';') for a in book['author'] ]
+            if hasattr(db, 'new_api'): # new_api starts in calibre 1.0.0
+                authorids = db.new_api.get_item_ids('authors',authorlist)
+                authordata = db.new_api.author_data(authorids.values())
+                # print("\n\nauthorids:%s"%authorids)
+                # print("authordata:%s"%authordata)

+                author_id_to_link_map = dict()
+                for i, author in enumerate(authorlist):
+                    author_id_to_link_map[authorids[author]] = authurls[i]
+
+                # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
+                db.new_api.set_link_for_authors(author_id_to_link_map)
+            else:
+                # keep for pre-calibre 1.0.0
+                for i, auth in enumerate(authorlist):
+                    #print("===Update author url for %s to %s"%(auth,authurls[i]))
+                    autid=db.get_author_id(auth)
+                    db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
        db.commit()

        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
@@ -1534,9 +1572,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                            break

                if setting_name:
-                    print("Generate Cover Setting from generate_cover_settings(%s)"%line)
+                    logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%line)
                    if setting_name not in gc_plugin.get_saved_setting_names():
-                        print("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
+                        logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
                        setting_name = None
                
            if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
@@ -1546,7 +1584,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                setting_name =  prefs['gc_site_settings']['Default']
                
            if setting_name:
-                print("Running Generate Cover with settings %s."%setting_name)
+                logger.debug("Running Generate Cover with settings %s."%setting_name)
                realmi = db.get_metadata(book_id, index_is_id=True)
                gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)

@@ -1606,6 +1644,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

    def make_mi_from_book(self,book):
        mi = MetaInformation(book['title'],book['author']) # author is a list.
+        if prefs['suppressauthorsort']:
+            # otherwise author names will have calibre's sort algs
+            # applied automatically.
+            mi.author_sort = ' & '.join(book['author'])
+        if prefs['suppresstitlesort']:
+            # otherwise titles will have calibre's sort algs applied
+            # automatically.
+            mi.title_sort = book['title']
        mi.set_identifiers({'url':book['url']})
        mi.publisher = book['publisher']
        mi.tags = book['tags']
@@ -1852,11 +1898,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                book['title'] = series
                for sr in serieslist:
                    if series != sr:
-                        book['title'] = deftitle;
+                        book['title'] = deftitle
                        break
            
            configuration = get_ffdl_config(book['url'],fileform)
-            print("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
+            logger.debug("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
            if configuration.getConfig('anthology_title_pattern'):
                tmplt = Template(configuration.getConfig('anthology_title_pattern'))
                book['title'] = tmplt.safe_substitute({'title':book['title']}).encode('utf8')
@@ -1891,3 +1937,17 @@ def split_text_to_urls(urls):
 def escapehtml(txt):
    return txt.replace("&","&amp;").replace(">","&gt;").replace("<","&lt;")
        
+def pretty_book(d, indent=0, spacer='     '):
+    kindent = spacer * indent
+
+    # if isinstance(d, list):
+    #     return '\n'.join([(pretty_book(v, indent, spacer)) for v in d])
+
+    if isinstance(d, dict):
+        for k in ('password','username'):
+            if k in d and d[k]:
+                d[k]='<was set, removed for security>'
+        return '\n'.join(['%s%s:\n%s' % (kindent, k, pretty_book(v, indent + 1, spacer)) 
+                          for k, v in d.items()])
+    return "%s%s"%(kindent, d)
+
@@ -8,6 +8,9 @@ __copyright__ = '2012, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'

+import logging
+logger = logging.getLogger(__name__)
+
 import time, os, traceback

 from StringIO import StringIO
@@ -38,13 +41,13 @@ def do_download_worker(book_list, options,
    '''
    server = Server(pool_size=cpus)

-    print(options['version'])
+    logger.info(options['version'])
    total = 0
    alreadybad = []
    # Queue all the jobs
-    print("Adding jobs for URLs:")
+    logger.info("Adding jobs for URLs:")
    for book in book_list:
-        print("%s"%book['url'])
+        logger.info("%s"%book['url'])
        if book['good']:
            total += 1
            args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
@@ -87,19 +90,19 @@ def do_download_worker(book_list, options,
        count = count + 1
        notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
        # Add this job's output to the current log
-        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
-        print(job.details)
+        logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        logger.info(job.details)

        if count >= total:
            # All done!  Output some lists for convenience of some users.
-            print("Successfully downloaded:")
+            logger.info("Successfully downloaded:")
            for book in book_list:
                if book['good']:
-                    print("%s %s"%(book['title'],book['url']))
-            print("\nUnsuccessful:")
+                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\nUnsuccessful:")
            for book in book_list:
                if not book['good']:
-                    print("%s %s"%(book['title'],book['url']))
+                    logger.info("%s %s"%(book['title'],book['url']))
            break

    server.close()
@@ -147,7 +150,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):

        ## No need to download at all.  Shouldn't ever get down here.
        if options['collision'] in (CALIBREONLY):
-            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
            book['comment'] = 'Metadata collected.'
            
        ## checks were done earlier, it's new or not dup or newer--just write it.
@@ -170,7 +173,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                if adapter.logfile:
                    adapter.logfile = adapter.logfile.replace("span id","span notid")
            
-            print("write to %s"%outfile)
+            logger.info("write to %s"%outfile)
            writer.writeStory(outfilename=outfile, forceOverwrite=True)
            book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
            
@@ -198,9 +201,13 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
            # dup handling from ffdl_plugin needed for anthology updates.
            if chaptercount > urlchaptercount:
                raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
-                            
-            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
-            print("write to %s"%outfile)
+
+            if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
+                    and adapter.getConfig("do_update_hook"):
+                chaptercount = adapter.hookForUpdates(chaptercount)
+
+            logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+            logger.info("write to %s"%outfile)

            writer.writeStory(outfilename=outfile, forceOverwrite=True)
            
@@ -222,7 +229,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
            
            log = Log(level=Log.DEBUG)
            # report = []
-            polish({outfile:outfile}, opts, log, print) # report.append
+            polish({outfile:outfile}, opts, log, logger.info) # report.append
        
    except NotGoingToDownload as d:
        book['good']=False
@@ -234,7 +241,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
        book['comment']=unicode(e)
        book['icon']='dialog_error.png'
        book['status'] = 'Error'
-        print("Exception: %s:%s"%(book,unicode(e)))
+        logger.info("Exception: %s:%s"%(book,unicode(e)))
        traceback.print_exc()
        
    #time.sleep(10)
@@ -30,6 +30,9 @@ default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False
 default_prefs['updateepubcover'] = False
 default_prefs['keeptags'] = False
+default_prefs['suppressauthorsort'] = False
+default_prefs['suppresstitlesort'] = False
+default_prefs['mark'] = False
 default_prefs['showmarked'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
@@ -40,6 +43,7 @@ default_prefs['adddialogstaysontop'] = False
 default_prefs['includeimages'] = False
 default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
+default_prefs['checkforurlchange'] = True
 default_prefs['injectseries'] = False
 default_prefs['smarten_punctuation'] = False

@@ -16,7 +16,22 @@
 [defaults]

 ## [defaults] section applies to all formats and sites but may be
-## overridden at several levels
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category

 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
@@ -152,6 +167,17 @@ extratags: FanFiction
 #extraships:
 #extrawarnings:

+## Add this to genre if there's more than one category.
+#add_genre_when_multi_category: Crossover
+
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -259,6 +285,8 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true

+## User-agent
+user_agent:FFDL/1.7

 ## Each output format has a section that overrides [defaults]
 [html]
@@ -447,8 +475,8 @@ nook_img_fix:true
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true

-## Each site has a section that overrides [defaults] *and* the format
-## sections test1.com specifically is not a real story site.  Instead,
+## Each site has a section that overrides [defaults].
+## test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
@@ -505,7 +533,8 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true

 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
+## If there's ever more than 4 series, add series04,series04Url etc.
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 fandoms_label:Fandoms
 freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
@@ -521,7 +550,7 @@ bookmarks_label:Bookmarks
 include_in_freefromtags:freeformtags

 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url

 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
@@ -739,6 +768,23 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword

+[fictionpad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+extra_valid_entries:followers,comments,views,likes,dislikes
+#extra_titlepage_entries:followers,comments,views,likes,dislikes
+
+followers_label:Followers
+comments_label:Comments
+views_label:Views
+likes_label:Likes
+dislikes_label:Dislikes
+
 [finestories.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -789,6 +835,10 @@ extracategories:Highlander
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death

+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/public/style_emoticons/.*
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1096,6 +1146,7 @@ context_label:Context
 type_label:Type of Couple

 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1126,7 +1177,12 @@ extracategories:Harry Potter
 ## fictionally.org storyIds are not unique.  Combine with authorId.
 output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}

+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:

@@ -1148,12 +1204,13 @@ extracategories:My Little Pony: Friendship is Magic

 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups

 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1165,6 +1222,13 @@ short_description_label:Short Summary
 ## when a password is required rather than prompting every time.
 #fail_on_password: false

+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 [www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -270,7 +270,6 @@ def main(argv,
           elif chaptercount == 0:
               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
           else:
-               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
               if not options.metaonly:

                   # update now handled by pre-populating the old
@@ -284,6 +283,12 @@ def main(argv,
                    adapter.calibrebookmark,
                    adapter.logfile) = get_update_data(output_filename)

+                   print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+
+                   if not (options.update and chaptercount == urlchaptercount) \
+                           and adapter.getConfig("do_update_hook"):
+                       chaptercount = adapter.hookForUpdates(chaptercount)
+
                   writeStory(configuration,adapter,"epub")
                   
       else:
@@ -4,13 +4,16 @@ try:
    # just a way to switch between web service and CLI/PI
    import google.appengine.api 
 except:
-    import sys
-    if sys.version_info >= (2, 7):
-        import logging
-        logger = logging.getLogger(__name__)
-        loghandler=logging.StreamHandler()
-        loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
-        logger.addHandler(loghandler)
-        loghandler.setLevel(logging.DEBUG)
-        logger.setLevel(logging.DEBUG)
+    try: # just a way to switch between CLI and PI
+        import calibre.constants
+    except:
+        import sys
+        if sys.version_info >= (2, 7):
+            import logging
+            logger = logging.getLogger(__name__)
+            loghandler=logging.StreamHandler()
+            loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+            logger.addHandler(loghandler)
+            loghandler.setLevel(logging.DEBUG)
+            logger.setLevel(logging.DEBUG)

@@ -119,6 +119,7 @@ import adapter_nickandgregnet
 import adapter_potterheadsanonymouscom
 import adapter_simplyundeniablecom
 import adapter_scarheadnet
+import adapter_fictionpadcom

 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -90,7 +90,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php"))
@@ -163,7 +163,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r"^/works/\d+$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
 		
        # Find authorid and URL from... author url.
        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
@@ -210,10 +210,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        if a != None:
            warnings = a.findAll('a',{'class':"tag"})
            for warning in warnings:
-                if warning.string == "Author Chose Not To Use Archive Warnings":
-                    warning.string = "No Archive Warnings Apply"
-                if warning.string != "No Archive Warnings Apply":
-                    self.story.addToList('warnings',warning.string)
+                self.story.addToList('warnings',warning.string)
 		
        a = metasoup.find('dd',{'class':"freeform tags"})
        if a != None:
@@ -288,19 +285,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))

 		
-        try:
-            # Find Series name from series URL.
-            a = metasoup.find('dd',{'class':"series"})
-            b = a.find('a', href=re.compile(r"/series/\d+"))
-            series_name = b.string
-            series_url = 'http://'+self.host+b['href']
-            series_index = int(a.text.split(' ')[1])
-            self.setSeries(series_name, series_index)
-            self.story.setMetadata('seriesUrl',series_url)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        # Find Series name from series URL.
+        ddseries = metasoup.find('dd',{'class':"series"})
+
+        if ddseries:
+            for i, a in enumerate(ddseries.findAll('a', href=re.compile(r"/series/\d+"))):
+                series_name = stripHTML(a)
+                series_url = 'http://'+self.host+a['href']
+                series_index = int(stripHTML(a.previousSibling).replace(', ','').split(' ')[1]) # "Part # of" or ", Part #"
+                self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index))
+                self.story.setMetadata('series%02dUrl'%i,series_url)
+                if i == 0:
+                    self.setSeries(series_name, series_index)
+                    self.story.setMetadata('seriesUrl',series_url)

    # grab the text for an individual chapter.
    def getChapterText(self, url):
@@ -101,7 +101,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
-        title=a.text.split('"')[1]
+        title=stripHTML(a).split('"')[1]
        self.story.setMetadata('title',title)
        
        # Find authorid and URL from... author url.
@@ -157,7 +157,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
            if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                break
 		

@@ -128,7 +128,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -207,7 +207,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -180,7 +180,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
        pagetitle = soup.find('div',{'id':'pagetitle'})
        ## Title
        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -116,7 +116,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -143,11 +143,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):
        # Now go hunting for all the meta data and the chapter list.

        ## Title
-        a = soup.findAll('span', {'class' : 'storytitle'})
-        self.story.setMetadata('title',a[0].string)
+        a = soup.find('span', {'class' : 'storytitle'})
+        self.story.setMetadata('title',stripHTML(a))

        # Find authorid and URL from... author url.
-        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
+        a = a.parent.find('a', href=re.compile(r"authors.php\?name\=\w+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
        self.story.setMetadata('author',a.string)
@@ -94,11 +94,10 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
        else:
            params['penname'] = self.getConfig("username")
            params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
        params['action'] = 'login'
        params['submit'] = 'Submit'
    
-        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                              params['penname']))
    
@@ -131,7 +131,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -164,7 +164,7 @@ class DokugaComAdapter(BaseSiteAdapter):
        aut.extract()
        
        a = a.string[:(len(a.string)-4)]
-        self.story.setMetadata('title',a)
+        self.story.setMetadata('title',stripHTML(a))

        # Find the chapters:
        chapters = soup.find('select').findAll('option')
@@ -205,7 +205,6 @@ class DokugaComAdapter(BaseSiteAdapter):
            
            a=div.text.split('Category: ')
            if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
-            self.story.addToList('category', 'Fanfiction')
        
            a=div.text.split('Created: ')
            if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))
@@ -141,7 +141,7 @@ class DotMoonNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = body.find('b')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url. http://www.dotmoon.net/board/index.php?action=profile;u=1'
        a = body.find('a', href=re.compile(r"index.php\?action=profile;u=\d+"))
@@ -170,7 +170,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -154,7 +154,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -104,7 +104,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
        pagetitle = soup.find('div',{'id':'pagetitle'})
        ## Title
        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -132,7 +132,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -199,7 +199,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        for g in genrelist:
            #logger.debug("g:(%s)"%g)
            if g.strip() not in ffnetgenres:
-                logger.info("g not in ffnetgenres")
+                #logger.info("g not in ffnetgenres")
                goodgenres=False
        if goodgenres:
            self.story.extendList('genre',genrelist)
@@ -266,11 +266,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        return

    def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
                        ## more if hit too fast.  This is in
                        ## additional to what ever the
                        ## slow_down_sleep_time setting is.
+        logger.debug('Getting chapter text from: %s' % url)
        data = self._fetchUrl(url)

        if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
@@ -281,10 +281,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        # don't care about anything before "<div class='storytextp"
        # (there's a space after storytextp, so no close quote(')) and
        # this kills any body tags.
-        if "<div class='storytextp" not in data:
+        divstr = "<div role='main'"
+        if divstr not in data:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
        else:
-            data = data[data.index("<div class='storytextp"):]
+            data = data[data.index(divstr):]
        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
        
        soup = bs.BeautifulSoup(data)
@@ -111,7 +111,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
            url=self.url
            logger.debug("Normalizing to URL: "+url)
            ## title's right there...
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
            data = self._fetchUrl(url)
            soup = bs.BeautifulSoup(data)
            chapterlinklist = soup.findAll('a',{'class':'chapterlink'})
@@ -121,7 +121,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
            ## same way.
            chapsoup = bs.BeautifulSoup(chapterdata)
            storya = chapsoup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
            del chapsoup

        del chapterdata
@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import time
+import json
+
+from .. import BeautifulSoup as bs
+#from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+class FictionPadSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev','fpad')
+        self.dateformat = "%Y-%m-%dT%H:%M:%SZ"
+        self.is_adult=False
+        self.username = None
+        self.password = None
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL.
+            self._setURL("https://"+self.getSiteDomain()
+                         +"/author/"+m.group('author')
+                         +"/stories/"+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+
+    @staticmethod
+    def getSiteDomain():
+        return 'fictionpad.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "https://fictionpad.com/author/Author/stories/1234/Some-Title"
+    
+    def getSiteURLPattern(self):
+        # http://fictionpad.com/author/Serdd/stories/4275
+        return r"http(s)?://(www\.)?fictionpad\.com/author/(?P<author>[^/]+)/stories/(?P<id>\d+)"
+
+# <form method="post" action="/signin">
+#     <input name="authenticity_token" type="hidden" value="u+cfdXh46dRnwVnSlmE2B2BFmHgu760paqgBG6KQeos=" />
+#     <input type="hidden" name="remember" value="1">
+#     <strong class="help-start text-center">or with FictionPad</strong>
+#     <label class="control-label hidden-placeholder">Pseudonym or Email Address</label>
+#     <input name="login" class="input-block-level" type="text" placeholder="Pseudonym or Email Address" maxlength="50" required autofocus>
+#     <label class="control-label hidden-placeholder">Password</label>
+#     <input name="password" class="input-block-level" type="password" placeholder="Password" minlength="6" required>
+#     <button type="submit" class="btn btn-primary btn-block">Sign In</button>
+#     <p class="help-end">
+#         <a href="/passwordreset">Forgot your password?</a>
+#     </p>
+# </form>
+    def performLogin(self):
+        params = {}
+
+        if self.password:
+            params['login'] = self.username
+            params['password'] = self.password
+        else:
+            params['login'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['remember'] = '1'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/signin'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['login']))
+
+        ## need to pull empty login page first to get authenticity_token
+        soup = bs.BeautifulSoup(self._fetchUrl(loginUrl))
+        params['authenticity_token']=soup.find('input', {'name':'authenticity_token'})['value']
+        
+        data = self._postUrl(loginUrl, params)
+
+        if "Invalid email/pseudonym and password combination." in data:
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['login']))
+            raise exceptions.FailedToLogin(loginUrl,params['login'])
+            
+
+    def extractChapterUrlsAndMetadata(self):
+        # fetch the chapter.  From that we will get almost all the
+        # metadata and chapter list
+
+        url=self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+            if "This is a mature story.  Please sign in to read it." in data:
+                self.performLogin()
+                data = self._fetchUrl(url)
+            
+            find = "wordyarn.config.page = "
+            data = data[data.index(find)+len(find):]
+            data = data[:data.index("</script>")]
+            data = data[:data.rindex(";")]
+            data = data.replace('tables:','"tables":')
+            tables = json.loads(data)['tables']
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+
+        # looks like only one author per story allowed.
+        author = tables['users'][0]
+        story = tables['stories'][0]
+        story_ver = tables['story_versions'][0]
+        
+        self.story.setMetadata('authorId',author['id'])
+        self.story.setMetadata('author',author['display_name'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/author/'+author['display_name']+'/stories')
+
+        self.story.setMetadata('title',story_ver['title'])
+        self.setDescription(url,story_ver['description'])
+        
+        if not ('assets/story_versions/covers' in story_ver['profile_image_url@2x']):
+            self.setCoverImage(url,story_ver['profile_image_url@2x'])
+
+        self.story.setMetadata('datePublished',makeDate(story['published_at'], self.dateformat))
+        self.story.setMetadata('dateUpdated',makeDate(story['published_at'], self.dateformat))
+
+        self.story.setMetadata('followers',story['followers_count'])
+        self.story.setMetadata('comments',story['comments_count'])
+        self.story.setMetadata('views',story['views_count'])
+        self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
+        self.story.setMetadata('dislikes',int(story['dislikes']))
+
+        if story_ver['is_complete']:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        self.story.setMetadata('rating', story_ver['maturity_level'])
+        self.story.setMetadata('numWords', unicode(story_ver['word_count']))
+
+        for i in tables['fandoms']:
+            self.story.addToList('category',i['name'])
+        
+        for i in tables['genres']:
+            self.story.addToList('genre',i['name'])
+        
+        for i in tables['characters']:
+            self.story.addToList('characters',i['name'])
+
+        for c in tables['chapters']:
+            chtitle = "Chapter %d"%c['number']
+            if c['title']:
+                chtitle += " - %s"%c['title']
+            self.chapterUrls.append((chtitle,c['body_url']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        return self.utf8FromSoup(url,soup)
+    
+def getClass():
+    return FictionPadSiteAdapter
+
@@ -116,7 +116,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):

        # title - first h4 tag will be title.
        titleh4 = soup.find('h4')
-        self.story.setMetadata('title', titleh4.a.string)
+        self.story.setMetadata('title', stripHTML(titleh4.a))

        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"^/author/\d+"))
@@ -21,7 +21,6 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
-from datetime import datetime
 import json

 from .. import BeautifulSoup as bs
@@ -42,6 +41,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
        self.is_adult = False
        
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
    @staticmethod
    def getSiteDomain():
        return 'www.fimfiction.net'
@@ -93,8 +96,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
            raise exceptions.StoryDoesNotExist(self.url)

-        if "/images/missing_story.png" in data:
-            raise exceptions.StoryDoesNotExist(self.url)
+        # Can cause problems if a missing story is referenced in a comment.
+        # Shouldn't be needed anyway.
+        # if "/images/missing_story.png" in data:
+        #     raise exceptions.StoryDoesNotExist(self.url)
        
        if "This story has been marked as having adult content." in data:
            raise exceptions.AdultCheckRequired(self.url)
@@ -117,7 +122,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
            
        ## Title
        a = soup.find('a', href=re.compile(r'^/story/'+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # self.story.setMetadata("title", storyMetadata["title"])
        # if not storyMetadata["title"]:
@@ -185,14 +190,42 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        hrstr="<hr />"
        descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
        self.setDescription(self.url,descdivstr)
-        
+
+        # Can't trust dates from API anymore I'm told.
        # Dates are in Unix time
        # Take the publish date from the first chapter posted
-        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        rawDateUpdated = storyMetadata["date_modified"]
-        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        # rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        # rawDateUpdated = storyMetadata["date_modified"]
+        # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        
+        oldestChapter = None
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on
+        # FiMFiction it's possible for authors to insert new chapters
+        # out-of-order or change the dates of earlier ones by editing
+        # them--That WILL break epub update.
+        for index, chapterDate in enumerate(soup.findAll('span', {'class':'date'})):
+            date=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",chapterDate.contents[1].strip())
+            chapterDate = makeDate(date,self.dateformat)
+            if oldestChapter == None or chapterDate < oldestChapter:
+                oldestChapter = chapterDate
+            if newestChapter == None or chapterDate > newestChapter:
+                newestChapter = chapterDate
+                self.newestChapterNum = index

+        self.story.setMetadata("dateUpdated", newestChapter)
+        
+        pubdatetag = soup.find('span', {'class':'date_approved'})
+        if pubdatetag is None:
+            self.story.setMetadata("datePublished", oldestChapter)            
+        else:
+            pubdateraw = pubdatetag('span')[1].text
+            datestripped=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",pubdateraw.strip())
+            pubDate = makeDate(datestripped,self.dateformat)
+            self.story.setMetadata("datePublished", pubDate)
+            
        chars = soup.find("div", {"class":"inner_data"})
        # fimfic stopped putting the char name on or around the char
        # icon now for some reason.  Pull it from the image name with
@@ -215,10 +248,22 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                if not isinstance(value,basestring):
                    value = unicode(value)
                self.story.setMetadata(metakey, value)
-        
+
+        rawGroupList = soup.find('ul', {'id':'story_group_list'})
+        if rawGroupList is not None:
+            for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
+                self.story.addToList("groups",stripHTML(groupName))
            
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
+        
    def getChapterText(self, url):
        logger.debug('Getting chapter text from: %s' % url)
+        
        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
        if soup == None:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
@@ -142,7 +142,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"/a/\w+"))
@@ -182,7 +182,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -92,7 +92,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        ## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995'
        if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")):
            raise exceptions.AdultCheckRequired(self.url)
@@ -131,55 +131,55 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):

        ## Finding the metadata is a bit of a pain.  Most of the meta
        ## data is in a center.table without a bgcolor.
-        for center in soup.findAll('center'):
-            table = center.find('table',{'bgcolor':None})
-            if table:
-                metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
-                # Rating: 12+ Story Reviews: 3
-                # Chapters: 3
-                # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
-                # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
-                # Status: Completed
-                # First Published: 2010.09.02
-                # Last Published Chapter: 2010.09.28
-                # Last Updated: 2010.09.28
-                # Favorite Story Of: 1 users
-                # Warnings: Scenes of a Mild Sexual Nature
+        #for center in soup.findAll('center'):
+        table = soup.find('table',{'class':'storymaininfo'})
+        if table:
+            metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
+            # Rating: 12+ Story Reviews: 3
+            # Chapters: 3
+            # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
+            # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
+            # Status: Completed
+            # First Published: 2010.09.02
+            # Last Published Chapter: 2010.09.28
+            # Last Updated: 2010.09.28
+            # Favorite Story Of: 1 users
+            # Warnings: Scenes of a Mild Sexual Nature

-                m = re.match(r".*?Status: Completed.*?",metastr)
-                if m:
-                    self.story.setMetadata('status','Completed')
-                else:
-                    self.story.setMetadata('status','In-Progress')
+            m = re.match(r".*?Status: Completed.*?",metastr)
+            if m:
+                self.story.setMetadata('status','Completed')
+            else:
+                self.story.setMetadata('status','In-Progress')

-                m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
-                if m:
-                    self.story.setMetadata('rating', m.group(1))
+            m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
+            if m:
+                self.story.setMetadata('rating', m.group(1))

-                m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('genre',g)
-        
-                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('characters',g)
-        
-                m = re.match(r".*?Warnings: (.+).*?",metastr)
-                if m:
-                    for w in m.group(1).split(','):
-                        if w != 'Now Warnings':
-                            self.story.addToList('warnings',w)
-        
-                m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
+            m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('genre',g)
+    
+            m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('characters',g)
+    
+            m = re.match(r".*?Warnings: (.+).*?",metastr)
+            if m:
+                for w in m.group(1).split(','):
+                    if w != 'Now Warnings':
+                        self.story.addToList('warnings',w)
+    
+            m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))

-                # Updated can have more than one space after it. <shrug>
-                m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
+            # Updated can have more than one space after it. <shrug>
+            m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))

    def getChapterText(self, url):

@@ -101,7 +101,7 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h2', {'id':'page_heading'})
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))

        # Find the chapters: chapter_view.cfm?stid=6663&amp;spordinal=1" 
        for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):
@@ -106,7 +106,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
        self.story.setMetadata('author',aut.string)
        aut.extract()
        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])

        # Find the chapters:
        chapters=soup.find('select')
@@ -37,11 +37,6 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)

-        # hpfanficarchive.com blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
-        
        self.decode = ["Windows-1252",
                       "utf8"] # 1252 is a superset of iso-8859-1.
                               # Most sites that claim to be
@@ -105,7 +100,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -183,7 +183,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -170,7 +170,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -46,8 +46,8 @@ class InDeathNetAdapter(BaseSiteAdapter):
        self.password = ""
        self.is_adult=False
        
-		
-		  # get storyId from url--url validation guarantees query correct
+        
+        # get storyId from url--url validation guarantees query correct
        m = re.match(self.getSiteURLPattern(),url)
        if m:
            self.story.setMetadata('storyId',m.group('id'))
@@ -87,6 +87,34 @@ class InDeathNetAdapter(BaseSiteAdapter):
        postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat)
        return postdate

+    def getAuthorData(self):
+        
+        mainUrl = self.url.replace("/archive","")
+        
+        try:
+            maindata = self._fetchUrl(mainUrl)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        mainsoup = bs.BeautifulSoup(maindata)
+
+        # find first entry
+        e = mainsoup.find('div',{'class':"entry"})
+        
+        # get post author as author
+        d = e.find('div',{'class':"desc"})
+        a = d.find('strong')
+        self.story.setMetadata('author',a.contents[0].string.strip())
+        
+        # Don't seem to be able to get author pages anymore
+        self.story.setMetadata('authorUrl','http://www.indeath.net/')
+        self.story.setMetadata('authorId','0')
+        
    ## Getting the chapter list and the meta data, plus 'is adult' checking.
    def extractChapterUrlsAndMetadata(self):

@@ -109,21 +137,14 @@ class InDeathNetAdapter(BaseSiteAdapter):
        ## Title
        h = soup.find('a', id="blog_title")
        t = h.find('span')
-        self.story.setMetadata('title',t.contents[0].string.strip())
-		
+        self.story.setMetadata('title',stripHTML(t.contents[0]).strip())
+        
        s = t.find('div')        
        if s != None:
            self.setDescription(url,s)
-		
-        # Find authorid and URL from first link in Recent Entries (don't yet reference 'recent entries' - let's see if that is required)
-        a = soup.find('a', href=re.compile(r"http://www.indeath.net/user/\d+\-[a-z0-9]+/$"))		#http://www.indeath.net/user/9083-cyrex/
-        m = re.search('http://www.indeath.net/user/(?P<id>\d+)\-(?P<name>[a-z0-9]*)/$',a['href'])
-        self.story.setMetadata('authorId',m.group('id'))
-        self.story.setMetadata('authorUrl',a['href'])
-        self.story.setMetadata('author',m.group('name'))
-
-

+        # Get Author from main blog page since it's not reliably on the archive page
+        self.getAuthorData()

        # Find the chapters:
        chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
@@ -152,21 +173,20 @@ class InDeathNetAdapter(BaseSiteAdapter):
            if len(chapters)==1:
                self.chapterUrls.append((self.story.getMetadata('title'),chapter['href']))
            else:
-            	ct = stripHTML(chapter)
-            	tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
-            	if tnew:
-            		chaptertitle = tnew.group('newtitle')
-            	else:
-            		chaptertitle = ct
+                ct = stripHTML(chapter)
+                tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
+                if tnew:
+                    chaptertitle = tnew.group('newtitle')
+                else:
+                    chaptertitle = ct
                self.chapterUrls.append((chaptertitle,chapter['href']))
-	
-		
+
        
        
    # grab the text for an individual chapter.
    def getChapterText(self, url):
        logger.debug('Getting chapter text from: %s' % url)
-		
+        
        #chapter=bs.BeautifulSoup('<div class="story"></div>')
        data = self._fetchUrl(url)
        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center'))
@@ -129,7 +129,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -116,7 +116,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -107,10 +107,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
        for font in title.findAll('font'):
            font.extract() # removes 'font' tags from inside the td.        
        if title.has_key('colspan'):
-            titlet = title.text
+            titlet = stripHTML(title)
        else:
            ## No colspan, it's part chapter title--even if it's a one-shot.
-            titlet = ':'.join(title.text.split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
+            titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
        self.story.setMetadata('title',titlet)
        ## The story title is difficult to reliably parse from the
        ## story pages.  Getting it from the author page is, but costs
@@ -170,7 +170,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -193,7 +193,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -104,7 +104,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
@@ -103,7 +103,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
@@ -37,11 +37,6 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)

-        # ncisfiction.net blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
-        
        self.decode = ["iso-8859-1",
                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
                               # Most sites that claim to be
@@ -97,7 +97,7 @@ class NetRaptorOrgAdapter(BaseSiteAdapter):
        ## Title
        pagetitle = soup.find('div',{'id':'pagetitle'})
        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -156,7 +156,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -115,7 +115,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
            a = info.find('a')
            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                break
 		

@@ -101,7 +101,7 @@ class NickAndGregNetAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -158,7 +158,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
        for info in asoup.findAll('table', {'class' : 'border'}):
            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
            if a != None:
-                self.story.setMetadata('title',a.text)
+                self.story.setMetadata('title',stripHTML(a))
                break
 		

@@ -158,7 +158,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -136,7 +136,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
        ## Title
        b = soup.find('div', {'id' : 'nav25'})
        a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.  /fanfiction/stories.php?psid=125
        a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+"))
@@ -180,7 +180,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+'))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -136,7 +136,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -251,8 +251,10 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX

        logger.debug('Getting chapter text from: %s' % url)
        data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        data = data.replace("HTML>","div>")
+        
+        soup = bs.BeautifulSoup(data)

        #print("soup:%s"%soup)
        tag = soup.find('td', {'class' : 'story'})
@@ -83,7 +83,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -147,7 +147,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
        #find first metadata block
        mb = cell.div.findNextSibling('div')
        #Get meta...
-        self.story.setMetadata('title', mb.b.string)
+        self.story.setMetadata('title', stripHTML(mb.b))
        #strip out brackets on rating
        self.story.setMetadata('rating', mb.span.string[1:-1])
        #Completion status is denoted by the presence of this image:
@@ -181,7 +181,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
        
        ## Title
        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -134,7 +134,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -130,7 +130,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -138,7 +138,7 @@ class QafFicComAdapter(BaseSiteAdapter):
        self.story.setMetadata('author',aut.string)
        aut.extract()
        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])

        # Find the chapters:
        chapters=soup.find('select')
@@ -22,7 +22,6 @@ import re
 import urllib2
 import cookielib as cl
 from datetime import datetime
-import json

 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
@@ -126,10 +125,12 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
        
        self.story.setMetadata('authorId',ahref.split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+ahref)
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))

        # title, remove byauthorname.
-        self.story.setMetadata('title',h2.text[:h2.text.index("by"+a.string)])
+        auth=stripHTML(a)
+        title=stripHTML(h2)
+        self.story.setMetadata('title',title[:title.index(" by "+auth)])

        dates = soup.findAll('span', {'class':'date'})
        if dates: # only for multi-chapter
@@ -110,7 +110,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
        self.story.setMetadata('author',aut.string)
        aut.extract()
        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)])

        # Find the chapters:
        chapters=soup.find('select')
@@ -181,7 +181,7 @@ class ScarHeadNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -139,7 +139,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -140,7 +140,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -143,7 +143,7 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h1')
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -155,7 +155,7 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
        for info in asoup.findAll('table', {'cellpadding' : '5'}):
            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
            if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                break
                
        # Find the chapters:
@@ -130,7 +130,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -122,7 +122,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
        # formating, so it's a little ugly.

        title = authorSoup.find('a',{'href':'viewstory.php?sid='+self.story.getMetadata('storyId')})
-        self.story.setMetadata('title',title.string)
+        self.story.setMetadata('title',stripHTML(title))
        titleblock=title.parent.parent
        
        chapterselect=soup.find('select',{'name':'chapter'})
@@ -106,7 +106,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
        self.story.setMetadata('author',aut.string)
        aut.extract()
        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)])

        # Find the chapters:
        chapters=soup.findAll('div', {'class' : 'content'})
@@ -104,7 +104,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
        
        a.find('em').extract()
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))

        # Find the chapters: chapterview.asp?sid=7000&cid=30919
        chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$"))
@@ -133,7 +133,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -99,7 +99,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -179,7 +179,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -177,7 +177,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -165,7 +165,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
            for info in asoup.findAll('td', {'class' : 'highlightcolor1'}):
                a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
                if a != None:
-                    self.story.setMetadata('title',a.text)                
+                    self.story.setMetadata('title',stripHTML(a))
                    found = 1
                    break
            index=index+1
@@ -165,7 +165,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -131,7 +131,7 @@ class TokraFandomnetComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -231,10 +231,10 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
        BtVS = True
        BtVSNonX = False
        for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
-            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
+            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'Non-BTVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
                self.story.addToList('category',cat.string)
            else:
-                if 'Non-BtVS' in cat.string:
+                if 'Non-BtVS' in cat.string or 'Non-BTVS' in cat.string:
                    BtVS = False
                if 'BtVS/AtS Non-Crossover' == cat.string:
                    BtVSNonX = True
@@ -100,7 +100,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find the chapters:
        chapters=soup.find('ol', {'class' : 'chapters'})
@@ -130,7 +130,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php"))
@@ -144,7 +144,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
        
        ## Title
        a = pagetitlediv.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitlediv.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -115,7 +115,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -22,6 +22,7 @@ import re
 import urllib2

 from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions

 from base_adapter import BaseSiteAdapter,  makeDate
@@ -70,7 +71,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
                raise e

        # pull title(title) and author from the HTML title.
-        title = soup.find('title').string
+        title = stripHTML(soup.find('title'))
        logger.debug('Title: %s' % title)
        title = title.split('::')[1].strip()
        self.story.setMetadata('title',title.split(' by ')[0].strip())
@@ -171,7 +171,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -101,7 +101,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -113,7 +113,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        alist = pt.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -77,6 +77,8 @@ class BaseSiteAdapter(Configurable):
        self.is_adult=False
        
        self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+        ## Specific UA because too many sites are blocking the default python UA.
+        self.opener.addheaders = [('User-agent', self.getConfig('user_agent'))]
        self.storyDone = False
        self.metadataDone = False
        self.story = Story(configuration)
@@ -245,6 +247,10 @@ class BaseSiteAdapter(Configurable):
            self.metadataDone = True
        return self.story

+    def hookForUpdates(self,chaptercount):
+        "Usually not needed."
+        return chaptercount
+
    ###############################
    
    @staticmethod
@@ -377,7 +383,7 @@ fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"0
           "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
           "November":"11", "December":"12" }

-def makeDate(string,format):
+def makeDate(string,dateform):
    # Surprise!  Abstracting this turned out to be more useful than
    # just saving bytes.

@@ -386,10 +392,10 @@ def makeDate(string,format):
    # there's non-english content. -- ficbook.net now makes that a
    # lie.  It has to do something even more complicated to get
    # Russian month names correct everywhere.
-    do_abbrev = "%b" in format
+    do_abbrev = "%b" in dateform
        
-    if "%B" in format or do_abbrev:
-        format = format.replace("%B","%m").replace("%b","%m")
+    if "%B" in dateform or do_abbrev:
+        dateform = dateform.replace("%B","%m").replace("%b","%m")
        for (name,num) in fullmon.items():
            if do_abbrev:
                name = name[:3] # first three for abbrev
@@ -397,5 +403,5 @@ def makeDate(string,format):
                string = string.replace(name,num)
                break
            
-    return datetime.datetime.strptime(string,format)
+    return datetime.datetime.strptime(string,dateform)

@@ -7,6 +7,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'

+import logging
+logger = logging.getLogger(__name__)
+
 import re, os, traceback
 from zipfile import ZipFile
 from xml.dom.minidom import parseString
@@ -72,7 +75,7 @@ def get_update_data(inputio,
                # remove all .. and the path part above it, if present.
                # Mostly for epubs edited by Sigil.
                src = re.sub(r"([^/]+/\.\./)","",src)
-                print("epubutils: found pre-existing cover image:%s"%src)
+                #print("epubutils: found pre-existing cover image:%s"%src)
                oldcoverimghref = src
                oldcoverimgdata = epub.read(src)
                for item in contentdom.getElementsByTagName("item"):
@@ -81,8 +84,8 @@ def get_update_data(inputio,
                        break
                oldcover = (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
            except Exception as e:
-                print("Cover Image %s not found"%src)
-                print("Exception: %s"%(unicode(e)))
+                logger.warn("Cover Image %s not found"%src)
+                logger.warn("Exception: %s"%(unicode(e)))
                traceback.print_exc()

    filecount = 0
@@ -118,8 +121,8 @@ def get_update_data(inputio,
                                images[longdesc] = data
                                img['src'] = img['longdesc']
                            except Exception as e:
-                                print("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
-                                print("Exception: %s"%(unicode(e)))
+                                logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
+                                logger.warn("Exception: %s"%(unicode(e)))
                                traceback.print_exc()
                        soup = soup.find('body')
                        # ffdl epubs have chapter title h3
@@ -143,8 +146,8 @@ def get_update_data(inputio,
    except:
        pass
                    
-    for k in images.keys():
-        print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
+    #for k in images.keys():
+        #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)

 def get_path_part(n):
@@ -15,6 +15,9 @@
 # limitations under the License.
 #

+import logging
+logger = logging.getLogger(__name__)
+
 import re

 def _unirepl(match):
@@ -31,7 +34,7 @@ def _unirepl(match):
    except:
        # This way, at least if there's more of entities out there
        # that fail, it doesn't blow the entire download.
-        print "Numeric entity translation failed, skipping: &#x%s%s"%(match.group(1),match.group(2))
+        logger.warn("Numeric entity translation failed, skipping: &#x%s%s"%(match.group(1),match.group(2)))
        retval = ""
    return retval

@@ -69,8 +72,9 @@ def removeEntities(text):
    
    if text is None:
        return ""
-    if not (isinstance(text,str) or isinstance(text,unicode)):
-        return str(text)
+    
+    if not isinstance(text,basestring):
+        return unicode(text)
    
    try:
        t = text.decode('utf-8')
@@ -349,6 +349,8 @@ class Story(Configurable):
                return removeAllEntities(value)
            else:
                return value
+        else: #if self.getConfig("default_value_"+key):
+            return self.getConfig("default_value_"+key)
        
    def getAllMetadata(self,
                       removeallentities=False,
@@ -411,6 +413,9 @@ class Story(Configurable):
        if not value in self.metadata[listname]:
            self.metadata[listname].append(value)

+        if listname == 'category' and self.getConfig('add_genre_when_multi_category') and len(self.metadata[listname]) > 1:
+            self.addToList('genre',self.getConfig('add_genre_when_multi_category'))
+
    def isList(self,listname):
        'Everything set with an include_in_* is considered a list.'
        return self.hasConfig("include_in_"+listname) or \
@@ -482,6 +487,8 @@ class Story(Configurable):

        if None in subjectset:
            subjectset.remove(None)
+        if '' in subjectset:
+            subjectset.remove('')

        return list(subjectset | set(self.getConfigList("extratags")))
            
@@ -624,7 +631,7 @@ class Story(Configurable):
                    ext)
                self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                
-            logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
+            #logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
        else:
            newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc']
            
@@ -221,7 +221,7 @@ class BaseStoryWriter(Configurable):
                    lastupdated=self.story.getMetadataRaw('dateUpdated').date()
                    fileupdated=datetime.datetime.fromtimestamp(os.stat(outfilename)[8]).date()
                    if fileupdated > lastupdated:
-                        print "File(%s) Updated(%s) more recently than Story(%s) - Skipping" % (outfilename,fileupdated,lastupdated)
+                        logger.warn("File(%s) Updated(%s) more recently than Story(%s) - Skipping" % (outfilename,fileupdated,lastupdated))
                        return
            if not metaonly:        
                self.story = self.adapter.getStory() # get full story
@@ -57,9 +57,17 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>Minor optimizations.</li>
+              <li>Fixes for dark-solace.org.</li>
 	    </ul>
 	  </p>
+          
+          <h3>fanfiction.net</h3>
+          <p>
+            Fanfiction.net appears to be blocking access from Google
+            App Engine, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
 	  
 	  <p>
 	    Questions?  Check out our 
@@ -69,7 +77,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-67.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-77.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -16,7 +16,22 @@
 [defaults]

 ## [defaults] section applies to all formats and sites but may be
-## overridden at several levels
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category

 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
@@ -124,6 +139,17 @@ extratags: FanFiction
 #extraships:
 #extrawarnings:

+## Add this to genre if there's more than one category.
+#add_genre_when_multi_category: Crossover
+
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -246,6 +272,8 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true

+## User-agent
+user_agent:FFDL/1.7

 ## Each output format has a section that overrides [defaults]
 [html]
@@ -416,8 +444,8 @@ nook_img_fix:true
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true

-## Each site has a section that overrides [defaults] *and* the format
-## sections test1.com specifically is not a real story site.  Instead,
+## Each site has a section that overrides [defaults].
+## test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
@@ -474,7 +502,8 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true

 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
+## If there's ever more than 4 series, add series04,series04Url etc.
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 fandoms_label:Fandoms
 freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
@@ -490,7 +519,7 @@ bookmarks_label:Bookmarks
 include_in_freefromtags:freeformtags

 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url

 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
@@ -724,6 +753,23 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword

+[fictionpad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+extra_valid_entries:followers,comments,views,likes,dislikes
+#extra_titlepage_entries:followers,comments,views,likes,dislikes
+
+followers_label:Followers
+comments_label:Comments
+views_label:Views
+likes_label:Likes
+dislikes_label:Dislikes
+
 [finestories.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -774,6 +820,10 @@ extracategories:Highlander
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death

+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/public/style_emoticons/.*
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1081,6 +1131,7 @@ context_label:Context
 type_label:Type of Couple

 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1108,7 +1159,12 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:

@@ -1130,12 +1186,13 @@ extracategories:My Little Pony: Friendship is Magic

 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups

 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1147,6 +1204,13 @@ short_description_label:Short Summary
 ## when a password is required rather than prompting every time.
 #fail_on_password: false

+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 [www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -2,6 +2,24 @@
 ## like.  Uncomment options by removing the '#' in front of them.

 [defaults]
+## [defaults] section applies to all formats and sites but may be
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category
+
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
 #is_adult:true
@@ -15,6 +33,9 @@
 #strip_chapter_numbers:true
 #add_chapter_numbers:true

+## Add this to genre if there's more than one category.
+#add_genre_when_multi_category: Crossover
+
 [epub]
 ## include images from img tags in the body and summary of stories.
 ## Images will be converted to jpg for size if possible.  Images work
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Jim Miller	ba7b718170	Fix for latest ffnet tweak.	2013-10-23 19:31:28 -05:00
Jim Miller	a43d9f7a03	Added tag fanfictionDownLoader-4.4.78 for changeset 3f7539804f25	2013-10-19 11:24:42 -05:00
Jim Miller	4b17ecf6fa	Added tag calibre-plugin-1.7.49 for changeset 3f7539804f25	2013-10-19 11:24:28 -05:00
Jim Miller	b6dd579c93	Bump versions.	2013-10-19 11:24:18 -05:00
Jim Miller	aa685a4c7d	Reorg Basic config a little and wrap the tab widget in a scrollarea.	2013-10-18 15:01:26 -05:00
Jim Miller	b0248daf07	dark-solace.org is now insisting on www.dark-solace.org, for login at least.	2013-10-17 19:26:31 -05:00
Jim Miller	be5fe49ab8	Added tag FanFictionDownLoader-4.4.77 for changeset e85271dd28c9	2013-10-13 14:33:42 -05:00
Jim Miller	f42f440f1b	Added tag calibre-plugin-1.7.48 for changeset e85271dd28c9	2013-10-13 14:33:33 -05:00
Jim Miller	cbf50a36ee	Trust ffnet to change their html right after I released a new version.	2013-10-13 14:33:26 -05:00
Jim Miller	fda0fda84e	Added tag FanFictionDownLoader-4.4.76 for changeset c60e9ebb849d	2013-10-13 11:00:00 -05:00
Jim Miller	c6f5c524be	Added tag calibre-plugin-1.7.47 for changeset c60e9ebb849d	2013-10-13 10:59:48 -05:00
Jim Miller	00a46a7cc0	Bump versions, make "Mark added/updated books when finished" default False.	2013-10-13 10:59:36 -05:00
Jim Miller	5c1ca5a188	Mark failed updates even when all failed.	2013-10-12 11:16:31 -05:00
Jim Miller	17c6dddfac	Add user/pass support for 18+ rated stories on fictionpad.com	2013-10-11 18:14:35 -05:00
Jim Miller	33451f1119	Make marking updated books optional since it's now visible.	2013-10-11 15:04:45 -05:00
Jim Miller	192ade1fca	Adding fictionpad.com. Doesn't support login and 18+ yet.	2013-10-11 15:04:21 -05:00
Jim Miller	aa286a9d0d	Change prints to logger calls and clean up some of them.	2013-10-10 21:34:46 -05:00
Jim Miller	a1c19ac12e	Added tag FanFictionDownLoader-4.4.75 for changeset 3779dfd8a385	2013-09-29 14:22:03 -05:00
Jim Miller	85b6e305be	Added tag calibre-plugin-1.7.46 for changeset 3779dfd8a385	2013-09-29 14:21:46 -05:00
Jim Miller	40fb061a86	Bump versions.	2013-09-29 14:21:33 -05:00
Jim Miller	7fdc59691f	do_update_hook for fimf, find newest chapter and update from there.	2013-09-29 14:19:18 -05:00
Jim Miller	78845d0d1e	TtH uses both Non-BtVS/... and Non-BTVS/...	2013-09-29 14:17:41 -05:00
Jim Miller	786b1d5cdf	Exclude indeath.net emoticons from being covers.	2013-09-26 21:45:32 -05:00
besnef	b5d176f007	Merge fix for inDeath.net	2013-09-26 22:20:19 -04:00
besnef	1c71dedfa3	Fix for inDeath.net	2013-09-21 14:57:45 -04:00
Jim Miller	2484f0f5c7	Added tag FanFictionDownLoader-4.4.74 for changeset 53f662218344	2013-09-21 13:24:25 -05:00
Jim Miller	a087927929	Added tag calibre-plugin-1.7.45 for changeset 53f662218344	2013-09-21 13:24:13 -05:00
Jim Miller	e64d49e3e6	Bump versions.	2013-09-21 13:23:58 -05:00
Jim Miller	d9ad95467b	Set custom column only if there's a value (mostly for int/float columns).	2013-09-21 12:53:28 -05:00
Jim Miller	668e0e08b5	Add ffnet warning to status page on error.	2013-09-16 19:06:44 -05:00
Jim Miller	2b0b4ca2af	Don't include empty string tags.	2013-09-16 18:55:02 -05:00
Jim Miller	dca3707eac	Fix for whofic.com.	2013-09-16 13:10:09 -05:00
Jim Miller	a3b23857a3	Added tag FanFictionDownLoader-4.4.73 for changeset 07134453ebb0	2013-09-14 12:25:49 -05:00
Jim Miller	9cdccb576c	Added tag calibre-plugin-1.7.44 for changeset 07134453ebb0	2013-09-14 12:25:28 -05:00
Jim Miller	cf24021f14	Bump versions.	2013-09-14 12:25:17 -05:00
Jim Miller	0bbe6287b6	Really commit changes to portkey.org.	2013-09-12 22:57:04 -05:00
Jim Miller	0c585b6e52	Remove call do dateutil.parser in fimf. Requires dateutil package for CLI.	2013-09-12 22:41:09 -05:00
Jim Miller	aacf6dc6a2	Switch portkey.org to non-stone soup and brute-force remove <HTML> tags.	2013-09-12 22:40:41 -05:00
Jim Miller	9f56952950	Fix for empty word num into int column.	2013-09-12 19:11:30 -05:00
Jim Miller	c49cad8889	Added tag FanFictionDownLoader-4.4.72 for changeset 75db10f77cb4	2013-09-11 10:34:07 -05:00
Jim Miller	bf1d8e18bf	Added tag calibre-plugin-1.7.43 for changeset 75db10f77cb4	2013-09-11 10:33:55 -05:00
Jim Miller	e41c7dabeb	Fix fimfic dates, add fimfic site specific 'groups' metadata. Bump versions.	2013-09-11 10:32:57 -05:00
Jim Miller	face0af074	Add default_value_status (etc) feature.	2013-09-10 10:24:45 -05:00
Jim Miller	02f38eaa2e	Added tag FanFictionDownLoader-4.4.71 for changeset b4e7c2ec827d	2013-09-09 09:11:47 -05:00
Jim Miller	d4b5438bc5	Added tag calibre-plugin-1.7.42 for changeset b4e7c2ec827d	2013-09-09 09:11:29 -05:00
Jim Miller	83db111ba7	Changes for ffnet--longer sleep, config UA, bump versions.	2013-09-09 09:10:56 -05:00
Jim Miller	2eb8c27c80	fanfiction.net blocked by IP - index.html	2013-09-08 14:55:52 -05:00
Jim Miller	a79066750c	Changes on index.html.	2013-09-08 14:48:05 -05:00
Jim Miller	3857a6a89b	Added tag FanFictionDownLoader-4.4.70 for changeset 9d99f91e062e	2013-09-08 14:33:14 -05:00
Jim Miller	016d4af149	Added tag calibre-plugin-1.7.41 for changeset 9d99f91e062e	2013-09-08 14:33:00 -05:00
Jim Miller	a4e499866e	Fix for AO3 stories without series, bump versions.	2013-09-08 14:32:41 -05:00
Jim Miller	5736f4f181	Added tag calibre-plugin-1.7.40 for changeset db9a23c65412	2013-09-07 14:43:55 -05:00
Jim Miller	e04453d73f	Removed tag calibre-plugin-1.7.340	2013-09-07 14:43:47 -05:00
Jim Miller	c3778502dc	Added tag FanFictionDownLoader-4.4.69 for changeset db9a23c65412	2013-09-07 14:42:23 -05:00
Jim Miller	925b12e776	Added tag calibre-plugin-1.7.340 for changeset db9a23c65412	2013-09-07 14:41:47 -05:00
Jim Miller	62ed503d10	Bump versions.	2013-09-07 14:41:30 -05:00
Jim Miller	7e35d0b710	Add User-agent="FFDL/1.7" for all adapters. (Remove from specific adapters.)	2013-09-07 14:36:48 -05:00
Jim Miller	20c7e7f075	Fixes for changes to harrypotterfanfictioncom.	2013-09-07 14:36:13 -05:00
Jim Miller	9f61cff7cd	Additional series as site specific data for AO3.	2013-09-07 14:35:32 -05:00
Jim Miller	cb5587b0c7	Fix for calibre pre-1.0.0.	2013-09-07 14:32:48 -05:00
Jim Miller	f2a74c9ffc	Better doc section override order in ini files.	2013-09-04 14:33:29 -05:00
Jim Miller	eb0f013550	Added tag calibre-plugin-1.7.39 for changeset 1af0c2a05450	2013-08-31 19:57:27 -05:00
Jim Miller	6674e19f94	Remove extra code for unchanged author link for calibre 1.0.0, bump version.	2013-08-31 19:50:18 -05:00
Jim Miller	29b1ca9a7e	Attempted debuging for another Mac issue.	2013-08-31 19:42:58 -05:00
Jim Miller	7cae65300a	Feature: add_genre_when_multi_category setting	2013-08-31 15:38:44 -05:00
Jim Miller	88fe13f603	Work around calibre's auto-split when author contains 'and' 'with' ',' or '&'.	2013-08-29 19:23:23 -05:00
Jim Miller	a9b6989341	Added tag calibre-plugin-1.7.38 for changeset d44d5f1006f6	2013-08-27 17:11:43 -05:00
Jim Miller	a54277058c	Bump version.	2013-08-27 17:11:32 -05:00
Jim Miller	4c27e19aa8	'Fix' author link updates bumping last_modified. User msg on metadata error.	2013-08-27 14:16:44 -05:00
Jim Miller	6f5ed2bcac	Added tag calibre-plugin-1.7.37 for changeset 73fa3ac021d3	2013-08-24 22:43:11 -05:00
Jim Miller	cc0bccd420	Added tag calibre-plugin-1.7.37 for changeset 73fa3ac021d3	2013-08-24 22:40:12 -05:00
Jim Miller	85bbcac92e	Fold only-case-different tags together to work around calibre bug 1216398 https://bugs.launchpad.net/calibre/+bug/1216398	2013-08-24 22:39:36 -05:00
Jim Miller	f8776e118a	Remove superfluous Fanfiction category set in adapter.	2013-08-24 22:38:30 -05:00
Jim Miller	31b7257496	Strip HTML from story titles in all the adapters that don't already.	2013-08-21 09:19:58 -05:00
Jim Miller	b5d00951de	Added tag calibre-plugin-1.7.36 for changeset e6f30eaf1bc7	2013-08-20 08:47:30 -05:00
Jim Miller	c73f7693a4	Bump PI version.	2013-08-20 08:47:18 -05:00
Jim Miller	e35927c9a2	Add options to force author & title values into author_sort and title_sort.	2013-08-18 17:19:26 -05:00
Jim Miller	e8e93b010a	Make check for changing story URL configurable.	2013-08-18 15:52:51 -05:00
Jim Miller	6414a09dbd	Added tag FanFictionDownLoader-4.4.68 for changeset db28f113a88e	2013-08-06 17:18:42 -05:00
Jim Miller	d5e7907068	Added tag calibre-plugin-1.7.35 for changeset db28f113a88e	2013-08-06 17:18:29 -05:00