Bump versions.

Reorg Basic config a little and wrap the tab widget in a scrollarea.
dark-solace.org is now insisting on www.dark-solace.org, for login at least.
2026-09-15 11:36:30 +08:00 · 2013-10-19 11:24:18 -05:00 · 2013-10-18 15:01:26 -05:00 · 2013-10-17 19:26:31 -05:00 · 2013-10-13 14:33:42 -05:00 · 2013-10-13 14:33:33 -05:00
33 changed files with 706 additions and 244 deletions
@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-68
+version: 4-4-78
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -7,6 +7,16 @@ __license__   = 'GPL v3'
 __copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'

+import sys
+if sys.version_info >= (2, 7):
+    import logging
+    logger = logging.getLogger(__name__)
+    loghandler=logging.StreamHandler()
+    loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+    logger.addHandler(loghandler)
+    loghandler.setLevel(logging.DEBUG)
+    logger.setLevel(logging.DEBUG)
+
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase

@@ -26,7 +36,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
    description         = 'UI plugin to download FanFiction stories from various sites.'
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (1, 7, 39)
+    version             = (1, 7, 49)
    minimum_calibre_version = (0, 8, 57)

    #: This field defines the GUI plugin class that contains all the code
@@ -7,6 +7,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'

+import logging
+logger = logging.getLogger(__name__)
+
 import traceback, copy, threading
 from collections import OrderedDict

@@ -126,9 +129,15 @@ class ConfigWidget(QWidget):
        label.setOpenExternalLinks(True)
        self.l.addWidget(label)

-        tab_widget = QTabWidget(self)
-        self.l.addWidget(tab_widget)

+        self.scroll_area = QScrollArea(self)
+        self.scroll_area.setFrameShape(QScrollArea.NoFrame)
+        self.scroll_area.setWidgetResizable(True)
+        self.l.addWidget(self.scroll_area)
+
+        tab_widget = QTabWidget(self)
+        self.scroll_area.setWidget(tab_widget)
+        
        self.basic_tab = BasicTab(self, plugin_action)
        tab_widget.addTab(self.basic_tab, 'Basic')

@@ -171,6 +180,7 @@ class ConfigWidget(QWidget):
        prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
        prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
        prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
+        prefs['mark'] = self.basic_tab.mark.isChecked()
        prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
@@ -375,8 +385,13 @@ class BasicTab(QWidget):
        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
        self.l.addWidget(self.lookforurlinhtml)

-        self.showmarked = QCheckBox("Show added/updated books when finished?",self)
-        self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.mark = QCheckBox("Mark added/updated books when finished?",self)
+        self.mark.setToolTip("Mark added/updated books when finished.  Use with option below.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.mark.setChecked(prefs['mark'])
+        self.l.addWidget(self.mark)
+
+        self.showmarked = QCheckBox("Show Marked books when finished?",self)
+        self.showmarked.setToolTip("Show Marked added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
        self.showmarked.setChecked(prefs['showmarked'])
        self.l.addWidget(self.showmarked)

@@ -437,15 +452,17 @@ class BasicTab(QWidget):
        topl.addWidget(defs_gb)

        horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(cali_gb)
-        horz.addWidget(rej_gb)
-        
-        horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(gui_gb)
-        horz.addWidget(misc_gb)

+        horz.addWidget(cali_gb)
+
+        vert = QVBoxLayout()
+        vert.addWidget(gui_gb)
+        vert.addWidget(misc_gb)
+        vert.addWidget(rej_gb)
+        
+        horz.addLayout(vert)
+        
+        topl.addLayout(horz)
        topl.insertStretch(-1)
        
    def set_collisions(self):
@@ -523,7 +540,7 @@ class PersonalIniTab(QWidget):
            self.ini.setFont(QFont("Courier",
                                   self.plugin_action.gui.font().pointSize()+1))
        except Exception as e:
-            print("Couldn't get font: %s"%e)
+            logger.error("Couldn't get font: %s"%e)
        self.ini.setLineWrapMode(QTextEdit.NoWrap)
        self.ini.setText(prefs['personal.ini'])
        self.l.addWidget(self.ini)
@@ -559,7 +576,7 @@ class ShowDefaultsIniDialog(QDialog):
            self.ini.setFont(QFont("Courier",
                                   get_gui().font().pointSize()+1))
        except Exception as e:
-            print("Couldn't get font: %s"%e)
+            logger.error("Couldn't get font: %s"%e)
        self.ini.setLineWrapMode(QTextEdit.NoWrap)
        self.ini.setText(text)
        self.ini.setReadOnly(True)
@@ -7,9 +7,15 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'

+import logging
+logger = logging.getLogger(__name__)
+
 import traceback, re
 from functools import partial

+import logging
+logger = logging.getLogger(__name__)
+
 import urllib
 import email

@@ -207,7 +213,7 @@ class AddNewDialog(SizePersistedDialog):
        self.groupbox.setCheckable(True)
        self.groupbox.setChecked(False)
        self.groupbox.setFlat(True)
-        print("style:%s"%self.groupbox.styleSheet())
+        #print("style:%s"%self.groupbox.styleSheet())
        self.groupbox.setStyleSheet(gpstyle)

        self.gbf = QFrame()
@@ -537,7 +543,7 @@ class LoopProgressDialog(QProgressDialog):
    def updateStatus(self):
        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
        self.setValue(self.i+1)
-        print(self.labelText())
+        #print(self.labelText())

    def do_loop(self):

@@ -558,7 +564,7 @@ class LoopProgressDialog(QProgressDialog):
        except Exception as e:
            book['good']=False
            book['comment']=unicode(e)
-            print("Exception: %s:%s"%(book,unicode(e)))
+            logger.error("Exception: %s:%s"%(book,unicode(e)))
            traceback.print_exc()
            
        self.updateStatus()
@@ -7,6 +7,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'

+import logging
+logger = logging.getLogger(__name__)
+
 import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
@@ -143,7 +146,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            file_path = os.path.join(calibre_config_dir,
                                     *("plugins/fanfictiondownloader_macmenuhack.txt".split('/')))
            file_path = os.path.abspath(file_path)
-            print("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
+            logger.debug("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
            self.macmenuhack = os.access(file_path, os.F_OK)
            return self.macmenuhack

@@ -387,7 +390,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        show_copy_button=False)

    def get_urls_from_page(self,url):
-        print("get_urls_from_page URL:%s"%url)
+        logger.debug("get_urls_from_page URL:%s"%url)
        if 'archiveofourown.org' in url:
            configuration = get_ffdl_config(url)
        else:
@@ -541,7 +544,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            return
            
        tdir = PersistentTemporaryDirectory(prefix='ffdl_anthology_')
-        print("tdir:\n%s"%tdir)
+        logger.debug("tdir:\n%s"%tdir)

        bookepubio = StringIO(db.format(book_id,'EPUB',index_is_id=True))
        
@@ -617,7 +620,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            '''%(len(urlmapfile),"</li><li>".join(urlmapfile.keys()))
            if not question_dialog(self.gui, 'Stories Removed',
                               text, show_copy_button=False):
-                print("Canceling anthology update due to removed stories.")
+                logger.debug("Canceling anthology update due to removed stories.")
                return

        # Now that we've 
@@ -696,7 +699,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            books = self.convert_urls_to_books(url_list)

        options['version'] = self.version
-        print(self.version)
+        logger.debug(self.version)
        
        #print("prep_downloads:%s"%books)

@@ -731,7 +734,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        '''

        url = book['url']
-        print("url:%s"%url)
+        logger.debug("url:%s"%url)
        mi = None

        if not merge: # skip reject list when merging.
@@ -787,7 +790,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            try:
                adapter.getStoryMetadataOnly()
            except exceptions.FailedToLogin, f:
-                print("Login Failed, Need Username/Password.")
+                logger.warn("Login Failed, Need Username/Password.")
                userpass = UserPassDialog(self.gui,url,f)
                userpass.exec_() # exec_ will make it act modal
                if userpass.status:
@@ -869,14 +872,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            
            if book['calibre_id'] != None:
                # updating an existing book.  Update mode applies.
-                print("update existing id:%s"%book['calibre_id'])
+                logger.debug("update existing id:%s"%book['calibre_id'])
                book_id = book['calibre_id']
                # No handling needed: OVERWRITEALWAYS,CALIBREONLY
                
            # only care about collisions when not ADDNEW
            elif collision != ADDNEW:
                # 'new' book from URL.  collision handling applies.
-                print("from URL(%s)"%url)
+                logger.debug("from URL(%s)"%url)
    
                # try to find by identifier url or uri first.
                searchstr = 'identifiers:"~ur(i|l):=%s"'%url.replace(":","|")
@@ -889,16 +892,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        ## meantime, if it matches the title *and* first
                        ## 100 authors, I'm prepared to assume it's a
                        ## match.
-                        print("reduce author list to 100 only when calibre < 0.8.61")
+                        logger.debug("reduce author list to 100 only when calibre < 0.8.61")
                        authlist = authlist[:100]
                    mi = MetaInformation(story.getMetadata("title", removeallentities=True),
                                         authlist)
                    identicalbooks = db.find_identical_books(mi)
                    if len(identicalbooks) > 0:
-                        print("existing found by title/author(s)")
+                        logger.debug("existing found by title/author(s)")
    
                else:
-                    print("existing found by identifier URL")
+                    logger.debug("existing found by identifier URL")
                    
                if collision == SKIP and identicalbooks:
                    raise NotGoingToDownload("Skipping duplicate story.","list_remove.png")
@@ -997,7 +1000,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                  suffix='.epub',
                                                  dir=options['tdir'])
                    db.copy_format_to(book_id,fileform,tmp,index_is_id=True)
-                    print("existing epub tmp:"+tmp.name)
+                    logger.debug("existing epub tmp:"+tmp.name)
                    book['epub_for_update'] = tmp.name

            if book_id != None and prefs['injectseries']:
@@ -1014,8 +1017,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            tmp = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
                                          suffix='.'+options['fileform'],
                                          dir=options['tdir'])
-            print("title:"+book['title'])
-            print("outfile:"+tmp.name)
+            logger.debug("title:"+book['title'])
+            logger.debug("outfile:"+tmp.name)
            book['outfile'] = tmp.name            
                
        return
@@ -1093,7 +1096,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if book['calibre_id'] and prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
            label = custom_columns[prefs['errorcol']]['label']
            if not book['good']:
-                print("record/update error message column %s %s"%(book['title'],book['url']))
+                logger.debug("record/update error message column %s %s"%(book['title'],book['url']))
                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True) # book['comment']
            else:
                db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
@@ -1101,7 +1104,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if not book['good']:
            return # only update errorcol on error.
        
-        print("add/update %s %s"%(book['title'],book['url']))
+        logger.debug("add/update %s %s"%(book['title'],book['url']))
        mi = self.make_mi_from_book(book)
                
        if options['collision'] != CALIBREONLY:
@@ -1113,7 +1116,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                self.update_metadata(db, book['calibre_id'], book, mi, options)
            except:
                det_msg = "".join(traceback.format_exception(*sys.exc_info()))+"\nStory Details:\n%s"%pretty_book(book)
-                print("Error Updating Metadata:\n%s"%det_msg)
+                logger.error("Error Updating Metadata:\n%s"%det_msg)
                error_dialog(self.gui,
                             "Error Updating Metadata",
                             "<p>An error has occurred while FFDL was updating calibre's metadata for <a href='%s'>%s</a>.</p>"%(book['url'],book['title'])+
@@ -1124,7 +1127,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
    def update_books_finish(self, book_list, options={}, showlist=True):
        '''Notify calibre about updated rows, update external plugins
        (Reading Lists & Count Pages) as configured'''
-        
+
        add_list = filter(lambda x : x['good'] and x['added'], book_list)
        add_ids = [ x['calibre_id'] for x in add_list ]
        update_list = filter(lambda x : x['good'] and not x['added'], book_list)
@@ -1152,7 +1155,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if self.gui.cover_flow:
            self.gui.cover_flow.dataChanged()

-        if showlist: # don't use with anthology
+        if showlist and prefs['mark']: # don't use with anthology
            db = self.gui.current_db
            marked_ids = dict()
            marked_text = "ffdl_success"
@@ -1170,19 +1173,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                self.gui.library_view.sort_by_named_field('marked', True)
                
        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
-
-        1+1
-        
-        print("all done, remove temp dir.")
-
-        1+1
-        
        remove_dir(options['tdir'])
-
-        1+1
-        
-        print("removed temp dir.")
-
        
        if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
            cp_plugin = self.gui.iactions['Count Pages']
@@ -1277,7 +1268,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        (good_list,bad_list,options) = payload
        total_good = len(good_list)

-        print("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['listorder']) for x in good_list ]))
+        logger.debug("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['listorder']) for x in good_list ]))

        good_list = sorted(good_list,key=lambda x : x['listorder'])
        bad_list = sorted(bad_list,key=lambda x : x['listorder'])
@@ -1303,8 +1294,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            # if still 'good', make a temp file to write the output to.
            tmp = PersistentTemporaryFile(suffix='.'+options['fileform'],
                                          dir=options['tdir'])
-            print("title:"+mergebook['title'])
-            print("outfile:"+tmp.name)
+            logger.debug("title:"+mergebook['title'])
+            logger.debug("outfile:"+tmp.name)
            mergebook['outfile'] = tmp.name

        self.get_epubmerge_plugin().do_merge(tmp.name,
@@ -1325,7 +1316,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

        self.gui.status_bar.show_message(_('FFDL Adding/Updating books.'))

-        if good_list or (bad_list and prefs['errorcol'] != '' and prefs['errorcol'] in self.gui.library_view.model().custom_columns):
+        if good_list or prefs['mark'] or (bad_list and prefs['errorcol'] != '' and prefs['errorcol'] in self.gui.library_view.model().custom_columns):
            LoopProgressDialog(self.gui,
                               good_list+bad_list,
                               partial(self.update_books_loop, options=options, db=self.gui.current_db),
@@ -1338,22 +1329,24 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        '''Update custom error column if configured.'''
        (empty_list,book_list,options)=payload
        custom_columns = self.gui.library_view.model().custom_columns
-        if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+        if prefs['mark'] or (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
            self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%len(book_list)))
            label = custom_columns[prefs['errorcol']]['label']
            LoopProgressDialog(self.gui,
                               book_list,
                               partial(self.update_error_column_loop, db=self.gui.current_db, label=label),
-                               partial(self.update_books_finish, options=options, showlist=False),
+                               partial(self.update_books_finish, options=options),
                               init_label="Updating calibre for BAD FanFiction stories...",
                               win_title="Update calibre for BAD FanFiction stories",
                               status_prefix="Updated")

    def update_error_column_loop(self,book,db=None,label='errorcol'):
        if book['calibre_id']:
-            print("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
-            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+            custom_columns = self.gui.library_view.model().custom_columns
+            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
+                logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)

    def add_book_or_update_format(self,book,options,prefs,mi=None):
        db = self.gui.current_db
@@ -1382,7 +1375,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
            for fmt in fmts:
                if fmt != formmapping[options['fileform']]:
-                    print("remove f:"+fmt)
+                    logger.debug("remove f:"+fmt)
                    db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False

        return book_id
@@ -1423,7 +1416,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                try:
                    db.set_cover(book_id, epubmi.cover_data[1])
                except:
-                    print("Failed to set_cover, skipping")
+                    logger.info("Failed to set_cover, skipping")

        # implement 'newonly' flags here by setting to the current
        # value again.
@@ -1436,7 +1429,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        try:
                            mi.__setattr__(col,oldmi.__getattribute__(col))
                        except AttributeError:
-                            print("AttributeError? %s"%col)
+                            logger.warn("AttributeError? %s"%col)
                            pass

        db.set_metadata(book_id,mi)
@@ -1451,25 +1444,26 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        for col, meta in prefs['custom_cols'].iteritems():
            #print("setting %s to %s"%(col,meta))
            if col not in custom_columns:
-                print("%s not an existing column, skipping."%col)
+                logger.debug("%s not an existing column, skipping."%col)
                continue
            coldef = custom_columns[col]
            if col in prefs['custom_cols_newonly'] and prefs['custom_cols_newonly'][col] and not book['added']:
-                print("Skipping custom column(%s) update, set to New Books Only"%coldef['name'])
+                logger.debug("Skipping custom column(%s) update, set to New Books Only"%coldef['name'])
                continue
            if not meta.startswith('status-') and meta not in book['all_metadata'] or \
                    meta.startswith('status-') and 'status' not in book['all_metadata']:
-                print("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))
+                logger.debug("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))
                continue
            if meta not in permitted_values[coldef['datatype']]:
-                print("%s not a valid column type for %s, skipping."%(col,meta))
+                logger.debug("%s not a valid column type for %s, skipping."%(col,meta))
                continue
            label = coldef['label']
            if coldef['datatype'] in ('enumeration','text','comments','datetime','series'):
                db.set_custom(book_id, book['all_metadata'][meta], label, commit=False)
            elif coldef['datatype'] in ('int','float'):
                num = unicode(book['all_metadata'][meta]).replace(",","")
-                db.set_custom(book_id, num, label=label, commit=False)
+                if num != '':
+                    db.set_custom(book_id, num, label=label, commit=False)
            elif coldef['datatype'] == 'bool' and meta.startswith('status-'):
                if meta == 'status-C':
                    val = book['all_metadata']['status'] == 'Completed'
@@ -1490,7 +1484,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )

                    if meta not in book['all_metadata']:
-                        print("No value for %s, skipping custom column(%s) update."%(meta,custcol))
+                        logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
                        continue
                    
                    if custcol not in custom_columns:
@@ -1508,7 +1502,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                val = unicode(book['all_metadata'][meta]).replace(",","")
                        else:
                            val = book['all_metadata'][meta]
-                        db.set_custom(book_id, val, label=label, commit=False)
+                        if val != '':
+                            db.set_custom(book_id, val, label=label, commit=False)

                    if flag == 'a':
                        vallist = []
@@ -1530,8 +1525,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        # Moved down so author's already in the DB.
        if 'authorUrl' in book['all_metadata']:
            authurls = book['all_metadata']['authorUrl'].split(", ")
+            authorlist = [ a.replace('&',';') for a in book['author'] ]
            if hasattr(db, 'new_api'): # new_api starts in calibre 1.0.0
-                authorlist = [ a.replace('&',';') for a in book['author'] ]
                authorids = db.new_api.get_item_ids('authors',authorlist)
                authordata = db.new_api.author_data(authorids.values())
                # print("\n\nauthorids:%s"%authorids)
@@ -1577,9 +1572,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                            break

                if setting_name:
-                    print("Generate Cover Setting from generate_cover_settings(%s)"%line)
+                    logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%line)
                    if setting_name not in gc_plugin.get_saved_setting_names():
-                        print("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
+                        logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
                        setting_name = None
                
            if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
@@ -1589,7 +1584,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                setting_name =  prefs['gc_site_settings']['Default']
                
            if setting_name:
-                print("Running Generate Cover with settings %s."%setting_name)
+                logger.debug("Running Generate Cover with settings %s."%setting_name)
                realmi = db.get_metadata(book_id, index_is_id=True)
                gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)

@@ -1907,7 +1902,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        break
            
            configuration = get_ffdl_config(book['url'],fileform)
-            print("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
+            logger.debug("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
            if configuration.getConfig('anthology_title_pattern'):
                tmplt = Template(configuration.getConfig('anthology_title_pattern'))
                book['title'] = tmplt.safe_substitute({'title':book['title']}).encode('utf8')
@@ -8,6 +8,9 @@ __copyright__ = '2012, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'

+import logging
+logger = logging.getLogger(__name__)
+
 import time, os, traceback

 from StringIO import StringIO
@@ -38,13 +41,13 @@ def do_download_worker(book_list, options,
    '''
    server = Server(pool_size=cpus)

-    print(options['version'])
+    logger.info(options['version'])
    total = 0
    alreadybad = []
    # Queue all the jobs
-    print("Adding jobs for URLs:")
+    logger.info("Adding jobs for URLs:")
    for book in book_list:
-        print("%s"%book['url'])
+        logger.info("%s"%book['url'])
        if book['good']:
            total += 1
            args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
@@ -87,19 +90,19 @@ def do_download_worker(book_list, options,
        count = count + 1
        notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
        # Add this job's output to the current log
-        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
-        print(job.details)
+        logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        logger.info(job.details)

        if count >= total:
            # All done!  Output some lists for convenience of some users.
-            print("Successfully downloaded:")
+            logger.info("Successfully downloaded:")
            for book in book_list:
                if book['good']:
-                    print("%s %s"%(book['title'],book['url']))
-            print("\nUnsuccessful:")
+                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\nUnsuccessful:")
            for book in book_list:
                if not book['good']:
-                    print("%s %s"%(book['title'],book['url']))
+                    logger.info("%s %s"%(book['title'],book['url']))
            break

    server.close()
@@ -147,7 +150,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):

        ## No need to download at all.  Shouldn't ever get down here.
        if options['collision'] in (CALIBREONLY):
-            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
            book['comment'] = 'Metadata collected.'
            
        ## checks were done earlier, it's new or not dup or newer--just write it.
@@ -170,7 +173,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                if adapter.logfile:
                    adapter.logfile = adapter.logfile.replace("span id","span notid")
            
-            print("write to %s"%outfile)
+            logger.info("write to %s"%outfile)
            writer.writeStory(outfilename=outfile, forceOverwrite=True)
            book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
            
@@ -198,9 +201,13 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
            # dup handling from ffdl_plugin needed for anthology updates.
            if chaptercount > urlchaptercount:
                raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
-                            
-            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
-            print("write to %s"%outfile)
+
+            if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
+                    and adapter.getConfig("do_update_hook"):
+                chaptercount = adapter.hookForUpdates(chaptercount)
+
+            logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+            logger.info("write to %s"%outfile)

            writer.writeStory(outfilename=outfile, forceOverwrite=True)
            
@@ -222,7 +229,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
            
            log = Log(level=Log.DEBUG)
            # report = []
-            polish({outfile:outfile}, opts, log, print) # report.append
+            polish({outfile:outfile}, opts, log, logger.info) # report.append
        
    except NotGoingToDownload as d:
        book['good']=False
@@ -234,7 +241,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
        book['comment']=unicode(e)
        book['icon']='dialog_error.png'
        book['status'] = 'Error'
-        print("Exception: %s:%s"%(book,unicode(e)))
+        logger.info("Exception: %s:%s"%(book,unicode(e)))
        traceback.print_exc()
        
    #time.sleep(10)
@@ -32,6 +32,7 @@ default_prefs['updateepubcover'] = False
 default_prefs['keeptags'] = False
 default_prefs['suppressauthorsort'] = False
 default_prefs['suppresstitlesort'] = False
+default_prefs['mark'] = False
 default_prefs['showmarked'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
@@ -16,7 +16,22 @@
 [defaults]

 ## [defaults] section applies to all formats and sites but may be
-## overridden at several levels
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category

 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
@@ -155,6 +170,14 @@ extratags: FanFiction
 ## Add this to genre if there's more than one category.
 #add_genre_when_multi_category: Crossover

+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -262,6 +285,8 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true

+## User-agent
+user_agent:FFDL/1.7

 ## Each output format has a section that overrides [defaults]
 [html]
@@ -450,8 +475,8 @@ nook_img_fix:true
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true

-## Each site has a section that overrides [defaults] *and* the format
-## sections test1.com specifically is not a real story site.  Instead,
+## Each site has a section that overrides [defaults].
+## test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
@@ -508,7 +533,8 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true

 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
+## If there's ever more than 4 series, add series04,series04Url etc.
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 fandoms_label:Fandoms
 freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
@@ -524,7 +550,7 @@ bookmarks_label:Bookmarks
 include_in_freefromtags:freeformtags

 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url

 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
@@ -742,6 +768,23 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword

+[fictionpad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+extra_valid_entries:followers,comments,views,likes,dislikes
+#extra_titlepage_entries:followers,comments,views,likes,dislikes
+
+followers_label:Followers
+comments_label:Comments
+views_label:Views
+likes_label:Likes
+dislikes_label:Dislikes
+
 [finestories.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -792,6 +835,10 @@ extracategories:Highlander
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death

+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/public/style_emoticons/.*
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1099,6 +1146,7 @@ context_label:Context
 type_label:Type of Couple

 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1129,7 +1177,12 @@ extracategories:Harry Potter
 ## fictionally.org storyIds are not unique.  Combine with authorId.
 output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}

+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:

@@ -1151,12 +1204,13 @@ extracategories:My Little Pony: Friendship is Magic

 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups

 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1168,6 +1222,13 @@ short_description_label:Short Summary
 ## when a password is required rather than prompting every time.
 #fail_on_password: false

+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 [www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -270,7 +270,6 @@ def main(argv,
           elif chaptercount == 0:
               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
           else:
-               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
               if not options.metaonly:

                   # update now handled by pre-populating the old
@@ -284,6 +283,12 @@ def main(argv,
                    adapter.calibrebookmark,
                    adapter.logfile) = get_update_data(output_filename)

+                   print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+
+                   if not (options.update and chaptercount == urlchaptercount) \
+                           and adapter.getConfig("do_update_hook"):
+                       chaptercount = adapter.hookForUpdates(chaptercount)
+
                   writeStory(configuration,adapter,"epub")
                   
       else:
@@ -4,13 +4,16 @@ try:
    # just a way to switch between web service and CLI/PI
    import google.appengine.api 
 except:
-    import sys
-    if sys.version_info >= (2, 7):
-        import logging
-        logger = logging.getLogger(__name__)
-        loghandler=logging.StreamHandler()
-        loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
-        logger.addHandler(loghandler)
-        loghandler.setLevel(logging.DEBUG)
-        logger.setLevel(logging.DEBUG)
+    try: # just a way to switch between CLI and PI
+        import calibre.constants
+    except:
+        import sys
+        if sys.version_info >= (2, 7):
+            import logging
+            logger = logging.getLogger(__name__)
+            loghandler=logging.StreamHandler()
+            loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+            logger.addHandler(loghandler)
+            loghandler.setLevel(logging.DEBUG)
+            logger.setLevel(logging.DEBUG)

@@ -119,6 +119,7 @@ import adapter_nickandgregnet
 import adapter_potterheadsanonymouscom
 import adapter_simplyundeniablecom
 import adapter_scarheadnet
+import adapter_fictionpadcom

 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -210,10 +210,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        if a != None:
            warnings = a.findAll('a',{'class':"tag"})
            for warning in warnings:
-                if warning.string == "Author Chose Not To Use Archive Warnings":
-                    warning.string = "No Archive Warnings Apply"
-                if warning.string != "No Archive Warnings Apply":
-                    self.story.addToList('warnings',warning.string)
+                self.story.addToList('warnings',warning.string)
 		
        a = metasoup.find('dd',{'class':"freeform tags"})
        if a != None:
@@ -288,19 +285,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))

 		
-        try:
-            # Find Series name from series URL.
-            a = metasoup.find('dd',{'class':"series"})
-            b = a.find('a', href=re.compile(r"/series/\d+"))
-            series_name = b.string
-            series_url = 'http://'+self.host+b['href']
-            series_index = int(a.text.split(' ')[1])
-            self.setSeries(series_name, series_index)
-            self.story.setMetadata('seriesUrl',series_url)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        # Find Series name from series URL.
+        ddseries = metasoup.find('dd',{'class':"series"})
+
+        if ddseries:
+            for i, a in enumerate(ddseries.findAll('a', href=re.compile(r"/series/\d+"))):
+                series_name = stripHTML(a)
+                series_url = 'http://'+self.host+a['href']
+                series_index = int(stripHTML(a.previousSibling).replace(', ','').split(' ')[1]) # "Part # of" or ", Part #"
+                self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index))
+                self.story.setMetadata('series%02dUrl'%i,series_url)
+                if i == 0:
+                    self.setSeries(series_name, series_index)
+                    self.story.setMetadata('seriesUrl',series_url)

    # grab the text for an individual chapter.
    def getChapterText(self, url):
@@ -94,11 +94,10 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
        else:
            params['penname'] = self.getConfig("username")
            params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
        params['action'] = 'login'
        params['submit'] = 'Submit'
    
-        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                              params['penname']))
    
@@ -199,7 +199,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        for g in genrelist:
            #logger.debug("g:(%s)"%g)
            if g.strip() not in ffnetgenres:
-                logger.info("g not in ffnetgenres")
+                #logger.info("g not in ffnetgenres")
                goodgenres=False
        if goodgenres:
            self.story.extendList('genre',genrelist)
@@ -266,11 +266,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        return

    def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
                        ## more if hit too fast.  This is in
                        ## additional to what ever the
                        ## slow_down_sleep_time setting is.
+        logger.debug('Getting chapter text from: %s' % url)
        data = self._fetchUrl(url)

        if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
@@ -281,10 +281,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        # don't care about anything before "<div class='storytextp"
        # (there's a space after storytextp, so no close quote(')) and
        # this kills any body tags.
-        if "<div class='storytextp" not in data:
+        divstr = "<div role='main' class='storytextp"
+        if divstr not in data:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
        else:
-            data = data[data.index("<div class='storytextp"):]
+            data = data[data.index(divstr):]
        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
        
        soup = bs.BeautifulSoup(data)
@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import time
+import json
+
+from .. import BeautifulSoup as bs
+#from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+class FictionPadSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev','fpad')
+        self.dateformat = "%Y-%m-%dT%H:%M:%SZ"
+        self.is_adult=False
+        self.username = None
+        self.password = None
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL.
+            self._setURL("https://"+self.getSiteDomain()
+                         +"/author/"+m.group('author')
+                         +"/stories/"+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+
+    @staticmethod
+    def getSiteDomain():
+        return 'fictionpad.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "https://fictionpad.com/author/Author/stories/1234/Some-Title"
+    
+    def getSiteURLPattern(self):
+        # http://fictionpad.com/author/Serdd/stories/4275
+        return r"http(s)?://(www\.)?fictionpad\.com/author/(?P<author>[^/]+)/stories/(?P<id>\d+)"
+
+# <form method="post" action="/signin">
+#     <input name="authenticity_token" type="hidden" value="u+cfdXh46dRnwVnSlmE2B2BFmHgu760paqgBG6KQeos=" />
+#     <input type="hidden" name="remember" value="1">
+#     <strong class="help-start text-center">or with FictionPad</strong>
+#     <label class="control-label hidden-placeholder">Pseudonym or Email Address</label>
+#     <input name="login" class="input-block-level" type="text" placeholder="Pseudonym or Email Address" maxlength="50" required autofocus>
+#     <label class="control-label hidden-placeholder">Password</label>
+#     <input name="password" class="input-block-level" type="password" placeholder="Password" minlength="6" required>
+#     <button type="submit" class="btn btn-primary btn-block">Sign In</button>
+#     <p class="help-end">
+#         <a href="/passwordreset">Forgot your password?</a>
+#     </p>
+# </form>
+    def performLogin(self):
+        params = {}
+
+        if self.password:
+            params['login'] = self.username
+            params['password'] = self.password
+        else:
+            params['login'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['remember'] = '1'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/signin'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['login']))
+
+        ## need to pull empty login page first to get authenticity_token
+        soup = bs.BeautifulSoup(self._fetchUrl(loginUrl))
+        params['authenticity_token']=soup.find('input', {'name':'authenticity_token'})['value']
+        
+        data = self._postUrl(loginUrl, params)
+
+        if "Invalid email/pseudonym and password combination." in data:
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['login']))
+            raise exceptions.FailedToLogin(loginUrl,params['login'])
+            
+
+    def extractChapterUrlsAndMetadata(self):
+        # fetch the chapter.  From that we will get almost all the
+        # metadata and chapter list
+
+        url=self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+            if "This is a mature story.  Please sign in to read it." in data:
+                self.performLogin()
+                data = self._fetchUrl(url)
+            
+            find = "wordyarn.config.page = "
+            data = data[data.index(find)+len(find):]
+            data = data[:data.index("</script>")]
+            data = data[:data.rindex(";")]
+            data = data.replace('tables:','"tables":')
+            tables = json.loads(data)['tables']
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+
+        # looks like only one author per story allowed.
+        author = tables['users'][0]
+        story = tables['stories'][0]
+        story_ver = tables['story_versions'][0]
+        
+        self.story.setMetadata('authorId',author['id'])
+        self.story.setMetadata('author',author['display_name'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/author/'+author['display_name']+'/stories')
+
+        self.story.setMetadata('title',story_ver['title'])
+        self.setDescription(url,story_ver['description'])
+        
+        if not ('assets/story_versions/covers' in story_ver['profile_image_url@2x']):
+            self.setCoverImage(url,story_ver['profile_image_url@2x'])
+
+        self.story.setMetadata('datePublished',makeDate(story['published_at'], self.dateformat))
+        self.story.setMetadata('dateUpdated',makeDate(story['published_at'], self.dateformat))
+
+        self.story.setMetadata('followers',story['followers_count'])
+        self.story.setMetadata('comments',story['comments_count'])
+        self.story.setMetadata('views',story['views_count'])
+        self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
+        self.story.setMetadata('dislikes',int(story['dislikes']))
+
+        if story_ver['is_complete']:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        self.story.setMetadata('rating', story_ver['maturity_level'])
+        self.story.setMetadata('numWords', unicode(story_ver['word_count']))
+
+        for i in tables['fandoms']:
+            self.story.addToList('category',i['name'])
+        
+        for i in tables['genres']:
+            self.story.addToList('genre',i['name'])
+        
+        for i in tables['characters']:
+            self.story.addToList('characters',i['name'])
+
+        for c in tables['chapters']:
+            chtitle = "Chapter %d"%c['number']
+            if c['title']:
+                chtitle += " - %s"%c['title']
+            self.chapterUrls.append((chtitle,c['body_url']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        return self.utf8FromSoup(url,soup)
+    
+def getClass():
+    return FictionPadSiteAdapter
+
@@ -21,7 +21,6 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
-from datetime import datetime
 import json

 from .. import BeautifulSoup as bs
@@ -42,6 +41,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
        self.is_adult = False
        
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
    @staticmethod
    def getSiteDomain():
        return 'www.fimfiction.net'
@@ -93,8 +96,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
            raise exceptions.StoryDoesNotExist(self.url)

-        if "/images/missing_story.png" in data:
-            raise exceptions.StoryDoesNotExist(self.url)
+        # Can cause problems if a missing story is referenced in a comment.
+        # Shouldn't be needed anyway.
+        # if "/images/missing_story.png" in data:
+        #     raise exceptions.StoryDoesNotExist(self.url)
        
        if "This story has been marked as having adult content." in data:
            raise exceptions.AdultCheckRequired(self.url)
@@ -185,14 +190,42 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        hrstr="<hr />"
        descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
        self.setDescription(self.url,descdivstr)
-        
+
+        # Can't trust dates from API anymore I'm told.
        # Dates are in Unix time
        # Take the publish date from the first chapter posted
-        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        rawDateUpdated = storyMetadata["date_modified"]
-        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        # rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        # rawDateUpdated = storyMetadata["date_modified"]
+        # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        
+        oldestChapter = None
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on
+        # FiMFiction it's possible for authors to insert new chapters
+        # out-of-order or change the dates of earlier ones by editing
+        # them--That WILL break epub update.
+        for index, chapterDate in enumerate(soup.findAll('span', {'class':'date'})):
+            date=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",chapterDate.contents[1].strip())
+            chapterDate = makeDate(date,self.dateformat)
+            if oldestChapter == None or chapterDate < oldestChapter:
+                oldestChapter = chapterDate
+            if newestChapter == None or chapterDate > newestChapter:
+                newestChapter = chapterDate
+                self.newestChapterNum = index

+        self.story.setMetadata("dateUpdated", newestChapter)
+        
+        pubdatetag = soup.find('span', {'class':'date_approved'})
+        if pubdatetag is None:
+            self.story.setMetadata("datePublished", oldestChapter)            
+        else:
+            pubdateraw = pubdatetag('span')[1].text
+            datestripped=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",pubdateraw.strip())
+            pubDate = makeDate(datestripped,self.dateformat)
+            self.story.setMetadata("datePublished", pubDate)
+            
        chars = soup.find("div", {"class":"inner_data"})
        # fimfic stopped putting the char name on or around the char
        # icon now for some reason.  Pull it from the image name with
@@ -215,10 +248,22 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                if not isinstance(value,basestring):
                    value = unicode(value)
                self.story.setMetadata(metakey, value)
-        
+
+        rawGroupList = soup.find('ul', {'id':'story_group_list'})
+        if rawGroupList is not None:
+            for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
+                self.story.addToList("groups",stripHTML(groupName))
            
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
+        
    def getChapterText(self, url):
        logger.debug('Getting chapter text from: %s' % url)
+        
        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
        if soup == None:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
@@ -131,55 +131,55 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):

        ## Finding the metadata is a bit of a pain.  Most of the meta
        ## data is in a center.table without a bgcolor.
-        for center in soup.findAll('center'):
-            table = center.find('table',{'bgcolor':None})
-            if table:
-                metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
-                # Rating: 12+ Story Reviews: 3
-                # Chapters: 3
-                # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
-                # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
-                # Status: Completed
-                # First Published: 2010.09.02
-                # Last Published Chapter: 2010.09.28
-                # Last Updated: 2010.09.28
-                # Favorite Story Of: 1 users
-                # Warnings: Scenes of a Mild Sexual Nature
+        #for center in soup.findAll('center'):
+        table = soup.find('table',{'class':'storymaininfo'})
+        if table:
+            metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
+            # Rating: 12+ Story Reviews: 3
+            # Chapters: 3
+            # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
+            # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
+            # Status: Completed
+            # First Published: 2010.09.02
+            # Last Published Chapter: 2010.09.28
+            # Last Updated: 2010.09.28
+            # Favorite Story Of: 1 users
+            # Warnings: Scenes of a Mild Sexual Nature

-                m = re.match(r".*?Status: Completed.*?",metastr)
-                if m:
-                    self.story.setMetadata('status','Completed')
-                else:
-                    self.story.setMetadata('status','In-Progress')
+            m = re.match(r".*?Status: Completed.*?",metastr)
+            if m:
+                self.story.setMetadata('status','Completed')
+            else:
+                self.story.setMetadata('status','In-Progress')

-                m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
-                if m:
-                    self.story.setMetadata('rating', m.group(1))
+            m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
+            if m:
+                self.story.setMetadata('rating', m.group(1))

-                m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('genre',g)
-        
-                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('characters',g)
-        
-                m = re.match(r".*?Warnings: (.+).*?",metastr)
-                if m:
-                    for w in m.group(1).split(','):
-                        if w != 'Now Warnings':
-                            self.story.addToList('warnings',w)
-        
-                m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
+            m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('genre',g)
+    
+            m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('characters',g)
+    
+            m = re.match(r".*?Warnings: (.+).*?",metastr)
+            if m:
+                for w in m.group(1).split(','):
+                    if w != 'Now Warnings':
+                        self.story.addToList('warnings',w)
+    
+            m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))

-                # Updated can have more than one space after it. <shrug>
-                m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
+            # Updated can have more than one space after it. <shrug>
+            m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))

    def getChapterText(self, url):

@@ -37,11 +37,6 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)

-        # hpfanficarchive.com blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
-        
        self.decode = ["Windows-1252",
                       "utf8"] # 1252 is a superset of iso-8859-1.
                               # Most sites that claim to be
@@ -46,8 +46,8 @@ class InDeathNetAdapter(BaseSiteAdapter):
        self.password = ""
        self.is_adult=False
        
-		
-		  # get storyId from url--url validation guarantees query correct
+        
+        # get storyId from url--url validation guarantees query correct
        m = re.match(self.getSiteURLPattern(),url)
        if m:
            self.story.setMetadata('storyId',m.group('id'))
@@ -87,6 +87,34 @@ class InDeathNetAdapter(BaseSiteAdapter):
        postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat)
        return postdate

+    def getAuthorData(self):
+        
+        mainUrl = self.url.replace("/archive","")
+        
+        try:
+            maindata = self._fetchUrl(mainUrl)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        mainsoup = bs.BeautifulSoup(maindata)
+
+        # find first entry
+        e = mainsoup.find('div',{'class':"entry"})
+        
+        # get post author as author
+        d = e.find('div',{'class':"desc"})
+        a = d.find('strong')
+        self.story.setMetadata('author',a.contents[0].string.strip())
+        
+        # Don't seem to be able to get author pages anymore
+        self.story.setMetadata('authorUrl','http://www.indeath.net/')
+        self.story.setMetadata('authorId','0')
+        
    ## Getting the chapter list and the meta data, plus 'is adult' checking.
    def extractChapterUrlsAndMetadata(self):

@@ -110,17 +138,13 @@ class InDeathNetAdapter(BaseSiteAdapter):
        h = soup.find('a', id="blog_title")
        t = h.find('span')
        self.story.setMetadata('title',stripHTML(t.contents[0]).strip())
-		
+        
        s = t.find('div')        
        if s != None:
            self.setDescription(url,s)
-		
-        # Find authorid and URL from first link in Recent Entries (don't yet reference 'recent entries' - let's see if that is required)
-        a = soup.find('a', href=re.compile(r"http://www.indeath.net/user/\d+\-[a-z0-9]+/$"))		#http://www.indeath.net/user/9083-cyrex/
-        m = re.search('http://www.indeath.net/user/(?P<id>\d+)\-(?P<name>[a-z0-9]*)/$',a['href'])
-        self.story.setMetadata('authorId',m.group('id'))
-        self.story.setMetadata('authorUrl',a['href'])
-        self.story.setMetadata('author',m.group('name'))
+
+        # Get Author from main blog page since it's not reliably on the archive page
+        self.getAuthorData()

        # Find the chapters:
        chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
@@ -149,21 +173,20 @@ class InDeathNetAdapter(BaseSiteAdapter):
            if len(chapters)==1:
                self.chapterUrls.append((self.story.getMetadata('title'),chapter['href']))
            else:
-            	ct = stripHTML(chapter)
-            	tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
-            	if tnew:
-            		chaptertitle = tnew.group('newtitle')
-            	else:
-            		chaptertitle = ct
+                ct = stripHTML(chapter)
+                tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
+                if tnew:
+                    chaptertitle = tnew.group('newtitle')
+                else:
+                    chaptertitle = ct
                self.chapterUrls.append((chaptertitle,chapter['href']))
-	
-		
+
        
        
    # grab the text for an individual chapter.
    def getChapterText(self, url):
        logger.debug('Getting chapter text from: %s' % url)
-		
+        
        #chapter=bs.BeautifulSoup('<div class="story"></div>')
        data = self._fetchUrl(url)
        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center'))
@@ -37,11 +37,6 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)

-        # ncisfiction.net blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
-        
        self.decode = ["iso-8859-1",
                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
                               # Most sites that claim to be
@@ -251,8 +251,10 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX

        logger.debug('Getting chapter text from: %s' % url)
        data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        data = data.replace("HTML>","div>")
+        
+        soup = bs.BeautifulSoup(data)

        #print("soup:%s"%soup)
        tag = soup.find('td', {'class' : 'story'})
@@ -22,7 +22,6 @@ import re
 import urllib2
 import cookielib as cl
 from datetime import datetime
-import json

 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
@@ -231,10 +231,10 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
        BtVS = True
        BtVSNonX = False
        for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
-            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
+            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'Non-BTVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
                self.story.addToList('category',cat.string)
            else:
-                if 'Non-BtVS' in cat.string:
+                if 'Non-BtVS' in cat.string or 'Non-BTVS' in cat.string:
                    BtVS = False
                if 'BtVS/AtS Non-Crossover' == cat.string:
                    BtVSNonX = True
@@ -22,6 +22,7 @@ import re
 import urllib2

 from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions

 from base_adapter import BaseSiteAdapter,  makeDate
@@ -77,6 +77,8 @@ class BaseSiteAdapter(Configurable):
        self.is_adult=False
        
        self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+        ## Specific UA because too many sites are blocking the default python UA.
+        self.opener.addheaders = [('User-agent', self.getConfig('user_agent'))]
        self.storyDone = False
        self.metadataDone = False
        self.story = Story(configuration)
@@ -245,6 +247,10 @@ class BaseSiteAdapter(Configurable):
            self.metadataDone = True
        return self.story

+    def hookForUpdates(self,chaptercount):
+        "Usually not needed."
+        return chaptercount
+
    ###############################
    
    @staticmethod
@@ -377,7 +383,7 @@ fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"0
           "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
           "November":"11", "December":"12" }

-def makeDate(string,format):
+def makeDate(string,dateform):
    # Surprise!  Abstracting this turned out to be more useful than
    # just saving bytes.

@@ -386,10 +392,10 @@ def makeDate(string,format):
    # there's non-english content. -- ficbook.net now makes that a
    # lie.  It has to do something even more complicated to get
    # Russian month names correct everywhere.
-    do_abbrev = "%b" in format
+    do_abbrev = "%b" in dateform
        
-    if "%B" in format or do_abbrev:
-        format = format.replace("%B","%m").replace("%b","%m")
+    if "%B" in dateform or do_abbrev:
+        dateform = dateform.replace("%B","%m").replace("%b","%m")
        for (name,num) in fullmon.items():
            if do_abbrev:
                name = name[:3] # first three for abbrev
@@ -397,5 +403,5 @@ def makeDate(string,format):
                string = string.replace(name,num)
                break
            
-    return datetime.datetime.strptime(string,format)
+    return datetime.datetime.strptime(string,dateform)

@@ -7,6 +7,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'

+import logging
+logger = logging.getLogger(__name__)
+
 import re, os, traceback
 from zipfile import ZipFile
 from xml.dom.minidom import parseString
@@ -72,7 +75,7 @@ def get_update_data(inputio,
                # remove all .. and the path part above it, if present.
                # Mostly for epubs edited by Sigil.
                src = re.sub(r"([^/]+/\.\./)","",src)
-                print("epubutils: found pre-existing cover image:%s"%src)
+                #print("epubutils: found pre-existing cover image:%s"%src)
                oldcoverimghref = src
                oldcoverimgdata = epub.read(src)
                for item in contentdom.getElementsByTagName("item"):
@@ -81,8 +84,8 @@ def get_update_data(inputio,
                        break
                oldcover = (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
            except Exception as e:
-                print("Cover Image %s not found"%src)
-                print("Exception: %s"%(unicode(e)))
+                logger.warn("Cover Image %s not found"%src)
+                logger.warn("Exception: %s"%(unicode(e)))
                traceback.print_exc()

    filecount = 0
@@ -118,8 +121,8 @@ def get_update_data(inputio,
                                images[longdesc] = data
                                img['src'] = img['longdesc']
                            except Exception as e:
-                                print("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
-                                print("Exception: %s"%(unicode(e)))
+                                logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
+                                logger.warn("Exception: %s"%(unicode(e)))
                                traceback.print_exc()
                        soup = soup.find('body')
                        # ffdl epubs have chapter title h3
@@ -143,8 +146,8 @@ def get_update_data(inputio,
    except:
        pass
                    
-    for k in images.keys():
-        print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
+    #for k in images.keys():
+        #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)

 def get_path_part(n):
@@ -15,6 +15,9 @@
 # limitations under the License.
 #

+import logging
+logger = logging.getLogger(__name__)
+
 import re

 def _unirepl(match):
@@ -31,7 +34,7 @@ def _unirepl(match):
    except:
        # This way, at least if there's more of entities out there
        # that fail, it doesn't blow the entire download.
-        print "Numeric entity translation failed, skipping: &#x%s%s"%(match.group(1),match.group(2))
+        logger.warn("Numeric entity translation failed, skipping: &#x%s%s"%(match.group(1),match.group(2)))
        retval = ""
    return retval

@@ -69,8 +72,9 @@ def removeEntities(text):
    
    if text is None:
        return ""
-    if not (isinstance(text,str) or isinstance(text,unicode)):
-        return str(text)
+    
+    if not isinstance(text,basestring):
+        return unicode(text)
    
    try:
        t = text.decode('utf-8')
@@ -349,6 +349,8 @@ class Story(Configurable):
                return removeAllEntities(value)
            else:
                return value
+        else: #if self.getConfig("default_value_"+key):
+            return self.getConfig("default_value_"+key)
        
    def getAllMetadata(self,
                       removeallentities=False,
@@ -485,6 +487,8 @@ class Story(Configurable):

        if None in subjectset:
            subjectset.remove(None)
+        if '' in subjectset:
+            subjectset.remove('')

        return list(subjectset | set(self.getConfigList("extratags")))
            
@@ -627,7 +631,7 @@ class Story(Configurable):
                    ext)
                self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                
-            logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
+            #logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
        else:
            newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc']
            
@@ -221,7 +221,7 @@ class BaseStoryWriter(Configurable):
                    lastupdated=self.story.getMetadataRaw('dateUpdated').date()
                    fileupdated=datetime.datetime.fromtimestamp(os.stat(outfilename)[8]).date()
                    if fileupdated > lastupdated:
-                        print "File(%s) Updated(%s) more recently than Story(%s) - Skipping" % (outfilename,fileupdated,lastupdated)
+                        logger.warn("File(%s) Updated(%s) more recently than Story(%s) - Skipping" % (outfilename,fileupdated,lastupdated))
                        return
            if not metaonly:        
                self.story = self.adapter.getStory() # get full story
@@ -57,9 +57,17 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>Minor optimizations.</li>
+              <li>Fixes for dark-solace.org.</li>
 	    </ul>
 	  </p>
+          
+          <h3>fanfiction.net</h3>
+          <p>
+            Fanfiction.net appears to be blocking access from Google
+            App Engine, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
 	  
 	  <p>
 	    Questions?  Check out our 
@@ -69,7 +77,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-67.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-77.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -16,7 +16,22 @@
 [defaults]

 ## [defaults] section applies to all formats and sites but may be
-## overridden at several levels
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category

 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
@@ -127,6 +142,14 @@ extratags: FanFiction
 ## Add this to genre if there's more than one category.
 #add_genre_when_multi_category: Crossover

+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -249,6 +272,8 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true

+## User-agent
+user_agent:FFDL/1.7

 ## Each output format has a section that overrides [defaults]
 [html]
@@ -419,8 +444,8 @@ nook_img_fix:true
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true

-## Each site has a section that overrides [defaults] *and* the format
-## sections test1.com specifically is not a real story site.  Instead,
+## Each site has a section that overrides [defaults].
+## test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
@@ -477,7 +502,8 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true

 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
+## If there's ever more than 4 series, add series04,series04Url etc.
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 fandoms_label:Fandoms
 freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
@@ -493,7 +519,7 @@ bookmarks_label:Bookmarks
 include_in_freefromtags:freeformtags

 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url

 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
@@ -727,6 +753,23 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword

+[fictionpad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+extra_valid_entries:followers,comments,views,likes,dislikes
+#extra_titlepage_entries:followers,comments,views,likes,dislikes
+
+followers_label:Followers
+comments_label:Comments
+views_label:Views
+likes_label:Likes
+dislikes_label:Dislikes
+
 [finestories.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -777,6 +820,10 @@ extracategories:Highlander
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death

+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/public/style_emoticons/.*
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1084,6 +1131,7 @@ context_label:Context
 type_label:Type of Couple

 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1111,7 +1159,12 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:

@@ -1133,12 +1186,13 @@ extracategories:My Little Pony: Friendship is Magic

 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups

 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1150,6 +1204,13 @@ short_description_label:Short Summary
 ## when a password is required rather than prompting every time.
 #fail_on_password: false

+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 [www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -2,6 +2,24 @@
 ## like.  Uncomment options by removing the '#' in front of them.

 [defaults]
+## [defaults] section applies to all formats and sites but may be
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category
+
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
 #is_adult:true
@@ -51,7 +51,14 @@
 	      by {{ fic.author }} ({{ fic.format }})
 	  {% endif %}
 	  {% if fic.failure %}
-	      <span id='error'>{{ fic.failure }}</span>
+          <h3>fanfiction.net</h3>
+          <p>
+            FYI, fanfiction.net appears to be blocking access from Google
+            App Engine, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
+	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}
 	  {% if  not fic.completed and not fic.failure %}
 	      <p>Not done yet.  This page will periodically poll to see if your story has finished.</p>
Author	SHA1	Message	Date
Jim Miller	b6dd579c93	Bump versions.	2013-10-19 11:24:18 -05:00
Jim Miller	aa685a4c7d	Reorg Basic config a little and wrap the tab widget in a scrollarea.	2013-10-18 15:01:26 -05:00
Jim Miller	b0248daf07	dark-solace.org is now insisting on www.dark-solace.org, for login at least.	2013-10-17 19:26:31 -05:00
Jim Miller	be5fe49ab8	Added tag FanFictionDownLoader-4.4.77 for changeset e85271dd28c9	2013-10-13 14:33:42 -05:00
Jim Miller	f42f440f1b	Added tag calibre-plugin-1.7.48 for changeset e85271dd28c9	2013-10-13 14:33:33 -05:00
Jim Miller	cbf50a36ee	Trust ffnet to change their html right after I released a new version.	2013-10-13 14:33:26 -05:00
Jim Miller	fda0fda84e	Added tag FanFictionDownLoader-4.4.76 for changeset c60e9ebb849d	2013-10-13 11:00:00 -05:00
Jim Miller	c6f5c524be	Added tag calibre-plugin-1.7.47 for changeset c60e9ebb849d	2013-10-13 10:59:48 -05:00
Jim Miller	00a46a7cc0	Bump versions, make "Mark added/updated books when finished" default False.	2013-10-13 10:59:36 -05:00
Jim Miller	5c1ca5a188	Mark failed updates even when all failed.	2013-10-12 11:16:31 -05:00
Jim Miller	17c6dddfac	Add user/pass support for 18+ rated stories on fictionpad.com	2013-10-11 18:14:35 -05:00
Jim Miller	33451f1119	Make marking updated books optional since it's now visible.	2013-10-11 15:04:45 -05:00
Jim Miller	192ade1fca	Adding fictionpad.com. Doesn't support login and 18+ yet.	2013-10-11 15:04:21 -05:00
Jim Miller	aa286a9d0d	Change prints to logger calls and clean up some of them.	2013-10-10 21:34:46 -05:00
Jim Miller	a1c19ac12e	Added tag FanFictionDownLoader-4.4.75 for changeset 3779dfd8a385	2013-09-29 14:22:03 -05:00
Jim Miller	85b6e305be	Added tag calibre-plugin-1.7.46 for changeset 3779dfd8a385	2013-09-29 14:21:46 -05:00
Jim Miller	40fb061a86	Bump versions.	2013-09-29 14:21:33 -05:00
Jim Miller	7fdc59691f	do_update_hook for fimf, find newest chapter and update from there.	2013-09-29 14:19:18 -05:00
Jim Miller	78845d0d1e	TtH uses both Non-BtVS/... and Non-BTVS/...	2013-09-29 14:17:41 -05:00
Jim Miller	786b1d5cdf	Exclude indeath.net emoticons from being covers.	2013-09-26 21:45:32 -05:00
besnef	b5d176f007	Merge fix for inDeath.net	2013-09-26 22:20:19 -04:00
besnef	1c71dedfa3	Fix for inDeath.net	2013-09-21 14:57:45 -04:00
Jim Miller	2484f0f5c7	Added tag FanFictionDownLoader-4.4.74 for changeset 53f662218344	2013-09-21 13:24:25 -05:00
Jim Miller	a087927929	Added tag calibre-plugin-1.7.45 for changeset 53f662218344	2013-09-21 13:24:13 -05:00
Jim Miller	e64d49e3e6	Bump versions.	2013-09-21 13:23:58 -05:00
Jim Miller	d9ad95467b	Set custom column only if there's a value (mostly for int/float columns).	2013-09-21 12:53:28 -05:00
Jim Miller	668e0e08b5	Add ffnet warning to status page on error.	2013-09-16 19:06:44 -05:00
Jim Miller	2b0b4ca2af	Don't include empty string tags.	2013-09-16 18:55:02 -05:00
Jim Miller	dca3707eac	Fix for whofic.com.	2013-09-16 13:10:09 -05:00
Jim Miller	a3b23857a3	Added tag FanFictionDownLoader-4.4.73 for changeset 07134453ebb0	2013-09-14 12:25:49 -05:00
Jim Miller	9cdccb576c	Added tag calibre-plugin-1.7.44 for changeset 07134453ebb0	2013-09-14 12:25:28 -05:00
Jim Miller	cf24021f14	Bump versions.	2013-09-14 12:25:17 -05:00
Jim Miller	0bbe6287b6	Really commit changes to portkey.org.	2013-09-12 22:57:04 -05:00
Jim Miller	0c585b6e52	Remove call do dateutil.parser in fimf. Requires dateutil package for CLI.	2013-09-12 22:41:09 -05:00
Jim Miller	aacf6dc6a2	Switch portkey.org to non-stone soup and brute-force remove <HTML> tags.	2013-09-12 22:40:41 -05:00
Jim Miller	9f56952950	Fix for empty word num into int column.	2013-09-12 19:11:30 -05:00
Jim Miller	c49cad8889	Added tag FanFictionDownLoader-4.4.72 for changeset 75db10f77cb4	2013-09-11 10:34:07 -05:00
Jim Miller	bf1d8e18bf	Added tag calibre-plugin-1.7.43 for changeset 75db10f77cb4	2013-09-11 10:33:55 -05:00
Jim Miller	e41c7dabeb	Fix fimfic dates, add fimfic site specific 'groups' metadata. Bump versions.	2013-09-11 10:32:57 -05:00
Jim Miller	face0af074	Add default_value_status (etc) feature.	2013-09-10 10:24:45 -05:00
Jim Miller	02f38eaa2e	Added tag FanFictionDownLoader-4.4.71 for changeset b4e7c2ec827d	2013-09-09 09:11:47 -05:00
Jim Miller	d4b5438bc5	Added tag calibre-plugin-1.7.42 for changeset b4e7c2ec827d	2013-09-09 09:11:29 -05:00
Jim Miller	83db111ba7	Changes for ffnet--longer sleep, config UA, bump versions.	2013-09-09 09:10:56 -05:00
Jim Miller	2eb8c27c80	fanfiction.net blocked by IP - index.html	2013-09-08 14:55:52 -05:00
Jim Miller	a79066750c	Changes on index.html.	2013-09-08 14:48:05 -05:00
Jim Miller	3857a6a89b	Added tag FanFictionDownLoader-4.4.70 for changeset 9d99f91e062e	2013-09-08 14:33:14 -05:00
Jim Miller	016d4af149	Added tag calibre-plugin-1.7.41 for changeset 9d99f91e062e	2013-09-08 14:33:00 -05:00
Jim Miller	a4e499866e	Fix for AO3 stories without series, bump versions.	2013-09-08 14:32:41 -05:00
Jim Miller	5736f4f181	Added tag calibre-plugin-1.7.40 for changeset db9a23c65412	2013-09-07 14:43:55 -05:00
Jim Miller	e04453d73f	Removed tag calibre-plugin-1.7.340	2013-09-07 14:43:47 -05:00
Jim Miller	c3778502dc	Added tag FanFictionDownLoader-4.4.69 for changeset db9a23c65412	2013-09-07 14:42:23 -05:00
Jim Miller	925b12e776	Added tag calibre-plugin-1.7.340 for changeset db9a23c65412	2013-09-07 14:41:47 -05:00
Jim Miller	62ed503d10	Bump versions.	2013-09-07 14:41:30 -05:00
Jim Miller	7e35d0b710	Add User-agent="FFDL/1.7" for all adapters. (Remove from specific adapters.)	2013-09-07 14:36:48 -05:00
Jim Miller	20c7e7f075	Fixes for changes to harrypotterfanfictioncom.	2013-09-07 14:36:13 -05:00
Jim Miller	9f61cff7cd	Additional series as site specific data for AO3.	2013-09-07 14:35:32 -05:00
Jim Miller	cb5587b0c7	Fix for calibre pre-1.0.0.	2013-09-07 14:32:48 -05:00
Jim Miller	f2a74c9ffc	Better doc section override order in ini files.	2013-09-04 14:33:29 -05:00
Jim Miller	eb0f013550	Added tag calibre-plugin-1.7.39 for changeset 1af0c2a05450	2013-08-31 19:57:27 -05:00