Workaround for poor HTML in summary(yourfanfictioncom), minor tweaks.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-21
+version: 4-4-22
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 6, 0)
+    version             = (1, 6, 4)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -8,6 +8,7 @@ __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import traceback, copy
+from collections import OrderedDict
 
 from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont, QWidget,
                       QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea)
@@ -62,6 +63,9 @@ default_prefs['countpagesstats'] = []
 
 default_prefs['errorcol'] = ''
 default_prefs['custom_cols'] = {}
+default_prefs['custom_cols_newonly'] = {}
+
+default_prefs['std_cols_newonly'] = {}
 
 def set_library_config(library_config):
     get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE,
@@ -75,14 +79,14 @@ def get_library_config():
     # Check whether this is a configuration needing to be migrated
     # from json into database.  If so: get it, set it, rename it in json.
     if library_id in old_prefs:
-        print("get prefs from old_prefs")
+        #print("get prefs from old_prefs")
         library_config = old_prefs[library_id]
         set_library_config(library_config)
         old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
         del old_prefs[library_id]
 
     if library_config is None:
-        print("get prefs from db")
+        #print("get prefs from db")
         library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
                                                  copy.deepcopy(default_prefs))
     return library_config
@@ -106,7 +110,7 @@ class PrefsFacade():
     def _get_prefs(self):
         libraryid = get_library_uuid(get_gui().current_db)
         if self.current_prefs == None or self.libraryid != libraryid:
-            print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
+            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
             self.libraryid = libraryid
             self.current_prefs = get_library_config()
         return self.current_prefs
@@ -172,8 +176,11 @@ class ConfigWidget(QWidget):
         if 'Count Pages' not in plugin_action.gui.iactions:
             self.countpages_tab.setEnabled(False)
 
-        self.columns_tab = CustomColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.columns_tab, 'Custom Columns')
+        self.std_columns_tab = StandardColumnsTab(self, plugin_action)
+        tab_widget.addTab(self.std_columns_tab, 'Standard Columns')
+
+        self.cust_columns_tab = CustomColumnsTab(self, plugin_action)
+        tab_widget.addTab(self.cust_columns_tab, 'Custom Columns')
 
         self.other_tab = OtherTab(self, plugin_action)
         tab_widget.addTab(self.other_tab, 'Other')
@@ -241,19 +248,30 @@ class ConfigWidget(QWidget):
             
         prefs['countpagesstats'] = countpagesstats
         
+        # Standard Columns tab
+        colsnewonly = {}
+        for (col,checkbox) in self.std_columns_tab.stdcol_newonlycheck.iteritems():
+            colsnewonly[col] = checkbox.isChecked()
+        prefs['std_cols_newonly'] = colsnewonly
+
         # Custom Columns tab
         # error column
-        prefs['errorcol'] = unicode(self.columns_tab.errorcol.itemData(self.columns_tab.errorcol.currentIndex()).toString())
+        prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())
 
         # cust cols
         colsmap = {}
-        for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems():
+        for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
             val = unicode(combo.itemData(combo.currentIndex()).toString())
             if val != 'none':
                 colsmap[col] = val
                 #print("colsmap[%s]:%s"%(col,colsmap[col]))
         prefs['custom_cols'] = colsmap
 
+        colsnewonly = {}
+        for (col,checkbox) in self.cust_columns_tab.custcol_newonlycheck.iteritems():
+            colsnewonly[col] = checkbox.isChecked()
+        prefs['custom_cols_newonly'] = colsnewonly
+        
         prefs.save_to_db()
 
     def edit_shortcuts(self):
@@ -313,7 +331,7 @@ class BasicTab(QWidget):
         self.l.addLayout(horz)
 
         self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off.")
+        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books.")
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
@@ -322,16 +340,25 @@ class BasicTab(QWidget):
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         self.l.addWidget(self.updateepubcover)
 
+        self.l.addSpacing(10)        
+
+        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
+        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
+        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
+        self.l.addWidget(self.deleteotherforms)
+        
         self.updatecover = QCheckBox('Update Calibre Cover when Updating Metadata?',self)
         self.updatecover.setToolTip("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
         self.updatecover.setChecked(prefs['updatecover'])
         self.l.addWidget(self.updatecover)
 
         self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
-        self.keeptags.setToolTip('Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.')
+        self.keeptags.setToolTip("Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")
         self.keeptags.setChecked(prefs['keeptags'])
         self.l.addWidget(self.keeptags)
 
+        self.l.addSpacing(10)        
+
         self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
         self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
         self.urlsfromclip.setChecked(prefs['urlsfromclip'])
@@ -343,16 +370,13 @@ class BasicTab(QWidget):
         self.updatedefault.setChecked(prefs['updatedefault'])
         self.l.addWidget(self.updatedefault)
 
-        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
-        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
-        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
-        self.l.addWidget(self.deleteotherforms)
-        
         self.adddialogstaysontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
         self.adddialogstaysontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
         self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
         self.l.addWidget(self.adddialogstaysontop)
 
+        self.l.addSpacing(10)        
+
         # this is a cheat to make it easier for users to realize there's a new include_images features.
         self.includeimages = QCheckBox("Include images in EPUBs?",self)
         self.includeimages.setToolTip("Download and include images in EPUB stories.  This is equivalent to adding:\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n ...to the top of personal.ini.  Your settings in personal.ini will override this.")
@@ -604,27 +628,27 @@ class CountPagesTab(QWidget):
         self.l.addSpacing(5)
         
         # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
-        self.pagecount = QCheckBox('PageCount',self)
+        self.pagecount = QCheckBox('Page Count',self)
         self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
         self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
         self.l.addWidget(self.pagecount)
             
-        self.wordcount = QCheckBox('WordCount',self)
+        self.wordcount = QCheckBox('Word Count',self)
         self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
         self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
         self.l.addWidget(self.wordcount)
 
-        self.fleschreading = QCheckBox('FleschReading',self)
+        self.fleschreading = QCheckBox('Flesch Reading Ease',self)
         self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
         self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
         self.l.addWidget(self.fleschreading)
         
-        self.fleschgrade = QCheckBox('Fleschgrade',self)
+        self.fleschgrade = QCheckBox('Flesch-Kincaid Grade Level',self)
         self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
-        self.fleschgrade.setChecked('Fleschgrade' in prefs['countpagesstats'])
+        self.fleschgrade.setChecked('FleschGrade' in prefs['countpagesstats'])
         self.l.addWidget(self.fleschgrade)
         
-        self.gunningfog = QCheckBox('GunningFog',self)
+        self.gunningfog = QCheckBox('Gunning Fog Index',self)
         self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
         self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
         self.l.addWidget(self.gunningfog)
@@ -767,6 +791,7 @@ class CustomColumnsTab(QWidget):
         self.l.addSpacing(5)
         
         self.custcol_dropdowns = {}
+        self.custcol_newonlycheck = {}
 
         for key, column in custom_columns.iteritems():
 
@@ -775,8 +800,8 @@ class CustomColumnsTab(QWidget):
                 # for (k,v) in column.iteritems():
                 #     print("column['%s'] => %s"%(k,v))
                 horz = QHBoxLayout()
-                label = QLabel('%s(%s)'%(column['name'],key))
-                label.setToolTip("Update this %s column with..."%column['datatype'])
+                label = QLabel(column['name'])
+                label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
                 horz.addWidget(label)
                 dropdown = QComboBox(self)
                 dropdown.addItem('',QVariant('none'))
@@ -789,8 +814,15 @@ class CustomColumnsTab(QWidget):
                     dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
                 else:
                     dropdown.setToolTip("Metadata values valid for this type of column.")
-
                 horz.addWidget(dropdown)
+
+                newonlycheck = QCheckBox("New Only",self)
+                newonlycheck.setToolTip("Write to %s(%s) only for new\nbooks, not updates to existing books."%(column['name'],key))
+                self.custcol_newonlycheck[key] = newonlycheck
+                if key in prefs['custom_cols_newonly']:
+                    newonlycheck.setChecked(prefs['custom_cols_newonly'][key])
+                horz.addWidget(newonlycheck)
+                
                 self.l.addLayout(horz)
         
         self.l.insertStretch(-1)
@@ -818,3 +850,51 @@ class CustomColumnsTab(QWidget):
         
         
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
+
+
+class StandardColumnsTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+
+        columns=OrderedDict()
+        
+        columns["title"]="Title"
+        columns["authors"]="Author(s)"
+        columns["publisher"]="Publisher"
+        columns["tags"]="Tags"
+        columns["languages"]="Languages"
+        columns["pubdate"]="Published Date"
+        columns["timestamp"]="Date"
+        columns["comments"]="Comments"
+        columns["series"]="Series"
+        columns["identifiers"]="Ids(url id only)"
+
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel("The standard calibre metadata columns are listed below.  You may choose whether FFDL will fill each column automatically on updates or only for new books.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.stdcol_newonlycheck = {}
+
+        for key, column in columns.iteritems():
+            horz = QHBoxLayout()
+            label = QLabel(column)
+            #label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
+            horz.addWidget(label)
+
+            newonlycheck = QCheckBox("New Only",self)
+            newonlycheck.setToolTip("Write to %s only for new\nbooks, not updates to existing books."%column)
+            self.stdcol_newonlycheck[key] = newonlycheck
+            if key in prefs['std_cols_newonly']:
+                newonlycheck.setChecked(prefs['std_cols_newonly'][key])
+            horz.addWidget(newonlycheck)
+            
+            self.l.addLayout(horz)
+        
+        self.l.insertStretch(-1)

calibre-plugin/dialogs.py

@@ -106,26 +106,21 @@ class AddNewDialog(SizePersistedDialog):
 
         horz = QHBoxLayout()
         label = QLabel('If Story Already Exists?')
-        label.setToolTip("What to do if there's already an existing story with the same title and author.")
         horz.addWidget(label)
         self.collision = QComboBox(self)
+        self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
         # add collision options
         self.set_collisions()
         i = self.collision.findText(prefs['collision'])
         if i > -1:
             self.collision.setCurrentIndex(i)
-        # self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+
-        #                           UPDATE+' will download new chapters only and add to existing EPUB.\n'+
-        #                           ADDNEW+' will create a new story with the same title and author.\n'+
-        #                           SKIP+' will not download existing stories.\n'+
-        #                           CALIBREONLY+' will not download stories, but will update Calibre metadata.')
         label.setBuddy(self.collision)
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
 
         horz = QHBoxLayout()
         self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?')
+        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
         self.updatemeta.setChecked(prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
 
@@ -177,15 +172,17 @@ class FakeLineEdit():
     def text(self):
         pass
     
-class CollectURLDialog(QDialog):
+class CollectURLDialog(SizePersistedDialog):
     '''
     Collect single url for get urls.
     '''
-    def __init__(self, gui, title):
-        QDialog.__init__(self, gui)
+    def __init__(self, gui, title, url_text): 
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
         self.gui = gui
         self.status=False
 
+        self.setMinimumWidth(300)
+        
         self.l = QGridLayout()
         self.setLayout(self.l)
 
@@ -194,6 +191,7 @@ class CollectURLDialog(QDialog):
         
         self.l.addWidget(QLabel("URL:"),1,0)
         self.url = QLineEdit(self)
+        self.url.setText(url_text)
         self.l.addWidget(self.url,1,1)
    
         self.ok_button = QPushButton('OK', self)
@@ -204,15 +202,16 @@ class CollectURLDialog(QDialog):
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,2,1)
 
-        self.resize(self.sizeHint())
+        # restore saved size.
+        self.resize_dialog()
 
     def ok(self):
         self.status=True
-        self.hide()
+        self.accept()
 
     def cancel(self):
         self.status=False
-        self.hide()
+        self.reject()
 
 class UserPassDialog(QDialog):
     '''
@@ -431,9 +430,9 @@ class UpdateExistingDialog(SizePersistedDialog):
         options_layout.addWidget(self.fileform)
         
         label = QLabel('Update Mode:')
-        label.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
         options_layout.addWidget(label)
         self.collision = QComboBox(self)
+        self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
         # add collision options
         self.set_collisions()
         i = self.collision.findText(prefs['collision'])
@@ -444,7 +443,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         options_layout.addWidget(self.collision)
 
         self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?  May set default from plugin configuration.')
+        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
         self.updatemeta.setChecked(prefs['updatemeta'])
         options_layout.addWidget(self.updatemeta)
                 

calibre-plugin/ffdl_plugin.py

@@ -116,9 +116,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # items to prevent GC removing it.
         self.menu_actions = []
         self.qaction.setMenu(self.menu)
-        self.menu.aboutToShow.connect(self.about_to_show_menu)
-
         self.menus_lock = threading.RLock()
+        self.menu.aboutToShow.connect(self.about_to_show_menu)
 
     def initialization_complete(self):
         # otherwise configured hot keys won't work until the menu's
@@ -134,10 +133,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
     def rebuild_menus(self):
         with self.menus_lock:
-            # Show the config dialog
-            # The config dialog can also be shown from within
-            # Preferences->Plugins, which is why the do_user_config
-            # method is defined on the base plugin class
             do_user_config = self.interface_action_base_plugin.do_user_config
             self.menu.clear()
             self.actions_unique_map = {}
@@ -179,15 +174,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                                       shortcut_name=rmmenutxt,
                                                                       triggered=partial(self.update_lists,add=False))
                 
-                # try:
-                #     self.add_send_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
-                # except:
-                #     pass
-                # try:
-                #     self.add_remove_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
-                # except:
-                #     pass
-
             self.menu.addSeparator()
             self.get_list_action = self.create_menu_item_ex(self.menu, 'Get URLs from Selected Books', image='bookmarks.png',
                                                             unique_name='Get URLs from Selected Books',
@@ -203,13 +189,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False,
                                                            image= 'config.png',
                                                            unique_name='Configure FanFictionDownLoader',
-                                                           shortcut_name='Configure FanFictionDownLoader',
                                                            triggered=partial(do_user_config,parent=self.gui))
             
-            self.about_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False,
+            self.about_action = create_menu_action_unique(self, self.menu, 'About Plugin', shortcut=False,
                                                           image= 'images/icon.png',
                                                           unique_name='About FanFictionDownLoader',
-                                                          shortcut_name='About FanFictionDownLoader',
                                                           triggered=self.about)
             
             # Before we finalize, make sure we delete any actions for menus that are no longer displayed
@@ -254,13 +238,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)
 
     def get_urls_from_page(self):
-        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page")
+
+        if prefs['urlsfromclip']:
+            try:
+                urltxt = self.get_urls_clip(storyurls=False)[0]
+            except:
+                urltxt = ""
+        
+        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page",urltxt)
         d.exec_()
         if not d.status:
             return
-        print("URL:%s"%d.url.text())
+        print("get_urls_from_page URL:%s"%d.url.text())
 
-        url_list = get_urls_from_page("%s"%d.url.text())
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        ffdlconfig.readfp(StringIO(prefs['personal.ini']))
+        url_list = get_urls_from_page("%s"%d.url.text(),ffdlconfig)
 
         if url_list:
             d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
@@ -377,13 +371,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             options['version'] = self.version
             print(self.version)
             self.start_downloads( options, update_books )
-        
-    def get_urls_clip(self):
+            
+    def get_urls_clip(self,storyurls=True):
         url_list = []
         if prefs['urlsfromclip']:
             for url in unicode(QApplication.instance().clipboard().text()).split():
-                if( self._is_good_downloader_url(url) ):
+                if not storyurls or self._is_good_downloader_url(url):
                     url_list.append(url)
+                    
         return url_list
 
     def apply_settings(self):
@@ -551,7 +546,7 @@ make_firstimage_cover:true
                 raise NotGoingToDownload("Skipping duplicate story.","list_remove.png")
 
             if len(identicalbooks) > 1:
-                raise NotGoingToDownload("More than one identical book--can't tell which to update/overwrite.","minusminus.png")
+                raise NotGoingToDownload("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite.","minusminus.png")
 
             ## changed: add new book when CALIBREONLY if none found.
             if collision == CALIBREONLY and not identicalbooks:
@@ -708,7 +703,7 @@ make_firstimage_cover:true
             self._add_or_update_book(book,options,prefs,mi)
 
         if options['collision'] == CALIBREONLY or \
-                (options['updatemeta'] and book['good']):
+                ( (options['updatemeta'] or book['added']) and book['good'] ):
             self._update_metadata(db, book['calibre_id'], book, mi, options)
 
     def _update_bad_book(self,book,db=None,label='errorcol',
@@ -770,22 +765,6 @@ make_firstimage_cover:true
 
         self.previous = self.gui.library_view.currentIndex()
         db = self.gui.current_db
-        # if display_story_list(self.gui,
-        #                       'Downloads finished, confirm to update Calibre',
-        #                       prefs,
-        #                       self.qaction.icon(),
-        #                       job.result,
-        #                       label_text='Stories will not be added or updated in Calibre without confirmation.',
-        #                       offer_skip=True):
-
-        # payload = (job.statistics_cols_map, book_statistics_map)
-        # all_ids = set(book_statistics_map.keys())
-        # msg = '<p>Count Pages plugin found <b>%d statistics(s)</b>. ' % len(all_ids) + \
-        #       'Proceed with updating columns in your library?'
-        # self.gui.proceed_question(self._update_database_columns,
-        #         payload, job.details,
-        #         'Count log', 'Count complete', msg,
-        #         show_copy_button=False)
 
         book_list = job.result
         good_list = filter(lambda x : x['good'], book_list)
@@ -798,20 +777,20 @@ make_firstimage_cover:true
 <p>Proceed with updating your library?</p>
 '''%(len(good_list),len(bad_list))
 
-        htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>URL</th><th>Comment</th></tr>'
+        htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
         for book in good_list:
             if 'status' in book:
                 status = book['status']
             else:
                 status = 'Good'
-            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['url'],book['comment']]) + '</td></tr>'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
         
         for book in bad_list:
             if 'status' in book:
                 status = book['status']
             else:
                 status = 'Bad'
-            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['url'],book['comment']]) + '</td></tr>'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
         
         htmllog = htmllog + '</table></body></html>'
         
@@ -890,6 +869,7 @@ make_firstimage_cover:true
         return book_id
 
     def _update_metadata(self, db, book_id, book, mi, options):
+        oldmi = db.get_metadata(book_id,index_is_id=True)
         if prefs['keeptags']:
             old_tags = db.get_tags(book_id)
             # remove old Completed/In-Progress only if there's a new one.
@@ -905,7 +885,6 @@ make_firstimage_cover:true
             mi.languages=[book['all_metadata']['langcode']]
         else:
             # Set language english, but only if not already set.
-            oldmi = db.get_metadata(book_id,index_is_id=True)
             if not oldmi.languages:
                 mi.languages=['eng']
 
@@ -922,6 +901,32 @@ make_firstimage_cover:true
                 autid=db.get_author_id(auth)
                 db.set_link_field_for_author(autid, unicode(authurls[i]),
                                              commit=False, notify=False)
+
+        # mi.title = oldmi.title
+        # mi.authors = oldmi.authors
+        # mi.publisher = oldmi.publisher
+        # mi.tags = oldmi.tags
+        # mi.languages = oldmi.languages
+        # mi.pubdate = oldmi.pubdate
+        # mi.timestamp = oldmi.timestamp
+        # mi.comments = oldmi.comments
+        # mi.series = oldmi.series
+                
+        # mi.set_identifiers(oldmi.get_identifiers())
+
+        # implement 'newonly' flags here by setting to the current
+        # value again.
+        if not book['added']:
+            for (col,newonly) in prefs['std_cols_newonly'].iteritems():
+                if newonly:
+                    if col == "identifiers":
+                        mi.set_identifiers(oldmi.get_identifiers())
+                    else:
+                        try:
+                            mi.__setattr__(col,oldmi.__getattribute__(col))
+                        except AttributeError:
+                            print("AttributeError? %s"%col)
+                            pass
             
         db.set_metadata(book_id,mi)
 
@@ -936,6 +941,9 @@ make_firstimage_cover:true
                 print("%s not an existing column, skipping."%col)
                 continue
             coldef = custom_columns[col]
+            if col in prefs['custom_cols_newonly'] and prefs['custom_cols_newonly'][col] and not book['added']:
+                print("Skipping custom column(%s) update, set to New Books Only"%coldef['name'])
+                continue
             if not meta.startswith('status-') and meta not in book['all_metadata'] or \
                     meta.startswith('status-') and 'status' not in book['all_metadata']:
                 print("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))

calibre-plugin/jobs.py

@@ -154,11 +154,13 @@ def do_download_for_worker(book,options):
             # update now handled by pre-populating the old images and
             # chapters in the adapter rather than merging epubs.
             urlchaptercount = int(story.getMetadata('numChapters'))
-            (url,chaptercount,
+            (url,
+             chaptercount,
              adapter.oldchapters,
              adapter.oldimgs,
              adapter.oldcover,
-             adapter.calibrebookmark) = get_update_data(book['epub_for_update'])
+             adapter.calibrebookmark,
+             adapter.logfile) = get_update_data(book['epub_for_update'])
 
             print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
             print("write to %s"%outfile)

defaults.ini

@@ -197,9 +197,24 @@ windows_eol: true
 zip_output: false
 
 ## epub carries the TOC in metadata.
-## mobi generated from epub will have a TOC at the end.
+## mobi generated from epub by calibre will have a TOC at the end.
 include_tocpage: false
 
+## include a Update Log page before the story text.  If included, the
+## log will be updated each time the epub is an all the metadata
+## fields that have changed since the last update (typically
+## dateUpdated,numChapters,numWords at a minimum) will be shown.
+## Great for tracking when chapters came out and when the description,
+## etc changed.
+include_logpage: false
+
+## items to include in the log page Empty metadata entries, or those
+## that haven't changed since the last update, will *not* appear, even
+## if in the list.  You can include extra text or HTML that will be
+## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
+## summary,<br />,...
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
+
 ## epub->mobi conversions typically don't like tables.
 titlepage_use_table: false
 

downloader.py

@@ -128,7 +128,7 @@ def main():
            config.set("overrides",var,val)
 
    if options.list:
-       retlist = get_urls_from_page(args[0])
+       retlist = get_urls_from_page(args[0], config)
        print "\n".join(retlist)
                
        return
@@ -193,11 +193,13 @@ def main():
                    # update now handled by pre-populating the old
                    # images and chapters in the adapter rather than
                    # merging epubs.
-                   (url,chaptercount,
+                   (url,
+                    chaptercount,
                     adapter.oldchapters,
                     adapter.oldimgs,
                     adapter.oldcover,
-                    adapter.calibrebookmark) = get_update_data(args[0])
+                    adapter.calibrebookmark,
+                    adapter.logfile) = get_update_data(args[0])
 
                    writeStory(config,adapter,"epub")
                    

fanficdownloader/adapters/__init__.py

@@ -127,7 +127,7 @@ def getAdapter(config,url,fileform=None):
     if( domain != parsedUrl.netloc ):
         fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
 
-    logging.debug("site:"+domain)
+    logging.debug("trying url:"+url)
     cls = getClassFor(domain)
     if not cls and domain.startswith("www."):
         domain = domain.replace("www.","")

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -50,8 +50,17 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
         logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ao3')
@@ -70,10 +79,11 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         return ['www.archiveofourown.org','archiveofourown.org']
 
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456"
+        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456"
 
     def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+        # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
+        return re.escape("http://")+"(www.)?"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)(/chapters/\d+)?/?$"
         
     ## Login
     def needToLoginCheck(self, data):

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -176,7 +176,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         genrelist = metalist[0].split('/') # Hurt/Comfort already changed above.
         goodgenres=True
         for g in genrelist:
-            if g not in ffnetgenres:
+            print("g:(%s)"%g)
+            if g.strip() not in ffnetgenres:
+                print("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
@@ -239,6 +241,15 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.
         data = self._fetchUrl(url)
+
+        # some ancient stories have body tags inside them that cause
+        # soup parsing to discard the content.  For story text we
+        # don't care about anything before "<div class='storytextp"
+        # (there's a space after storytextp, so no close quote(')) and
+        # this kills any body tags.
+        data = data[data.index("<div class='storytextp"):]
+        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
+        
         soup = bs.BeautifulSoup(data)
 
         ## Remove the 'share' button.

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -122,7 +122,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
 		
 
         # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d$'))
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
         if len(chapters) == 0:
             self.chapterUrls.append((self.story.getMetadata('title'),url))
         else:

fanficdownloader/adapters/adapter_test1.py

@@ -140,25 +140,25 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
-                            ('Chapter 6',self.url+"&chapter=6"),
-                            # ('Chapter 7',self.url+"&chapter=6"),
-                            # ('Chapter 8',self.url+"&chapter=6"),
-                            # ('Chapter 9',self.url+"&chapter=6"),
-                            # ('Chapter 0',self.url+"&chapter=6"),
-                            # ('Chapter a',self.url+"&chapter=6"),
-                            # ('Chapter b',self.url+"&chapter=6"),
-                            # ('Chapter c',self.url+"&chapter=6"),
-                            # ('Chapter d',self.url+"&chapter=6"),
-                            # ('Chapter e',self.url+"&chapter=6"),
-                            # ('Chapter f',self.url+"&chapter=6"),
-                            # ('Chapter g',self.url+"&chapter=6"),
-                            # ('Chapter h',self.url+"&chapter=6"),
-                            # ('Chapter i',self.url+"&chapter=6"),
-                            # ('Chapter j',self.url+"&chapter=6"),
-                            # ('Chapter k',self.url+"&chapter=6"),
-                            # ('Chapter l',self.url+"&chapter=6"),
-                            # ('Chapter m',self.url+"&chapter=6"),
-                            # ('Chapter n',self.url+"&chapter=6"),
+                            #('Chapter 6',self.url+"&chapter=7"),
+                            #('Chapter 7',self.url+"&chapter=8"),
+                            #('Chapter 8',self.url+"&chapter=9"),
+                            #('Chapter 9',self.url+"&chapter=0"),
+                            #('Chapter 0',self.url+"&chapter=a"),
+                            #('Chapter a',self.url+"&chapter=b"),
+                            #('Chapter b',self.url+"&chapter=c"),
+                            #('Chapter c',self.url+"&chapter=d"),
+                            #('Chapter d',self.url+"&chapter=e"),
+                            #('Chapter e',self.url+"&chapter=f"),
+                            #('Chapter f',self.url+"&chapter=g"),
+                            #('Chapter g',self.url+"&chapter=h"),
+                            #('Chapter h',self.url+"&chapter=i"),
+                            #('Chapter i',self.url+"&chapter=j"),
+                            #('Chapter j',self.url+"&chapter=k"),
+                            #('Chapter k',self.url+"&chapter=l"),
+                            #('Chapter l',self.url+"&chapter=m"),
+                            #('Chapter m',self.url+"&chapter=n"),
+                            #('Chapter n',self.url+"&chapter=o"),
                             ]
         self.story.setMetadata('numChapters',len(self.chapterUrls))
                             

fanficdownloader/adapters/adapter_yourfanfictioncom.py

@@ -41,7 +41,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         # targeted as us and offered to 'whitelist our IP'.  Clearly,
         # that wouldn't work, but it does let me do this in good
         # conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.5')]
+        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
 
         self.decode = ["Windows-1252",
                        "utf8"] # 1252 is a superset of iso-8859-1.
@@ -115,7 +115,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         # viewstory.php?sid=1654&ageconsent=ok&warning=5
         #print data
         #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
                 # We tried the default and still got a warning, so
@@ -123,7 +123,8 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 # link and reload data.
                 addurl = m.group(1)
                 # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
+                # explicitly put ageconsent because google appengine regexp doesn't include it for some reason.
+                addurl = addurl.replace("&","&")+'&ageconsent=ok'
                 url = self.url+'&index=1'+addurl
                 logging.debug("URL 2nd try: "+url)
 
@@ -140,9 +141,17 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
+        # because for some reason, this works while simple 'print data' errors on ascii conversion.
+        # loopdata = data
+        # chklen=5000
+        # while len(loopdata) > 0:
+        #     if len(loopdata) < 5000:
+        #         chklen = len(loopdata)
+        #     logging.info("loopdata: %s" % loopdata[:chklen])
+        #     loopdata = loopdata[chklen:]
+            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
-        # print data
 
         # Now go hunting for all the meta data and the chapter list.
         
@@ -182,9 +191,11 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
+                while value and not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
+                # sometimes poorly formated desc (<p> w/o </p>) leads
+                # to all labels being included.
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
@@ -245,7 +256,6 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                print("series a['href']:%s"%a['href'])
                 # skip 'report this' and 'TOC' links
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):

fanficdownloader/adapters/base_adapter.py

@@ -89,6 +89,7 @@ class BaseSiteAdapter(Configurable):
         self.oldimgs = None
         self.oldcover = None # (data of existing cover html, data of existing cover image)
         self.calibrebookmark = None
+        self.logfile = None
         ## order of preference for decoding.
         self.decode = ["utf8",
                        "Windows-1252"] # 1252 is a superset of
@@ -229,6 +230,8 @@ class BaseSiteAdapter(Configurable):
             # cheesy way to carry calibre bookmark file forward across update.
             if self.calibrebookmark:
                 self.story.calibrebookmark = self.calibrebookmark
+            if self.logfile:
+                self.story.logfile = self.logfile
                 
         return self.story
 

fanficdownloader/epubutils.py

@@ -42,6 +42,7 @@ def get_update_data(inputio,
             
     oldcover = None
     calibrebookmark = None
+    logfile = None
     # Looking for pre-existing cover.
     for item in contentdom.getElementsByTagName("reference"):
         if item.getAttribute("type") == "cover":
@@ -96,6 +97,11 @@ def get_update_data(inputio,
             if( item.getAttribute("media-type") == "application/xhtml+xml" ):
                 href=relpath+item.getAttribute("href")
                 #print("---- item href:%s path part: %s"%(href,get_path_part(href)))
+                if re.match(r'.*/log_page\.x?html',href):
+                    try:
+                        logfile = epub.read(href).decode("utf-8")
+                    except:
+                        pass # corner case I bumped into while testing.
                 if re.match(r'.*/(file|chapter)\d+\.x?html',href):
                     if getsoups:
                         soup = bs.BeautifulSoup(epub.read(href).decode("utf-8"))
@@ -136,7 +142,7 @@ def get_update_data(inputio,
                     
     for k in images.keys():
         print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
-    return (source,filecount,soups,images,oldcover,calibrebookmark)
+    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)
 
 def get_path_part(n):
     relpath = os.path.dirname(n)

fanficdownloader/geturls.py

@@ -25,14 +25,38 @@ from gziphttp import GZipProcessor
 
 import adapters
 
-def get_urls_from_page(url):
-    
-    opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
-    soup = BeautifulSoup(opener.open(url).read())
-    
+def get_urls_from_page(url,config=None):
+
     normalized = set() # normalized url
     retlist = [] # orig urls.
-    config = ConfigParser.SafeConfigParser()
+    if not config:
+        config = ConfigParser.SafeConfigParser()
+
+    data = None
+    
+    # special stuff to log into archiveofourown.org, if possible.
+    # Unlike most that show the links to 'adult' stories, but protect
+    # them, AO3 doesn't even show them if not logged in.  Only works
+    # with saved user/pass--not going to prompt for list.
+    if 'archiveofourown.org' in url:
+        ao3adapter = adapters.getAdapter(config,"http://www.archiveofourown.org/works/0","EPUB")
+        if ao3adapter.getConfig("username"):
+            if ao3adapter.getConfig("is_adult"):
+                addurl = "?view_adult=true"
+            else:
+                addurl=""
+            # just to get an authenticity_token.
+            data = ao3adapter._fetchUrl(url+addurl)
+            # login the session.
+            ao3adapter.performLogin(url,data)
+            # get the list page with logged in session.
+            data = ao3adapter._fetchUrl(url)
+
+    if not data:
+        opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+        data = opener.open(url).read()
+    
+    soup = BeautifulSoup(data)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):

fanficdownloader/story.py

@@ -201,6 +201,7 @@ class Story:
         self.cover=None # *href* of new cover image--need to create html.
         self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
         self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update.
+        self.logfile=None # cheesy way to carry log file forward across update.
 
     def setMetadata(self, key, value):
         ## still keeps < < and &
@@ -230,6 +231,8 @@ class Story:
             if value:
                 if key == "numWords":
                     value = commaGroups(value)
+                if key == "numChapters":
+                    value = commaGroups("%d"%value)
                 if key == "dateCreated":
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
@@ -279,7 +282,7 @@ class Story:
     # just for less clutter in adapters.
     def extendList(self,listname,l):
         for v in l:
-            self.addToList(listname,v)
+            self.addToList(listname,v.strip())
     
     def addToList(self,listname,value):
         if value==None:

fanficdownloader/writers/writer_epub.py

@@ -140,6 +140,101 @@ ${value}<br />
 </html>
 ''')
 
+        self.EPUB_LOG_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Update Log</title>
+<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
+</head>
+<body>
+<h3>Update Log</h3>
+''')
+
+        self.EPUB_LOG_ENTRY = string.Template('''
+<b>${label}:</b> <span id="${id}">${value}</span>
+''')
+                          
+        self.EPUB_LOG_PAGE_END = string.Template('''
+</body>
+</html>
+''')
+
+    def writeLogPage(self, out):
+        """
+       XXX 
+
+        
+        Write the log page, but only include entries that there's
+        metadata for.  START, ENTRY and END are expected to already by
+        string.Template().  START and END are expected to use the same
+        names as Story.metadata, but ENTRY should use id, label and value.
+        """
+        if self.getConfig("include_logpage"):
+
+            # if there's a self.story.logfile, there's an existing log
+            # to add to.
+            if self.story.logfile:
+                print("existing logfile found, appending")
+                print("existing data:%s"%self._getLastLogData(self.story.logfile))
+                replace_string = "</body>" # "</h3>"
+                self._write(out,self.story.logfile.replace(replace_string,self._makeLogEntry(self._getLastLogData(self.story.logfile))+replace_string))
+            else:
+                # otherwise, write a new one.
+                self._write(out,self.EPUB_LOG_PAGE_START.substitute(self.story.getAllMetadata()))
+                self._write(out,self._makeLogEntry())
+                self._write(out,self.EPUB_LOG_PAGE_END.substitute(self.story.getAllMetadata()))
+
+    # self parsing instead of Soup because it should be simple and not
+    # worth the overhead.
+    def _getLastLogData(self,logfile):
+        """
+        Make a dict() of the most recent(last) log entry for each piece of metadata.
+        Switch rindex to index to search from top instead of bottom.
+        """
+        values = {}
+        for entry in self.getConfigList("logpage_entries"):
+            if entry in self.validEntries:
+                try:
+                    # <span id="dateUpdated">1975-04-15</span>
+                    span = '<span id="%s">'%entry
+                    idx = logfile.rindex(span)+len(span)
+                    values[entry] = logfile[idx:logfile.index('</span>',idx)]
+                except Exception, e:
+                    #print("e:%s"%e)
+                    pass
+
+        return values
+        
+    def _makeLogEntry(self, oldvalues={}):
+        retval = "<p class='log_entry'>"
+
+        for entry in self.getConfigList("logpage_entries"):
+            if entry in self.validEntries:
+                val = self.story.getMetadata(entry)
+                if val and ( entry not in oldvalues or val != oldvalues[entry] ):
+                    if self.hasConfig(entry+"_label"):
+                        label=self.getConfig(entry+"_label")
+                    else:
+                        print("Using fallback label for %s_label"%entry)
+                        label=self.titleLabels[entry]
+
+                    retval = retval + self.EPUB_LOG_ENTRY.substitute({'id':entry,
+                                                                      'label':label,
+                                                                      'value':val})
+            else:
+                # could be useful for introducing extra text, but
+                # mostly it makes it easy to tell when you get the
+                # keyword wrong.
+                retval = retval + entry
+                
+        retval = retval + "</p><hr />"
+        
+        if self.getConfig('replace_hr'):
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+            
+        return retval
+                
     def writeStoryImpl(self, out):
 
         ## Python 2.5 ZipFile is rather more primative than later
@@ -349,6 +444,11 @@ div { margin: 0pt; padding: 0pt; }
         if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
             items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents"))
             itemrefs.append("toc_page")
+
+        if self.getConfig("include_logpage"):
+            items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
+            itemrefs.append("log_page")
+            
         for index, (title,html) in enumerate(self.story.getChapters()):
             if html:
                 i=index+1
@@ -480,6 +580,13 @@ div { margin: 0pt; padding: 0pt; }
             outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue())
         tocpageIO.close()
 
+        # write log page.
+        logpageIO = StringIO.StringIO()
+        self.writeLogPage(logpageIO)
+        if logpageIO.getvalue(): # will be false if no log page.
+            outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue())
+        logpageIO.close()
+        
         for index, (title,html) in enumerate(self.story.getChapters()):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)

index.html

@@ -54,13 +54,6 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>New Sites</h3>
-	  <p>
-	    
-	    New sites grangerenchanted.com, hlfiction.net and nha.magical-worlds.us.
-	    <br />Thanks, Ida!
-	    
-	  </p>
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -69,7 +62,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-20.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-21.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -19,8 +19,7 @@
 ## overridden at several levels
 
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.
 #is_adult:true
 
 ## All available titlepage_entries and the label used for them:
@@ -183,9 +182,24 @@ windows_eol: true
 [epub]
 
 ## epub carries the TOC in metadata.
-## mobi generated from epub will have a TOC at the end.
+## mobi generated from epub by calibre will have a TOC at the end.
 include_tocpage: false
 
+## include a Update Log page before the story text.  If included, the
+## log will be updated each time the epub is an all the metadata
+## fields that have changed since the last update (typically
+## dateUpdated,numChapters,numWords at a minimum) will be shown.
+## Great for tracking when chapters came out and when the description,
+## etc changed.
+include_logpage: false
+
+## items to include in the log page Empty metadata entries, or those
+## that haven't changed since the last update, will *not* appear, even
+## if in the list.  You can include extra text or HTML that will be
+## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
+## summary,<br />,...
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
+
 ## epub->mobi conversions typically don't like tables.
 titlepage_use_table: false
 

plugin-example.ini

@@ -3,8 +3,7 @@
 
 [defaults]
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.
 #is_adult:true
 
 ## include images from img tags in the body and summary of