Plugin-Make ini edit courier font & 1pt larger than default.

Many(most?) of the sites ignore gzip because of our User-Agent. ffnet honors it.
Add progress bar while getting URLs or getting URLs for update.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-2-0
+version: 4-3-2
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 1, 4)
+    version             = (1, 4, 1)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code
@@ -65,7 +65,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
         # GUI libraries to be loaded, which we do not want when using calibre
         # from the command line
         from calibre_plugins.fanfictiondownloader_plugin.config import ConfigWidget
-        return ConfigWidget()
+        return ConfigWidget(self.actual_plugin_)
 
     def save_settings(self, config_widget):
         '''

calibre-plugin/about.txt

@@ -1,4 +1,3 @@
-<p>FanFictionDownLoader Plugin</p>
 <hr />
 
 <p>Created by Jim Miller, borrowing heavily from Grant Drake's
@@ -12,10 +11,10 @@ Calibre officially distributes plugins from the mobileread.com forum site.
 The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>
 </p>
 
-<p> However, I monitor the
+<p> I also monitor the
 <a href="http://groups.google.com/group/fanfic-downloader">general users
-group</a> for the downloader more closely.  That also covers the web application and CLI.
+group</a> for the downloader.  That covers the web application and CLI, too.
 </p>
 
-The source project for this plugin is <a href="http://code.google.com/p/fanficdownloader/source/checkout">
-also available</a>.
+The source for this plugin is available
+<a href="http://code.google.com/p/fanficdownloader/source/checkout">here</a>.

calibre-plugin/config.py

@@ -7,41 +7,198 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit,
-                      QTextEdit, QComboBox, QCheckBox, QPushButton)
+import traceback, copy
+
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont,
+                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant)
 
 from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
+from calibre.gui2.ui import get_gui
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
     import (SKIP, ADDNEW, UPDATE, UPDATEALWAYS, OVERWRITE, OVERWRITEALWAYS,
              CALIBREONLY,collision_order)
 
+from calibre_plugins.fanfictiondownloader_plugin.common_utils \
+    import ( get_library_uuid, KeyboardConfigDialog )
+
+from calibre.gui2.complete import MultiCompleteLineEdit
+
 # This is where all preferences for this plugin will be stored
 # Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
 # in a global namespace, so make it as unique as possible.
 # You should always prefix your config file name with plugins/,
 # so as to ensure you dont accidentally clobber a calibre config file
-prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+all_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
 
-# Set defaults
-prefs.defaults['personal.ini'] = get_resources('example.ini')
-prefs.defaults['updatemeta'] = True
-prefs.defaults['keeptags'] = False
-#prefs.defaults['onlyoverwriteifnewer'] = False
-prefs.defaults['urlsfromclip'] = True
-prefs.defaults['updatedefault'] = True
-prefs.defaults['fileform'] = 'epub'
-prefs.defaults['collision'] = OVERWRITE
-prefs.defaults['deleteotherforms'] = False
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+all_prefs.defaults['personal.ini'] = get_resources('plugin-example.ini')
+all_prefs.defaults['updatemeta'] = True
+all_prefs.defaults['keeptags'] = False
+all_prefs.defaults['urlsfromclip'] = True
+all_prefs.defaults['updatedefault'] = True
+all_prefs.defaults['fileform'] = 'epub'
+all_prefs.defaults['collision'] = OVERWRITE
+all_prefs.defaults['deleteotherforms'] = False
+all_prefs.defaults['send_lists'] = ''
+all_prefs.defaults['read_lists'] = ''
+all_prefs.defaults['addtolists'] = False
+all_prefs.defaults['addtoreadlists'] = False
+all_prefs.defaults['addtolistsonread'] = False
+all_prefs.defaults['custom_cols'] = {}
 
+# The list of settings to copy from all_prefs or the previous library
+# when config is called for the first time on a library.
+copylist = ['personal.ini',
+            'updatemeta',
+            'keeptags',
+            'urlsfromclip',
+            'updatedefault',
+            'fileform',
+            'collision',
+            'deleteotherforms']
+
+# fake out so I don't have to change the prefs calls anywhere.  The
+# Java programmer in me is offended by op-overloading, but it's very
+# tidy.
+class PrefsFacade():
+    def __init__(self,all_prefs):
+        self.all_prefs = all_prefs
+        self.lastlibid = None
+
+    def _get_copylist_prefs(self,frompref):
+        return filter( lambda x : x[0] in copylist, frompref.items() )
+        
+    def _get_prefs(self):
+        libraryid = get_library_uuid(get_gui().current_db)
+        if libraryid not in self.all_prefs:
+            if self.lastlibid == None:
+                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs))
+            else:
+                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs[self.lastlibid]))
+            self.lastlibid = libraryid
+            
+        return self.all_prefs[libraryid]
+
+    def _save_prefs(self,prefs):
+        libraryid = get_library_uuid(get_gui().current_db)
+        self.all_prefs[libraryid] = prefs
+        
+    def __getitem__(self,k):            
+        prefs = self._get_prefs()
+        if k not in prefs:
+            # pulls from all_prefs.defaults automatically if not set
+            # in all_prefs
+            return self.all_prefs[k]
+        return prefs[k]
+
+    def __setitem__(self,k,v):
+        prefs = self._get_prefs()
+        prefs[k]=v
+        self._save_prefs(prefs)
+
+    # to be avoided--can cause unexpected results as possibly ancient
+    # all_pref settings may be pulled.
+    def __delitem__(self,k):
+        prefs = self._get_prefs()
+        del prefs[k]
+        self._save_prefs(prefs)
+
+prefs = PrefsFacade(all_prefs)
+    
 class ConfigWidget(QWidget):
 
-    def __init__(self):
+    def __init__(self, plugin_action):
         QWidget.__init__(self)
+        self.plugin_action = plugin_action
+        
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
+        tab_widget = QTabWidget(self)
+        self.l.addWidget(tab_widget)
+
+        self.basic_tab = BasicTab(self, plugin_action)
+        tab_widget.addTab(self.basic_tab, 'Basic')
+
+        self.personalini_tab = PersonalIniTab(self, plugin_action)
+        tab_widget.addTab(self.personalini_tab, 'personal.ini')
+        
+        self.list_tab = ListTab(self, plugin_action)
+        tab_widget.addTab(self.list_tab, 'Reading Lists')
+        if 'Reading List' not in plugin_action.gui.iactions:
+            self.list_tab.setEnabled(False)
+
+        self.columns_tab = ColumnsTab(self, plugin_action)
+        tab_widget.addTab(self.columns_tab, 'Custom Columns')
+
+        self.other_tab = OtherTab(self, plugin_action)
+        tab_widget.addTab(self.other_tab, 'Other')
+
+
+    def save_settings(self):
+
+        # basic
+        prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
+        prefs['collision'] = unicode(self.basic_tab.collision.currentText())
+        prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
+        prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
+        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
+        prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
+
+        if self.list_tab:
+            # lists
+            prefs['send_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.list_tab.send_lists_box.text()).split(','))))
+            prefs['read_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.list_tab.read_lists_box.text()).split(','))))
+            # print("send_lists: %s"%prefs['send_lists'])
+            # print("read_lists: %s"%prefs['read_lists'])
+            prefs['addtolists'] = self.list_tab.addtolists.isChecked()
+            prefs['addtoreadlists'] = self.list_tab.addtoreadlists.isChecked()
+            prefs['addtolistsonread'] = self.list_tab.addtolistsonread.isChecked()
+
+        # personal.ini
+        ini = unicode(self.personalini_tab.ini.toPlainText())
+        if ini:
+            prefs['personal.ini'] = ini
+        else:
+            # if they've removed everything, reset to default.
+            prefs['personal.ini'] = get_resources('plugin-example.ini')
+
+        # Custom Columns tab
+        colsmap = {}
+        for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems():
+            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            if val != 'none':
+                colsmap[col] = val
+                #print("colsmap[%s]:%s"%(col,colsmap[col]))
+        prefs['custom_cols'] = colsmap
+        
+    def edit_shortcuts(self):
+        self.save_settings()
+        # Force the menus to be rebuilt immediately, so we have all our actions registered
+        self.plugin_action.rebuild_menus()
+        d = KeyboardConfigDialog(self.plugin_action.gui, self.plugin_action.action_spec[0])
+        if d.exec_() == d.Accepted:
+            self.plugin_action.gui.keyboard.finalize()
+
+class BasicTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
         horz = QHBoxLayout()
         label = QLabel('Default Output &Format:')
         horz.addWidget(label)
@@ -73,7 +230,7 @@ class ConfigWidget(QWidget):
         self.l.addLayout(horz)
 
         self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update title, author, URL, tags, etc for story in Calibre from web site.')
+        self.updatemeta.setToolTip('Update title, author, URL, tags, custom columns, etc for story in Calibre from web site.')
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
@@ -82,13 +239,8 @@ class ConfigWidget(QWidget):
         self.keeptags.setChecked(prefs['keeptags'])
         self.l.addWidget(self.keeptags)
 
-        # self.onlyoverwriteifnewer = QCheckBox('Default Only Overwrite Story if Newer',self)
-        # self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer or from the same day.")
-        # self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer'])
-        # self.l.addWidget(self.onlyoverwriteifnewer)
-        
         self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
-        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New?')
+        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
         self.urlsfromclip.setChecked(prefs['urlsfromclip'])
         self.l.addWidget(self.urlsfromclip)
 
@@ -103,24 +255,7 @@ class ConfigWidget(QWidget):
         self.deleteotherforms.setChecked(prefs['deleteotherforms'])
         self.l.addWidget(self.deleteotherforms)
         
-        self.label = QLabel('personal.ini:')
-        self.l.addWidget(self.label)
-
-        self.ini = QTextEdit(self)
-        self.ini.setLineWrapMode(QTextEdit.NoWrap)
-        self.ini.setText(prefs['personal.ini'])
-        self.l.addWidget(self.ini)
-
-        self.defaults = QPushButton('View Defaults', self)
-        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
-        self.defaults.clicked.connect(self.show_defaults)
-        self.l.addWidget(self.defaults)
-        
-        reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self)
-        reset_confirmation_button.setToolTip(_(
-                    'Reset all show me again dialogs for the FanFictionDownLoader plugin'))
-        reset_confirmation_button.clicked.connect(self.reset_dialogs)
-        self.l.addWidget(reset_confirmation_button)
+        self.l.insertStretch(-1)
         
     def set_collisions(self):
         prev=self.collision.currentText()
@@ -132,37 +267,50 @@ class ConfigWidget(QWidget):
         if i > -1:
             self.collision.setCurrentIndex(i)
         
-    def save_settings(self):
-        prefs['fileform'] = unicode(self.fileform.currentText())
-        prefs['collision'] = unicode(self.collision.currentText())
-        prefs['updatemeta'] = self.updatemeta.isChecked()
-        prefs['keeptags'] = self.keeptags.isChecked()
-        prefs['urlsfromclip'] = self.urlsfromclip.isChecked()
-        prefs['updatedefault'] = self.updatedefault.isChecked()
-#        prefs['onlyoverwriteifnewer'] = self.onlyoverwriteifnewer.isChecked()
-        prefs['deleteotherforms'] = self.deleteotherforms.isChecked()
+    def show_defaults(self):
+        text = get_resources('plugin-defaults.ini')
+        ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
         
-        ini = unicode(self.ini.toPlainText())
-        if ini:
-            prefs['personal.ini'] = ini
-        else:
-            # if they've removed everything, clear it so they get the
-            # default next time.
-            del prefs['personal.ini']
+class PersonalIniTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set is_adult and user/password for different sites.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.label = QLabel('personal.ini:')
+        self.l.addWidget(self.label)
+
+        self.ini = QTextEdit(self)
+        try:
+            self.ini.setFont(QFont("Courier",
+                                   self.plugin_action.gui.font().pointSize()+1));
+        except Exception as e:
+            print("Couldn't get font: %s"%e)
+        self.ini.setLineWrapMode(QTextEdit.NoWrap)
+        self.ini.setText(prefs['personal.ini'])
+        self.l.addWidget(self.ini)
+
+        self.defaults = QPushButton('View Defaults', self)
+        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
+        self.defaults.clicked.connect(self.show_defaults)
+        self.l.addWidget(self.defaults)
+        
+        # self.l.insertStretch(-1)
+        # let edit box fill the space.
         
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
 
-    def reset_dialogs(self):
-        for key in dynamic.keys():
-            if key.startswith('fanfictiondownloader_') and key.endswith('_again') \
-                                                  and dynamic[key] is False:
-                dynamic[key] = True
-        info_dialog(self, _('Done'),
-                _('Confirmation dialogs have all been reset'), show=True)
-
-        
 class ShowDefaultsIniDialog(QDialog):
 
     def __init__(self, icon, text, parent=None):
@@ -171,13 +319,18 @@ class ShowDefaultsIniDialog(QDialog):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
         self.label = QLabel("Plugin Defaults (Read-Only)")
-        self.label.setToolTip("These all of the plugin's configurable settings\nand their default settings.")
+        self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
         self.setWindowTitle(_('Plugin Defaults'))
         self.setWindowIcon(icon)
         self.l.addWidget(self.label)
         
         self.ini = QTextEdit(self)
-        self.ini.setToolTip("These all of the plugin's configurable settings\nand their default settings.")
+        self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        try:
+            self.ini.setFont(QFont("Courier",
+                                   get_gui().font().pointSize()+1));
+        except Exception as e:
+            print("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(text)
         self.ini.setReadOnly(True)
@@ -187,3 +340,214 @@ class ShowDefaultsIniDialog(QDialog):
         self.ok_button.clicked.connect(self.hide)
         self.l.addWidget(self.ok_button)
         
+class ListTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        try:
+            rl_plugin = plugin_action.gui.iactions['Reading List']
+            reading_lists = rl_plugin.get_list_names()
+        except KeyError:
+            reading_lists= []
+            
+        label = QLabel('These settings provide integration with the Reading List Plugin.  Reading List can automatically send to devices and change custom columns.  You have to create and configure the lists in Reading List to be useful.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.addtolists = QCheckBox('Add new/updated stories to "Send to Device" Reading List(s).',self)
+        self.addtolists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.')
+        self.addtolists.setChecked(prefs['addtolists'])
+        self.l.addWidget(self.addtolists)
+            
+        horz = QHBoxLayout()
+        label = QLabel('"Send to Device" Reading Lists')
+        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        horz.addWidget(label)        
+        self.send_lists_box = MultiCompleteLineEdit(self)
+        self.send_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.send_lists_box.update_items_cache(reading_lists)
+        self.send_lists_box.setText(prefs['send_lists'])
+        horz.addWidget(self.send_lists_box)
+        self.l.addLayout(horz)
+        
+        self.addtoreadlists = QCheckBox('Add new/updated stories to "To Read" Reading List(s).',self)
+        self.addtoreadlists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.\nAlso offers menu option to remove stories from the "To Read" lists.')
+        self.addtoreadlists.setChecked(prefs['addtoreadlists'])
+        self.l.addWidget(self.addtoreadlists)
+            
+        horz = QHBoxLayout()
+        label = QLabel('"To Read" Reading Lists')
+        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        horz.addWidget(label)        
+        self.read_lists_box = MultiCompleteLineEdit(self)
+        self.read_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.read_lists_box.update_items_cache(reading_lists)
+        self.read_lists_box.setText(prefs['read_lists'])
+        horz.addWidget(self.read_lists_box)
+        self.l.addLayout(horz)
+        
+        self.addtolistsonread = QCheckBox('Add stories back to "Send to Device" Reading List(s) when marked "Read".',self)
+        self.addtolistsonread.setToolTip('Menu option to remove from "To Read" lists will also add stories back to "Send to Device" Reading List(s)')
+        self.addtolistsonread.setChecked(prefs['addtolistsonread'])
+        self.l.addWidget(self.addtolistsonread)
+            
+        self.l.insertStretch(-1)
+        
+class OtherTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        keyboard_shortcuts_button = QPushButton('Keyboard shortcuts...', self)
+        keyboard_shortcuts_button.setToolTip(_(
+                    'Edit the keyboard shortcuts associated with this plugin'))
+        keyboard_shortcuts_button.clicked.connect(parent_dialog.edit_shortcuts)
+        self.l.addWidget(keyboard_shortcuts_button)
+
+        reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self)
+        reset_confirmation_button.setToolTip(_(
+                    'Reset all show me again dialogs for the FanFictionDownLoader plugin'))
+        reset_confirmation_button.clicked.connect(self.reset_dialogs)
+        self.l.addWidget(reset_confirmation_button)
+        
+        self.l.insertStretch(-1)
+        
+    def reset_dialogs(self):
+        for key in dynamic.keys():
+            if key.startswith('fanfictiondownloader_') and key.endswith('_again') \
+                                                  and dynamic[key] is False:
+                dynamic[key] = True
+        info_dialog(self, _('Done'),
+                _('Confirmation dialogs have all been reset'), show=True)
+
+permitted_values = {
+    'int' : ['numWords','numChapters'],
+    'float' : ['numWords','numChapters'],
+    'bool' : ['status-C','status-I'],
+    'datetime' : ['datePublished', 'dateUpdated', 'dateCreated'],
+    'series' : ['series'],
+    'enumeration' : ['category',
+                     'genre',
+                     'series',
+                     'characters',
+                     'status',
+                     'datePublished',
+                     'dateUpdated',
+                     'dateCreated',
+                     'rating',
+                     'warnings',
+                     'numChapters',
+                     'numWords',
+                     'site',
+                     'storyId',
+                     'authorId',
+                     'extratags',
+                     'title',
+                     'storyUrl',
+                     'description',
+                     'author',
+                     'authorUrl',
+                     'formatname'
+                     #,'formatext'   # not useful information.
+                     #,'siteabbrev'
+                     #,'version'
+                     ]
+    }
+# no point copying the whole list.
+permitted_values['text'] = permitted_values['enumeration']
+permitted_values['comments'] = permitted_values['enumeration']
+
+titleLabels = {
+    'category':'Category',
+    'genre':'Genre',
+    'status':'Status',
+    'status-C':'Status:Completed',
+    'status-I':'Status:In-Progress',
+    'series':'Series',
+    'characters':'Characters',
+    'datePublished':'Published',
+    'dateUpdated':'Updated',
+    'dateCreated':'Packaged',
+    'rating':'Rating',
+    'warnings':'Warnings',
+    'numChapters':'Chapters',
+    'numWords':'Words',
+    'site':'Site',
+    'storyId':'Story ID',
+    'authorId':'Author ID',
+    'extratags':'Extra Tags',
+    'title':'Title',
+    'storyUrl':'Story URL',
+    'description':'Summary',
+    'author':'Author',
+    'authorUrl':'Author URL',
+    'formatname':'File Format',
+    'formatext':'File Extension',
+    'siteabbrev':'Site Abbrev',
+    'version':'FFDL Version'
+    }
+
+class ColumnsTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.custcol_dropdowns = {}
+
+        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
+
+        for key, column in custom_columns.iteritems():
+
+            if column['datatype'] in permitted_values:
+                # print("\n============== %s ===========\n"%key)
+                # for (k,v) in column.iteritems():
+                #     print("column['%s'] => %s"%(k,v))
+                horz = QHBoxLayout()
+                label = QLabel('%s(%s)'%(column['name'],key))
+                label.setToolTip("Update this %s column with..."%column['datatype'])
+                horz.addWidget(label)
+                dropdown = QComboBox(self)
+                dropdown.addItem('',QVariant('none'))
+                for md in permitted_values[column['datatype']]:
+                    dropdown.addItem(titleLabels[md],QVariant(md))
+                self.custcol_dropdowns[key] = dropdown
+                if key in prefs['custom_cols']:
+                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
+                if column['datatype'] == 'enumeration':
+                    dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
+                else:
+                    dropdown.setToolTip("Metadata values valid for this type of column.")
+
+                horz.addWidget(dropdown)
+                self.l.addLayout(horz)
+        
+        self.l.insertStretch(-1)
+
+        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])

calibre-plugin/dcsource.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+from zipfile import ZipFile
+
+from xml.dom.minidom import parseString
+
+def get_dcsource(inputio):
+    epub = ZipFile(inputio, 'r')
+
+    ## Find the .opf file.
+    container = epub.read("META-INF/container.xml")
+    containerdom = parseString(container)
+    rootfilenodelist = containerdom.getElementsByTagName("rootfile")
+    rootfilename = rootfilenodelist[0].getAttribute("full-path")
+
+    metadom = parseString(epub.read(rootfilename))
+    firstmetadom = metadom.getElementsByTagName("metadata")[0]
+    try:
+        source=firstmetadom.getElementsByTagName("dc:source")[0].firstChild.data.encode("utf-8")
+    except:
+        source=None
+
+    return source

calibre-plugin/dialogs.py

@@ -11,13 +11,16 @@ import traceback
 
 from PyQt4 import QtGui
 from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon,
+                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, QTextCursor,
                       QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
                       QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt,QAbstractItemView )
 
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 
+from calibre import confirm_config_name
+from calibre.gui2 import dynamic
+
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
@@ -45,7 +48,24 @@ class NotGoingToDownload(Exception):
 
     def __str__(self):
         return self.error
-    
+
+class DroppableQTextEdit(QTextEdit):
+    def __init__(self,parent):
+        QTextEdit.__init__(self,parent)
+        
+    def canInsertFromMimeData(self, source):
+        if source.hasUrls():
+            return True;
+        else:
+            return QTextEdit.canInsertFromMimeData(self,source)
+
+    def insertFromMimeData(self, source):
+        if source.hasUrls():
+            for u in source.urls():
+                self.append(u.toString())
+        else:
+            return QTextEdit.insertFromMimeData(self, source)
+                            
 class AddNewDialog(SizePersistedDialog):
 
     def __init__(self, gui, prefs, icon, url_list_text):
@@ -60,7 +80,7 @@ class AddNewDialog(SizePersistedDialog):
         self.setWindowIcon(icon)
 
         self.l.addWidget(QLabel('Story URL(s), one per line:'))
-        self.url = QTextEdit(self)
+        self.url = DroppableQTextEdit(self)
         self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.')
         self.url.setLineWrapMode(QTextEdit.NoWrap)
         self.url.setText(url_list_text)
@@ -132,7 +152,7 @@ class AddNewDialog(SizePersistedDialog):
         return {
             'fileform': unicode(self.fileform.currentText()),
             'collision': unicode(self.collision.currentText()),
-            'updatemeta': unicode(self.updatemeta.isChecked()),
+            'updatemeta': self.updatemeta.isChecked(),
             }
 
     def get_urlstext(self):
@@ -180,25 +200,27 @@ class UserPassDialog(QDialog):
         self.status=False
         self.hide()
 
-class MetadataProgressDialog(QProgressDialog):
+class LoopProgressDialog(QProgressDialog):
     '''
     ProgressDialog displayed while fetching metadata for each story.
     '''
     def __init__(self, gui,
                  book_list,
-                 options,
-                 metadata_function,
-                 startdownload_function):
+                 foreach_function,
+                 finish_function,
+                 init_label="Fetching metadata for stories...",
+                 win_title="Downloading metadata for stories",
+                 status_prefix="Fetched metadata for"):
         QProgressDialog.__init__(self,
-                                 "Fetching metadata for stories...",
+                                 init_label,
                                  QString(), 0, len(book_list), gui)
-        self.setWindowTitle("Downloading metadata for stories")
+        self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
         self.gui = gui
         self.book_list = book_list
-        self.options = options
-        self.metadata_function = metadata_function
-        self.startdownload_function = startdownload_function
+        self.foreach_function = foreach_function
+        self.finish_function = finish_function
+        self.status_prefix = status_prefix
         self.i = 0
         
         ## self.do_loop does QTimer.singleShot on self.do_loop also.
@@ -207,7 +229,7 @@ class MetadataProgressDialog(QProgressDialog):
         self.exec_()
 
     def updateStatus(self):
-        self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.book_list)))
+        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
         self.setValue(self.i+1)
         print(self.labelText())
 
@@ -220,7 +242,7 @@ class MetadataProgressDialog(QProgressDialog):
         try:
             ## collision spec passed into getadapter by partial from ffdl_plugin
             ## no retval only if it exists, but collision is SKIP
-            self.metadata_function(book)
+            self.foreach_function(book)
             
         except NotGoingToDownload as d:
             book['good']=False
@@ -245,7 +267,7 @@ class MetadataProgressDialog(QProgressDialog):
         self.hide()
         self.gui = None        
         # Queues a job to process these books in the background.
-        self.startdownload_function(self.book_list)
+        self.finish_function(self.book_list)
 
 class AboutDialog(QDialog):
 
@@ -290,7 +312,7 @@ class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):
 
 class UpdateExistingDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
-                 save_size_name='FanFictionDownLoader plugin:update list dialog'):
+                 save_size_name='fanfictiondownloader_plugin:update list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
         self.gui = gui
         
@@ -395,17 +417,38 @@ class UpdateExistingDialog(SizePersistedDialog):
         return {
             'fileform': unicode(self.fileform.currentText()),
             'collision': unicode(self.collision.currentText()),
-            'updatemeta': unicode(self.updatemeta.isChecked()),
+            'updatemeta': self.updatemeta.isChecked(),
             }
 
+def display_story_list(gui, header, prefs, icon, books,
+                       label_text='',
+                       save_size_name='fanfictiondownloader_plugin:display list dialog',
+                       offer_skip=False):
+    all_good = True
+    for b in books:
+        if not b['good']:
+            all_good=False
+            break
+        
+    ## 
+    if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
+        return True
+        pass
+            ## fake accept?
+    d = DisplayStoryListDialog(gui, header, prefs, icon, books,
+                               label_text,
+                               save_size_name,
+                               offer_skip and all_good)
+    d.exec_()
+    return d.result() == d.Accepted
+
 class DisplayStoryListDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
                  label_text='',
-                 save_size_name='FanFictionDownLoader plugin:display list dialog'):
+                 save_size_name='fanfictiondownloader_plugin:display list dialog',
+                 offer_skip=False):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        # UpdateExistingDialog.__init__(self, gui, header, prefs, icon, books,
-        #                               save_size_name='FanFictionDownLoader plugin:display list dialog')
-
+        self.name = save_size_name
         self.gui = gui
         
         self.setWindowTitle(header)
@@ -426,6 +469,15 @@ class DisplayStoryListDialog(SizePersistedDialog):
         #self.label.setWordWrap(True)
         options_layout.addWidget(self.label)
         
+        if offer_skip:
+            spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
+            options_layout.addItem(spacerItem1)
+            self.again = QCheckBox('Show this again?',self)
+            self.again.setChecked(True)
+            self.again.stateChanged.connect(self.toggle)
+            self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
+            options_layout.addWidget(self.again)
+        
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
         button_box.rejected.connect(self.reject)
@@ -437,10 +489,15 @@ class DisplayStoryListDialog(SizePersistedDialog):
         # Cause our dialog size to be restored from prefs or created on first usage
         self.resize_dialog()
         self.books_table.populate_table(books)
-        
+
     def get_books(self):
         return self.books_table.get_books()
 
+    def toggle(self, *args):
+        dynamic[confirm_config_name(self.name)] = self.again.isChecked()
+
+    
+
 class StoryListTableWidget(QTableWidget):
 
     def __init__(self, parent):

calibre-plugin/ffdl_plugin.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import time, os, copy
+import time, os, copy, threading
 from ConfigParser import SafeConfigParser
 from StringIO import StringIO
 from functools import partial
@@ -19,6 +19,9 @@ from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirect
 from calibre.ebooks.metadata import MetaInformation, authors_to_string
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
+from calibre.gui2.dialogs.message_box import ViewLog
+from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.utils.date import local_tz
 
 # The class that all interface action plugins must inherit from
 from calibre.gui2.actions import InterfaceAction
@@ -28,11 +31,12 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge
+from calibre_plugins.fanfictiondownloader_plugin.dcsource import get_dcsource
 
-from calibre_plugins.fanfictiondownloader_plugin.config import (prefs)
+from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values)
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
-    AddNewDialog, UpdateExistingDialog, DisplayStoryListDialog,
-    MetadataProgressDialog, UserPassDialog, AboutDialog,
+    AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog,
+    LoopProgressDialog, UserPassDialog, AboutDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
     NotGoingToDownload )
 
@@ -48,6 +52,9 @@ formmapping = {
 
 PLUGIN_ICONS = ['images/icon.png']
 
+sendlists = ["Send to Nook", "Send to Kindle", "Send to Droid", "Add to Nook", "Add to Kindle", "Add to Droid"]
+readlists = ["000"]
+
 class FanFictionDownLoaderPlugin(InterfaceAction):
 
     name = 'FanFictionDownLoader'
@@ -59,7 +66,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     # (text, icon_path, tooltip, keyboard shortcut)
     # icon_path isn't in the zip--icon loaded below.
     action_spec = (name, None,
-                   'Download FanFiction stories from various web sites', None)
+                   'Download FanFiction stories from various web sites', ())
+    # None for keyboard shortcut doesn't allow shortcut.  () does, there just isn't one yet
 
     action_type = 'global'
     # make button menu drop down only
@@ -73,11 +81,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         icon_resources = self.load_resources(PLUGIN_ICONS)
         set_plugin_icon_resources(self.name, icon_resources)
         
-        # Show the config dialog
-        # The config dialog can also be shown from within
-        # Preferences->Plugins, which is why the do_user_config
-        # method is defined on the base plugin class
-        do_user_config = self.interface_action_base_plugin.do_user_config
         base = self.interface_action_base_plugin
         self.version = base.name+" v%d.%d.%d"%base.version
 
@@ -102,38 +105,107 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         # Assign our menu to this action
         self.menu = QMenu(self.gui)
+        self.old_actions_unique_map = {}
         self.qaction.setMenu(self.menu)
-
         self.menu.aboutToShow.connect(self.about_to_show_menu)
 
-        self.actions_unique_map = {}
-
-        self.add_action = self.create_menu_item_ex(self.menu, '&Add New from URL(s)', image='plus.png',
-                                                   unique_name='Add New FanFiction Book(s) from URL(s)',
-                                                   shortcut_name='Add New FanFiction Book(s) from URL(s)',
-                                                   triggered=self.add_dialog )
-
-        self.update_action = self.create_menu_item_ex(self.menu, '&Update Existing FanFiction Book(s)', image='plusplus.png',
-                                                      unique_name='Update Existing FanFiction Book(s)',
-                                                      shortcut_name='Update Existing FanFiction Book(s)',
-                                                      triggered=self.update_existing) #partial(self._update_existing,'qwerty'))
-
-        self.menu.addSeparator()
-        self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False,
-                                                       image= 'config.png',
-                                                       unique_name='Configure FanFictionDownLoader',
-                                                       shortcut_name='Configure FanFictionDownLoader',
-                                                       triggered=partial(do_user_config,parent=self.gui))
-
-        self.config_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False,
-                                                       image= 'images/icon.png',
-                                                       unique_name='About FanFictionDownLoader',
-                                                       shortcut_name='About FanFictionDownLoader',
-                                                       triggered=self.about)
+        self.menus_lock = threading.RLock()
 
+    def initialization_complete(self):
+        # otherwise configured hot keys won't work until the menu's
+        # been displayed once.
+        self.rebuild_menus()
 
     def about_to_show_menu(self):
-        self.update_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
+        self.rebuild_menus()
+
+    def library_changed(self, db):
+        # We need to reset our menus after switching libraries
+        self.rebuild_menus()
+        
+    def rebuild_menus(self):
+        with self.menus_lock:
+            # Show the config dialog
+            # The config dialog can also be shown from within
+            # Preferences->Plugins, which is why the do_user_config
+            # method is defined on the base plugin class
+            do_user_config = self.interface_action_base_plugin.do_user_config
+            self.menu.clear()
+            self.actions_unique_map = {}
+            self.add_action = self.create_menu_item_ex(self.menu, '&Add New from URL(s)', image='plus.png',
+                                                       unique_name='Add New FanFiction Book(s) from URL(s)',
+                                                       shortcut_name='Add New FanFiction Book(s) from URL(s)',
+                                                       triggered=self.add_dialog )
+
+            self.update_action = self.create_menu_item_ex(self.menu, '&Update Existing FanFiction Book(s)', image='plusplus.png',
+                                                          unique_name='Update Existing FanFiction Book(s)',
+                                                          shortcut_name='Update Existing FanFiction Book(s)',
+                                                          triggered=self.update_existing)
+
+            if 'Reading List' in self.gui.iactions and (prefs['addtolists'] or prefs['addtoreadlists']) :
+                self.menu.addSeparator()
+                addmenutxt, rmmenutxt = None, None
+                if prefs['addtolists'] and prefs['addtoreadlists'] :
+                    addmenutxt = 'Add to "To Read" and "Send to Device" Lists'
+                    if prefs['addtolistsonread']:
+                        rmmenutxt = 'Remove from "To Read" and add to "Send to Device" Lists'
+                    else:
+                        rmmenutxt = 'Remove from "To Read" Lists'
+                elif prefs['addtolists'] :
+                    addmenutxt = 'Add Selected to "Send to Device" Lists'
+                elif prefs['addtoreadlists']:
+                    addmenutxt = 'Add to "To Read" Lists'
+                    rmmenutxt = 'Remove from "To Read" Lists'
+
+                if addmenutxt:
+                    self.add_send_action = self.create_menu_item_ex(self.menu, addmenutxt, image='plusplus.png',
+                                                                    unique_name=addmenutxt,
+                                                                    shortcut_name=addmenutxt,
+                                                                    triggered=partial(self.update_lists,add=True))
+
+                if rmmenutxt:
+                    self.add_remove_action = self.create_menu_item_ex(self.menu, rmmenutxt, image='minusminus.png',
+                                                                      unique_name=rmmenutxt,
+                                                                      shortcut_name=rmmenutxt,
+                                                                      triggered=partial(self.update_lists,add=False))
+                
+                # try:
+                #     self.add_send_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
+                # except:
+                #     pass
+                # try:
+                #     self.add_remove_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
+                # except:
+                #     pass
+
+            self.menu.addSeparator()
+            self.get_list_action = self.create_menu_item_ex(self.menu, 'Get URLs from Selected Books', image='bookmarks.png',
+                                                            unique_name='Get URLs from Selected Books',
+                                                            shortcut_name='Get URLs from Selected Books',
+                                                            triggered=self.get_list_urls)
+
+            self.menu.addSeparator()
+            self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False,
+                                                           image= 'config.png',
+                                                           unique_name='Configure FanFictionDownLoader',
+                                                           shortcut_name='Configure FanFictionDownLoader',
+                                                           triggered=partial(do_user_config,parent=self.gui))
+            
+            self.config_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False,
+                                                           image= 'images/icon.png',
+                                                           unique_name='About FanFictionDownLoader',
+                                                           shortcut_name='About FanFictionDownLoader',
+                                                           triggered=self.about)
+            
+            # self.update_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
+            # self.get_list_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
+
+            # Before we finalize, make sure we delete any actions for menus that are no longer displayed
+            for menu_id, unique_name in self.old_actions_unique_map.iteritems():
+                if menu_id not in self.actions_unique_map:
+                    self.gui.keyboard.unregister_shortcut(unique_name)
+            self.old_actions_unique_map = self.actions_unique_map
+            self.gui.keyboard.finalize()            
 
     def about(self):
         # Get the about text from a file inside the plugin zip file
@@ -145,10 +217,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # should pass a list of names to get_resources. In this case,
         # get_resources will return a dictionary mapping names to bytes. Names that
         # are not found in the zip file will not be in the returned dictionary.
+        
         text = get_resources('about.txt')
-#        QMessageBox.about(self, 'About the FanFictionDownLoader Plugin',
- #               text.decode('utf-8'))
-        AboutDialog(self.gui,self.qaction.icon(),text).exec_()
+        AboutDialog(self.gui,self.qaction.icon(),self.version + text).exec_()
         
     def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
                            shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
@@ -163,7 +234,44 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self.update_existing()
         else:
             self.add_dialog()
-    
+
+    def update_lists(self,add=True):
+        if len(self.gui.library_view.get_selected_ids()) > 0 and \
+                (prefs['addtolists'] or prefs['addtoreadlists']) :
+            self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)
+        #self.gui.library_view.model().refresh_ids(self.gui.library_view.get_selected_ids())
+
+    def get_list_urls(self):
+        if len(self.gui.library_view.get_selected_ids()) > 0:
+            book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
+
+            LoopProgressDialog(self.gui,
+                               book_list,
+                               partial(self._get_story_url_for_list, db=self.gui.current_db),
+                               self._finish_get_list_urls,
+                               init_label="Collecting URLs for stories...",
+                               win_title="Get URLs for stories",
+                               status_prefix="URL retrieved")
+            
+    def _get_story_url_for_list(self,book,db=None):
+        book['url'] = self._get_story_url(db,book['calibre_id'])
+        if book['url'] == None:
+            book['good']=False
+        else:
+            book['good']=True
+            
+    def _finish_get_list_urls(self, book_list):
+        url_list = [ x['url'] for x in book_list if x['good'] ]
+        if url_list:
+            d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
+            d.setWindowIcon(get_icon('bookmarks.png'))
+            d.exec_()
+        else:
+            info_dialog(self.gui, _('List of URLs'),
+                        _('No URLs found in selected books.'),
+                        show=True,
+                        show_copy_button=False)
+                
     def add_dialog(self):
 
         #print("add_dialog()")
@@ -196,18 +304,32 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         self.start_downloads( options, add_books )
         
     def update_existing(self):
+        if len(self.gui.library_view.get_selected_ids()) == 0:
+            return
         #print("update_existing()")
-        previous = self.gui.library_view.currentIndex()
+        
         db = self.gui.current_db
-        book_ids = self.gui.library_view.get_selected_ids()
-        books = self._convert_calibre_ids_to_books(db, book_ids)
+        book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
+        #book_ids = self.gui.library_view.get_selected_ids()
+
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self._populate_book_from_calibre_id, db=self.gui.current_db),
+                           self._update_existing_2,
+                           init_label="Collecting stories for update...",
+                           win_title="Get stories for updates",
+                           status_prefix="URL retrieved")            
+        
+        #books = self._convert_calibre_ids_to_books(db, book_ids)
         #print("update books:%s"%books)
 
+    def _update_existing_2(self,book_list):
+        
         d = UpdateExistingDialog(self.gui,
                                  'Update Existing List',
                                  prefs,
                                  self.qaction.icon(),
-                                 books,
+                                 book_list,
                                  )
         d.exec_()
         if d.result() != d.Accepted:
@@ -246,14 +368,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
         
-        MetadataProgressDialog(self.gui,
+        if 0 < len(filter(lambda x : x['good'], books)):
+            LoopProgressDialog(self.gui,
                                books,
-                               options,
                                partial(self.get_metadata_for_book, options = options),
                                partial(self.start_download_list, options = options))
-        # MetadataProgressDialog calls get_metadata_for_book for each 'good' story,
+        # LoopProgressDialog calls get_metadata_for_book for each 'good' story,
         # get_metadata_for_book updates book for each,
-        # MetadataProgressDialog calls start_download_list at the end which goes
+        # LoopProgressDialog calls start_download_list at the end which goes
         # into the BG, or shows list if no 'good' books.
 
     def get_metadata_for_book(self,book,
@@ -262,7 +384,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                        'updatemeta':True}):
         '''
         Update passed in book dict with metadata from website and
-        necessary data.  To be called from MetadataProgressDialog
+        necessary data.  To be called from LoopProgressDialog
         'loop'.  Also pops dialogs for is adult, user/pass.
         '''
         
@@ -282,6 +404,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
         url = book['url']
         print("url:%s"%url)
+        skip_date_update = False
         
         ## was self.ffdlconfig, but we need to be able to change it
         ## when doing epub update.
@@ -315,13 +438,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         story = adapter.getStoryMetadataOnly()
         writer = writers.getWriter(options['fileform'],adapter.config,adapter)
 
+        book['all_metadata'] = story.getAllMetadata(removeallentities=True)
         book['title'] = story.getMetadata("title", removeallentities=True)
         book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
         book['tags'] = writer.getTags()
-        book['pubdate'] = story.getMetadataRaw('datePublished')
-        book['timestamp'] = story.getMetadataRaw('dateCreated')
         book['comments'] = story.getMetadata("description") #, removeallentities=True) comments handles entities better.
+        book['series'] = story.getMetadata("series")
         
         # adapter.opener is the element with a threadlock.  But del
         # adapter.opener doesn't work--subproc fails when it tries
@@ -333,12 +456,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['password'] = adapter.password
 
         book['icon'] = 'plus.png'
+        if story.getMetadataRaw('datePublished'):
+            # should only happen when an adapter is broken, but better to
+            # fail gracefully.
+            book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
+        book['timestamp'] = None # filled below if not skipped.
         
         if collision in (CALIBREONLY):
             book['icon'] = 'metadata.png'
         
-        # XXX should really do a 'you can't do that' dialog when they
-        # hit 'OK' for this case.
+        # Dialogs should prevent this case now.
         if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub':
             raise NotGoingToDownload("Cannot update non-epub format.")
         
@@ -389,7 +516,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             
             ## newer/chaptercount checks are the same for both:
             # Update epub, but only if more chapters.
-            if collision == UPDATE:
+            if collision in (UPDATE,UPDATEALWAYS): # collision == UPDATE
                 # 'book' can exist without epub.  If there's no existing epub,
                 # let it go and it will download it.
                 if db.has_format(book_id,fileform,index_is_id=True):
@@ -400,12 +527,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                      titlenavpoints=False,
                                                      striptitletoc=True,
                                                      forceunique=False)
-    
                     urlchaptercount = int(story.getMetadata('numChapters'))
-                    if chaptercount == urlchaptercount: # and not onlyoverwriteifnewer:
-                        raise NotGoingToDownload("Already contains %d chapters."%chaptercount,'edit-undo.png')
+                    if chaptercount == urlchaptercount:
+                        if collision == UPDATE:
+                            raise NotGoingToDownload("Already contains %d chapters."%chaptercount,'edit-undo.png')
+                        else:
+                            # UPDATEALWAYS
+                            skip_date_update = True
                     elif chaptercount > urlchaptercount:
-                        raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d." % (chaptercount,urlchaptercount),'dialog_error.png')
+                        raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
     
             if collision == OVERWRITE and \
                     db.has_format(book_id,formmapping[fileform],index_is_id=True):
@@ -426,6 +556,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 print("existing epub tmp:"+tmp.name)
                 book['epub_for_update'] = tmp.name
 
+        if collision != CALIBREONLY and not skip_date_update:
+            # I'm half convinced this should be dateUpdated instead, but
+            # this behavior matches how epubs come out when imported
+            # dateCreated == packaged--epub/etc created.
+            book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
+        
         if book['good']: # there shouldn't be any !'good' books at this point.
             # if still 'good', make a temp file to write the output to.
             tmp = PersistentTemporaryFile(prefix='new-%s-'%book['calibre_id'],
@@ -442,7 +578,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                      'collision':ADDNEW,
                                      'updatemeta':True}):
         '''
-        Called by MetadataProgressDialog to start story downloads BG processing.
+        Called by LoopProgressDialog to start story downloads BG processing.
         adapter_list is a list of tuples of (url,adapter)
         '''
         #print("start_download_list:book_list:%s"%book_list)
@@ -457,18 +593,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self.download_list_completed(notjob,options=options)
             return
 
-        # ## XXX show list before starting download.
-        # d = DisplayStoryListDialog(self.gui,
-        #                            'Download List',
-        #                            prefs,
-        #                            self.qaction.icon(),
-        #                            book_list,
-        #                            label_text='Status of stories to be downloaded'
-        #                            )
-        # d.exec_()
-        # if d.result() != d.Accepted:
-        #     return
-            
         for book in book_list:
             if book['good']:
                 break
@@ -497,99 +621,82 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
         self.gui.status_bar.show_message('Starting %d FanFictionDownLoads'%len(book_list),3000)
 
+    def _update_book(self,book,db=None,
+                     options={'fileform':'epub',
+                              'collision':ADDNEW,
+                              'updatemeta':True}):
+        print("add/update %s %s"%(book['title'],book['url']))
+        mi = self._make_mi_from_book(book)
+                
+        if options['collision'] != CALIBREONLY:
+            self._add_or_update_book(book,options,prefs,mi)
+
+        if options['collision'] == CALIBREONLY or \
+                (options['updatemeta'] and book['good']):
+            self._update_metadata(db, book['calibre_id'], book, mi)        
+
+    def _update_books_completed(self, book_list, options={}):
+        
+        add_list = filter(lambda x : x['good'] and x['added'], book_list)
+        update_list = filter(lambda x : x['good'] and not x['added'], book_list)
+        update_ids = [ x['calibre_id'] for x in update_list ]
+        
+        if len(add_list):
+            ## even shows up added to searchs.  Nice.
+            self.gui.library_view.model().books_added(len(add_list))
+
+        if update_ids:
+            self.gui.library_view.model().refresh_ids(update_ids)
+
+        current = self.gui.library_view.currentIndex()
+        self.gui.library_view.model().current_changed(current, self.previous)
+        self.gui.tags_view.recount()
+        
+        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
+            
+        if len(update_list) + len(add_list) != len(book_list):
+            d = DisplayStoryListDialog(self.gui,
+                                       'Updates completed, final status',
+                                       prefs,
+                                       self.qaction.icon(),
+                                       book_list,
+                                       label_text='Stories have be added or updated in Calibre, some had additional problems.'
+                                       )
+            d.exec_()
+    
+        print("all done, remove temp dir.")
+        remove_dir(options['tdir'])
+        
     def download_list_completed(self, job, options={}):
         if job.failed:
             self.gui.job_exception(job, dialog_title='Failed to Download Stories')
             return
-        #print("download_list_completed job:%s"%job.result)
-        # for b in job.result:
-        #     print("job.result: %s"%b['title'])
 
+        self.previous = self.gui.library_view.currentIndex()
         db = self.gui.current_db
 
-        d = DisplayStoryListDialog(self.gui,
-                                   'Downloads finished, confirm to update Calibre',
-                                   prefs,
-                                   self.qaction.icon(),
-                                   job.result,
-                                   label_text='Stories will not be added or updated in Calibre without confirmation.'
-                                   )
-        d.exec_()
-        if d.result() == d.Accepted:
+        if display_story_list(self.gui,
+                              'Downloads finished, confirm to update Calibre',
+                              prefs,
+                              self.qaction.icon(),
+                              job.result,
+                              label_text='Stories will not be added or updated in Calibre without confirmation.',
+                              offer_skip=True):
             
-            ## in case the user removed any from the list.
-            book_list = d.get_books()
-            # for b in book_list:
-            #     print("d list: %s"%b['title'])
-
-            # update_list = filter(lambda x : x['good'] and x['calibre_id'] != None,
-            #                      book_list)
-
-            # add_list = filter(lambda x : x['good'] and x['calibre_id'] == None,
-            #                   book_list)
-
+            book_list = job.result
             good_list = filter(lambda x : x['good'], book_list)
+            total_good = len(good_list)
 
-            total_good = len(good_list) #update_list)+len(add_list)
+            self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
 
-            self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good), 3000)
-            
-            #print("==================================================")
-            # addfiles,addfileforms,addmis=[],[],[]
-
-            list_000_ids = []
-            
-            for book in good_list:
-                print("add/update %s %s"%(book['title'],book['url']))
-                mi = self._make_mi_from_book(book)
-                
-                if options['collision'] != CALIBREONLY:
-                    self._add_or_update_book(book,options,prefs,mi)
-                    # book000 = copy.copy(book)
-                    # book000['title'] = "000 %s"%book000['title']
-                    # book000['calibre_id'] = self._find_existing_book_id(db,book000)
-                    # list_000_ids.append(self._add_or_update_book(book000,options,prefs))
-
-                if options['collision'] == CALIBREONLY or \
-                        (options['updatemeta'] and book['good']) :
-                    if prefs['keeptags']:
-                        old_tags = db.get_tags(book['calibre_id'])
-                        # remove old Completed/In-Progress only if there's a new one.
-                        if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
-                            old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags)
-                        # remove old Last Update tags if there are new ones.
-                        if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
-                            old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
-                        # mi.tags needs to be list, but set kills dups.
-                        mi.tags = list(set(list(old_tags)+mi.tags)) 
-                        
-                    db.set_metadata(book['calibre_id'],mi)
-                    
-            add_list = filter(lambda x : x['good'] and x['added'], book_list)
-            update_list = filter(lambda x : x['good'] and not x['added'], book_list)
-            update_ids = [ x['calibre_id'] for x in update_list ]
-                    
-            if len(add_list)+len(list_000_ids):
-                ## even shows up added to searchs.  Nice.
-                self.gui.library_view.model().books_added(len(add_list)+len(list_000_ids))
-
-            if update_ids:
-                self.gui.library_view.model().refresh_ids(update_ids+list_000_ids)
-
-            self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
-            
-            if len(update_list) + len(add_list) != total_good:
-                d = DisplayStoryListDialog(self.gui,
-                                           'Updates completed, final status',
-                                           prefs,
-                                           self.qaction.icon(),
-                                           book_list,
-                                           label_text='Stories have be added or updated in Calibre, some had additional problems.'
-                                           )
-                d.exec_()
-    
-        print("all done, remove temp dir.")
-        remove_dir(options['tdir'])
+            if total_good > 0:
+                LoopProgressDialog(self.gui,
+                                   good_list,
+                                   partial(self._update_book, options=options, db=self.gui.current_db),
+                                   partial(self._update_books_completed, options=options),
+                                   init_label="Updating calibre for stories...",
+                                   win_title="Update calibre for stories",
+                                   status_prefix="Updated")
 
     def _add_or_update_book(self,book,options,prefs,mi=None):
         db = self.gui.current_db
@@ -620,14 +727,113 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     print("remove f:"+fmt)
                     db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
 
-        # rl_plugin = self.gui.iactions['Reading List']    
-        # rl_plugin.add_books_to_list("Send",
-        #                             [book_id],
-        #                             refresh_screen=False,
-        #                             display_warnings=False)
+        if prefs['addtolists'] or prefs['addtoreadlists']:
+            self._update_reading_lists([book_id],add=True)
                 
         return book_id
 
+    def _update_metadata(self, db, book_id, book, mi):
+        if prefs['keeptags']:
+            old_tags = db.get_tags(book_id)
+            # remove old Completed/In-Progress only if there's a new one.
+            if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
+                old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags)
+                # remove old Last Update tags if there are new ones.
+            if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
+                old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
+            # mi.tags needs to be list, but set kills dups.
+            mi.tags = list(set(list(old_tags)+mi.tags)) 
+        # Set language english, but only if not already set.
+        oldmi = db.get_metadata(book_id,index_is_id=True)
+        if not oldmi.languages:
+            mi.languages=['eng']
+        db.set_metadata(book_id,mi)
+
+        # do configured column updates here.
+        #print("all_metadata: %s"%book['all_metadata'])
+        custom_columns = self.gui.library_view.model().custom_columns
+
+        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
+        for col, meta in prefs['custom_cols'].iteritems():
+            #print("setting %s to %s"%(col,meta))
+            if col not in custom_columns:
+                print("%s not an existing column, skipping."%col)
+                continue
+            coldef = custom_columns[col]
+            if not meta.startswith('status-') and meta not in book['all_metadata']:
+                print("No value for %s, skipping."%meta)
+                continue
+            if meta not in permitted_values[coldef['datatype']]:
+                print("%s not a valid column type for %s, skipping."%(col,meta))
+                continue
+            label = coldef['label']
+            if coldef['datatype'] in ('enumeration','text','comments','datetime','series'):
+                db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
+            elif coldef['datatype'] in ('int','float'):
+                num = unicode(book['all_metadata'][meta]).replace(",","")
+                db.set_custom(book_id, num, label=label, commit=False)
+            elif coldef['datatype'] == 'bool' and meta.startswith('status-'):
+                if meta == 'status-C':
+                    val = book['all_metadata']['status'] == 'Completed'
+                if meta == 'status-I':
+                    val = book['all_metadata']['status'] == 'In-Progress'
+                db.set_custom(book_id, val, label=label, commit=False)
+                
+        db.commit()
+
+    def _get_clean_reading_lists(self,lists):
+        if lists == None or lists.strip() == "" :
+            return []
+        else:
+            return filter( lambda x : x, map( lambda x : x.strip(), lists.split(',') ) )
+        
+    def _update_reading_lists(self,book_ids,add=True):
+        try:
+            rl_plugin = self.gui.iactions['Reading List']
+        except:
+            if prefs['addtolists'] or prefs['addtoreadlists']:
+                message="<p>You configured FanFictionDownLoader to automatically update Reading Lists, but you don't have the Reading List plugin installed anymore?</p>"
+                confirm(message,'fanfictiondownloader_no_reading_list_plugin', self.gui)
+            return
+        
+        # XXX check for existence of lists, warning if not.
+        if prefs['addtoreadlists']:
+            if add:
+                addremovefunc = rl_plugin.add_books_to_list
+            else:
+                addremovefunc = rl_plugin.remove_books_from_list
+                
+            lists = self._get_clean_reading_lists(prefs['read_lists'])
+            if len(lists) < 1 :
+                message="<p>You configured FanFictionDownLoader to automatically update \"To Read\" Reading Lists, but you don't have any lists set?</p>"
+                confirm(message,'fanfictiondownloader_no_read_lists', self.gui)
+            for l in lists:
+                if l in rl_plugin.get_list_names():
+                    #print("add good read l:(%s)"%l)
+                    addremovefunc(l,
+                                  book_ids,
+                                  display_warnings=False)
+                else:
+                    if l != '':
+                        message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
+                        confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)
+                        
+        if prefs['addtolists'] and (add or (prefs['addtolistsonread'] and prefs['addtoreadlists']) ):
+            lists = self._get_clean_reading_lists(prefs['send_lists'])
+            if len(lists) < 1 :
+                message="<p>You configured FanFictionDownLoader to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?</p>"
+                confirm(message,'fanfictiondownloader_no_send_lists', self.gui)
+            for l in lists:
+                if l in rl_plugin.get_list_names():
+                    #print("good send l:(%s)"%l)
+                    rl_plugin.add_books_to_list(l,
+                                                book_ids,
+                                                display_warnings=False)
+                else:
+                    if l != '':
+                        message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
+                        confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)
+
     def _find_existing_book_id(self,db,book,matchurl=True):
         mi = MetaInformation(book["title"],(book["author"],)) # author is a list.
         identicalbooks = db.find_identical_books(mi)
@@ -637,65 +843,84 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     return ib
         if identicalbooks:
             return identicalbooks.pop()
+        return None
     
-        
-
     def _make_mi_from_book(self,book):
         mi = MetaInformation(book['title'],(book['author'],)) # author is a list.
         mi.set_identifiers({'url':book['url']})
         mi.publisher = book['publisher']
         mi.tags = book['tags']
-        #mi.languages = ['en']
+        #mi.languages = ['en'] # handled in _update_metadata so it can check for existing lang.
         mi.pubdate = book['pubdate']
         mi.timestamp = book['timestamp']
         mi.comments = book['comments']
+        mi.series = book['series']
         return mi
 
 
     def _convert_urls_to_books(self, urls):
         books = []
+        uniqueurls = set()
         for url in urls:
-            book = {}
-            book['good'] = True
-            book['calibre_id'] = None
-            book['title'] = 'Unknown'
-            book['author'] = 'Unknown'
-            book['author_sort'] = 'Unknown'
-                
-            book['comment'] = ''
-            book['url'] = ''
-            book['added'] = False
-            
-            self._set_book_url_and_comment(book,url)
-                    
+            book = self._convert_url_to_book(url)
+            if book['url'] in uniqueurls:
+                book['good'] = False
+                book['comment'] = "Same story already included."
+            uniqueurls.add(book['url'])
             books.append(book)
         return books
+
+    def _convert_url_to_book(self, url):
+        book = {}
+        book['good'] = True
+        book['calibre_id'] = None
+        book['title'] = 'Unknown'
+        book['author'] = 'Unknown'
+        book['author_sort'] = 'Unknown'
+            
+        book['comment'] = ''
+        book['url'] = ''
+        book['added'] = False
+        
+        self._set_book_url_and_comment(book,url)
+        return book
+        
+    def _convert_id_to_book(self, idval, good=True):
+        book = {}
+        book['good'] = good
+        book['calibre_id'] = idval
+        book['title'] = 'Unknown'
+        book['author'] = 'Unknown'
+        book['author_sort'] = 'Unknown'
+            
+        book['comment'] = ''
+        book['url'] = ''
+        book['added'] = False
+        
+        return book
+        
     
-    def _convert_calibre_ids_to_books(self, db, ids):
-        books = []
-        for book_id in ids:
-            mi = db.get_metadata(book_id, index_is_id=True)
-            book = {}
-            book['good'] = True
-            book['calibre_id'] = mi.id
-            book['title'] = mi.title
-            book['author'] = authors_to_string(mi.authors)
-            book['author_sort'] = mi.author_sort
-            # book['series'] = mi.series
-            # if mi.series:
-            #     book['series_index'] = mi.series_index
-            # else:
-            #     book['series_index'] = 0
-                
-            book['comment'] = ''
-            book['url'] = ""
-            book['added'] = False
+    # def _convert_calibre_ids_to_books(self, db, ids):
+    #     books = []
+    #     for book_id in ids:
+    #         books.append(self._convert_calibre_id_to_book(db,book_id))
+    #     return books
             
-            url = self._get_story_url(db,book_id)
-            self._set_book_url_and_comment(book,url)
-                    
-            books.append(book)
-        return books
+    def _populate_book_from_calibre_id(self, book, db=None):
+        mi = db.get_metadata(book['calibre_id'], index_is_id=True)
+        #book = {}
+        book['good'] = True
+        book['calibre_id'] = mi.id
+        book['title'] = mi.title
+        book['author'] = authors_to_string(mi.authors)
+        book['author_sort'] = mi.author_sort
+        book['comment'] = ''
+        book['url'] = ""
+        book['added'] = False
+        
+        url = self._get_story_url(db,book['calibre_id'])
+        self._set_book_url_and_comment(book,url)
+        #return book
 
     def _set_book_url_and_comment(self,book,url):
         if not url:
@@ -722,11 +947,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             if db.has_format(book_id,'EPUB',index_is_id=True):
                 existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
                 mi = get_metadata(existingepub,'EPUB')
-                #print("mi:%s"%mi)
                 identifiers = mi.get_identifiers()
                 if 'url' in identifiers:
                     #print("url from epub:"+identifiers['url'].replace('|',':'))
                     return identifiers['url'].replace('|',':')
+                # look for dc:source
+                return get_dcsource(existingepub)
         return None
 
     def _is_good_downloader_url(self,url):

calibre-plugin/jobs.py

@@ -16,7 +16,6 @@ from StringIO import StringIO
 #from threading import Event
 
 #from calibre.gui2.convert.single import sort_formats_by_preference
-from calibre.utils.config import prefs
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
 from calibre.utils.logging import Log
@@ -45,8 +44,10 @@ def do_download_worker(book_list, options,
     print(options['version'])
     total = 0
     # Queue all the jobs
+    print("Adding jobs for URLs:")
     for book in book_list:
         if book['good']:
+            print("%s"%book['url'])
             total += 1
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
                     'do_download_for_worker',

defaults.ini

@@ -1,4 +1,4 @@
-# Copyright 2011 Fanficdownloader team
+# Copyright 2012 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,6 +37,7 @@ formatext_label:File Extension
 category_label:Category
 genre_label:Genre
 characters_label:Characters
+series_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -61,12 +62,19 @@ authorId_label:Author ID
 extratags_label:Extra Tags
 ## The version of fanficdownloader
 ##
-version_label:FFD Version
+version_label:FFDL Version
 
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+collect_series: true
 
 ## include title page as first page.
 include_titlepage: true
@@ -120,21 +128,55 @@ extratags: FanFiction
 ## Primarily for commandline.
 #slow_down_sleep_time:0.5
 
-## output background color--only used by html and epub (and ignored in
-## epub by many readers).  Must be hex code, # will be added.
-background_color: ffffff
-
 ## For use only with stand-alone CLI version--run a command on the
 ## generated file after it's produced.  All of the titlepage_entries
 ## values are available, plus output_filename.
 #post_process_cmd: addbook -f "${output_filename}" -t "${title}"
 
+## Use regular expressions to find and replace (or remove) metadata.
+## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
+## etc.  See http://docs.python.org/library/re.html (look for re.sub)
+## for regexp details.
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+#replace_metadata:
+# Sci-Fi=>SF
+# Puella Magi Madoka Magica.* => Madoka
+# Comedy=>Humor
+# Crossover: (.*)=>\1
+# (.*)Great(.*)=>\1Moderate\2
+# .*-Centered=>
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css:
+ body { background-color: #%(background_color)s; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
@@ -154,11 +196,44 @@ titlepage_use_table: false
 ## When using tables, make these span both columns.
 wide_titlepage_entries: description, storyUrl, author URL
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css: 
+ body { background-color: #%(background_color)s;
+        text-align: justify;
+        margin: 2%%; }
+ pre { font-size: x-small; }
+ sml { font-size: small; }
+ h1 { text-align: center; }
+ h2 { text-align: center; }
+ h3 { text-align: center; }
+ h4 { text-align: center; }
+ h5 { text-align: center; }
+ h6 { text-align: center; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
 
-
 ## Each site has a section that overrides [defaults] *and* the format
 ## sections test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
@@ -197,6 +272,9 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+## twilighted.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.twiwrite.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -205,6 +283,9 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+## twiwrite.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.whofic.com]
 
 [www.mediaminer.org]
@@ -222,6 +303,9 @@ extratags:
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -262,6 +346,12 @@ output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
 
 [overrides]
 ## It may sometimes be useful to override all of the specific format,

downloader.py

@@ -46,7 +46,6 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     return output_filename
 
 def main():
-
    # read in args, anything starting with -- will be treated as --<varible>=<value>
    usage = "usage: %prog [options] storyurl"
    parser = OptionParser(usage)
@@ -84,14 +83,17 @@ def main():
 
    conflist = []
    homepath = join(expanduser("~"),".fanficdownloader")
+   
    if isfile(join(homepath,"defaults.ini")):
        conflist.append(join(homepath,"defaults.ini"))
-   if isfile(join(homepath,"personal.ini")):
-       conflist.append(join(homepath,"personal.ini"))
    if isfile("defaults.ini"):
        conflist.append("defaults.ini")
+       
+   if isfile(join(homepath,"personal.ini")):
+       conflist.append(join(homepath,"personal.ini"))
    if isfile("personal.ini"):
        conflist.append("personal.ini")
+       
    if options.configfile:
        conflist.extend(options.configfile)
       
@@ -212,4 +214,7 @@ def main():
        print us
    
 if __name__ == "__main__":
+    #import time
+    #start = time.time()
     main()
+    #print("Total time seconds:%f"%(time.time()-start))

editconfig.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -22,7 +22,7 @@
   <body>
     <div id='main' style="width: 80%; margin-left: 10%;">
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
 
       <div style="text-align: center">
@@ -67,7 +67,7 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
 	Copyright &copy; Fanficdownloader team
       </div>
 

example.ini

@@ -1,6 +1,12 @@
 ## This is an example of what your personal configuration might look
 ## like.
 
+[defaults]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Most common, I expect will be using this to save username/passwords
 ## for different sites.
 [www.twilighted.net]

fanficdownloader/adapters/__init__.py

@@ -1,96 +1,97 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os, re, sys, glob, types
-from os.path import dirname, basename, normpath
-import logging
-import urlparse as up
-
-from .. import exceptions as exceptions
-
-## must import each adapter here.
-
-import adapter_test1
-import adapter_fanfictionnet
-import adapter_castlefansorg
-import adapter_fanfictionnet
-import adapter_fictionalleyorg
-import adapter_fictionpresscom
-import adapter_ficwadcom
-import adapter_fimfictionnet
-import adapter_harrypotterfanfictioncom
-import adapter_mediaminerorg
-import adapter_potionsandsnitchesnet
-import adapter_tenhawkpresentscom
-import adapter_adastrafanficcom
-import adapter_thewriterscoffeeshopcom
-import adapter_tthfanficorg
-import adapter_twilightednet
-import adapter_twiwritenet
-import adapter_whoficcom
-import adapter_siyecouk
-
-## This bit of complexity allows adapters to be added by just adding
-## importing.  It eliminates the long if/else clauses we used to need
-## to pick out the adapter.
-    
-## List of registered site adapters.
-__class_list = []
-
-def imports():
-    for name, val in globals().items():
-        if isinstance(val, types.ModuleType):
-            yield val.__name__
-
-for x in imports():
-    if "fanficdownloader.adapters.adapter_" in x:
-        #print x
-        __class_list.append(sys.modules[x].getClass())
-            
-def getAdapter(config,url):
-    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
-    if not fixedurl.startswith("http"):
-        fixedurl = "http://%s"%url
-    ## remove any trailing '#' locations.
-    fixedurl = re.sub(r"#.*$","",fixedurl)
-    
-    ## remove any trailing '&' parameters--?sid=999 will be left.
-    ## that's all that any of the current adapters need or want.
-    fixedurl = re.sub(r"&.*$","",fixedurl)
-    
-    parsedUrl = up.urlparse(fixedurl)
-    domain = parsedUrl.netloc.lower()
-    if( domain != parsedUrl.netloc ):
-        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
-
-    logging.debug("site:"+domain)
-    cls = getClassFor(domain)
-    if not cls:
-        logging.debug("trying site:www."+domain)
-        cls = getClassFor("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
-    if cls:
-        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        return adapter
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getClassFor(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, re, sys, glob, types
+from os.path import dirname, basename, normpath
+import logging
+import urlparse as up
+
+from .. import exceptions as exceptions
+
+## must import each adapter here.
+
+import adapter_test1
+import adapter_fanfictionnet
+import adapter_castlefansorg
+import adapter_fanfictionnet
+import adapter_fictionalleyorg
+import adapter_fictionpresscom
+import adapter_ficwadcom
+import adapter_fimfictionnet
+import adapter_harrypotterfanfictioncom
+import adapter_mediaminerorg
+import adapter_potionsandsnitchesnet
+import adapter_tenhawkpresentscom
+import adapter_adastrafanficcom
+import adapter_thewriterscoffeeshopcom
+import adapter_tthfanficorg
+import adapter_twilightednet
+import adapter_twiwritenet
+import adapter_whoficcom
+import adapter_siyecouk
+import adapter_archiveofourownorg
+
+## This bit of complexity allows adapters to be added by just adding
+## importing.  It eliminates the long if/else clauses we used to need
+## to pick out the adapter.
+    
+## List of registered site adapters.
+__class_list = []
+
+def imports():
+    for name, val in globals().items():
+        if isinstance(val, types.ModuleType):
+            yield val.__name__
+
+for x in imports():
+    if "fanficdownloader.adapters.adapter_" in x:
+        #print x
+        __class_list.append(sys.modules[x].getClass())
+            
+def getAdapter(config,url):
+    ## fix up leading protocol.
+    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
+    if not fixedurl.startswith("http"):
+        fixedurl = "http://%s"%url
+    ## remove any trailing '#' locations.
+    fixedurl = re.sub(r"#.*$","",fixedurl)
+    
+    ## remove any trailing '&' parameters--?sid=999 will be left.
+    ## that's all that any of the current adapters need or want.
+    fixedurl = re.sub(r"&.*$","",fixedurl)
+    
+    parsedUrl = up.urlparse(fixedurl)
+    domain = parsedUrl.netloc.lower()
+    if( domain != parsedUrl.netloc ):
+        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
+
+    logging.debug("site:"+domain)
+    cls = getClassFor(domain)
+    if not cls:
+        logging.debug("trying site:www."+domain)
+        cls = getClassFor("www."+domain)
+        fixedurl = fixedurl.replace("http://","http://www.")
+    if cls:
+        adapter = cls(config,fixedurl) # raises InvalidStoryURL
+        return adapter
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getClassFor(domain):
+    for cls in __class_list:
+        if cls.matchesSite(domain):
+            return cls

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -79,6 +79,11 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
         if "Content is only suitable for mature adults. May contain explicit language and adult themes. Equivalent of NC-17." in data:
             raise exceptions.AdultCheckRequired(self.url)
             
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
@@ -142,6 +147,12 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 genrestext = [genre.string for genre in genres]
@@ -163,19 +174,45 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.strip(), "%m/%d/%Y"))
+                self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
             
             if 'Updated' in label:
                 # there's a stray [ at the end.
                 #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%m/%d/%Y"))
+                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
 
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+
+def getClass():
+    return ArchiveOfOurOwnOrgAdapter
+
+
+class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+							   
+							   
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ao3')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%b-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.archiveofourown.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/works/123456"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            addurl = "?view_adult=true"
+        else:
+            addurl=""
+
+        meta = self.url+addurl
+        url = self.url+'/navigate'+addurl
+        logging.debug("URL: "+meta)
+
+        try:
+            data = self._fetchUrl(url)
+            meta = self._fetchUrl(meta)
+
+            if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        metasoup = bs.BeautifulSoup(meta)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r"^/works/\w+"))
+        self.story.setMetadata('title',a.string)
+		
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.text)
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
+        self.story.setMetadata('numChapters',len(chapters))
+        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        for x in range(0,len(chapters)):
+            # just in case there's tags, like <i> in chapter titles.
+            chapter=chapters[x]
+            if len(chapters)==1:
+                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
+            else:
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
+
+
+
+        a = metasoup.find('blockquote',{'class':'userstuff'})
+        if a != None:
+            self.story.setMetadata('description',a.text)
+		
+        a = metasoup.find('dd',{'class':"rating tags"})
+        if a != None:
+            self.story.setMetadata('rating',stripHTML(a.text))
+		
+        a = metasoup.find('dd',{'class':"fandom tags"})
+        fandoms = a.findAll('a',{'class':"tag"})
+        fandomstext = [fandom.string for fandom in fandoms]
+        for fandom in fandomstext:
+            self.story.addToList('category',fandom.string)
+		
+        a = metasoup.find('dd',{'class':"warning tags"})
+        if a != None:
+            warnings = a.findAll('a',{'class':"tag"})
+            warningstext = [warning.string for warning in warnings]
+            for warning in warningstext:
+                if warning.string == "Author Chose Not To Use Archive Warnings":
+                    warning.string = "No Archive Warnings Apply"
+                if warning.string != "No Archive Warnings Apply":
+                    self.story.addToList('warnings',warning.string)
+		
+        a = metasoup.find('dd',{'class':"freeform tags"})
+        if a != None:
+            genres = a.findAll('a',{'class':"tag"})
+            genrestext = [genre.string for genre in genres]
+            for genre in genrestext:
+                self.story.addToList('genre',genre.string)
+        a = metasoup.find('dd',{'class':"category tags"})
+        if a != None:
+            genres = a.findAll('a',{'class':"tag"})
+            genrestext = [genre.string for genre in genres]
+            for genre in genrestext:
+                if genre != "Gen":
+                    self.story.addToList('genre',genre.string)
+		
+        a = metasoup.find('dd',{'class':"character tags"})
+        if a != None:
+            chars = a.findAll('a',{'class':"tag"})
+            charstext = [char.string for char in chars]
+            for char in charstext:
+                self.story.addToList('characters',char.string)
+        a = metasoup.find('dd',{'class':"relationship tags"})
+        if a != None:
+            chars = a.findAll('a',{'class':"tag"})
+            charstext = [char.string for char in chars]
+            for char in charstext:
+                self.story.addToList('characters',char.string)
+		
+
+        stats = metasoup.find('dl',{'class':'stats'})
+        dt = stats.findAll('dt')
+        dd = stats.findAll('dd')
+        for x in range(0,len(dt)):
+            label = dt[x].text
+            value = dd[x].text
+
+            if 'Words:' in label:
+                self.story.setMetadata('numWords', value)
+				
+            if 'Chapters:' in label:
+                if value.split('/')[0] == value.split('/')[1]:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+				
+            if 'Completed' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+		
+        try:
+            # Find Series name from series URL.
+            a = metasoup.find('dd',{'class':"series"})
+            b = a.find('a', href=re.compile(r"/series/\d+"))
+            series_name = b.string
+            series_url = 'http://'+self.host+'/fanfic/'+b['href']
+            series_index = int(a.text.split(' ')[1])
+            self.setSeries(series_name, series_index)
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr'))
+		
+        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
+        if headnotes != None:
+            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+            if headnotes != None:
+                chapter.append(bs.BeautifulSoup("<b>Author's Note:</b>"))
+                chapter.append(headnotes)
+        
+        chapsumm = soup.find('div', {'id' : "summary"})
+        if chapsumm != None:
+            chapsumm = chapsumm.find('blockquote')
+            chapter.append(bs.BeautifulSoup("<b>Summary for the Chapter:</b>"))
+            chapter.append(chapsumm)
+        chapnotes = soup.find('div', {'id' : "notes"})
+        if chapnotes != None:
+            chapnotes = chapnotes.find('blockquote')
+            if chapnotes != None:
+                chapter.append(bs.BeautifulSoup("<b>Notes for the Chapter:</b>"))
+                chapter.append(chapnotes)
+		
+        text = soup.find('div', {'class' : "userstuff module"})
+        chtext = text.find('h3', {'class' : "landmark heading"})
+        if chtext:
+            chtext.extract()
+        chapter.append(text)
+		
+        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+        if chapfoot != None:
+            chapfoot = chapfoot.find('blockquote')
+            chapter.append(bs.BeautifulSoup("<b>Notes for the Chapter:</b>"))
+            chapter.append(chapfoot)
+		
+        footnotes = soup.find('div', {'id' : "work_endnotes"})
+        if footnotes != None:
+            footnotes = footnotes.find('blockquote')
+            chapter.append(bs.BeautifulSoup("<b>Author's Note:</b>"))
+            chapter.append(footnotes)
+			
+        if None == soup:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return utf8FromSoup(chapter)

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -232,6 +232,12 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             ## Not all sites use Genre, but there's no harm to
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
@@ -266,6 +272,26 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfic/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -70,7 +70,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         url = self.origurl
         logging.debug("URL: "+url)
-        
+
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
             data = self._fetchUrl(url)
@@ -86,7 +86,29 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             
         if "Chapter not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
-            
+
+        try:
+            # rather nasty way to check for a newer chapter.  ffnet has a
+            # tendency to send out update notices in email before all
+            # their servers are showing the update on the first chapter.
+            try:
+                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+            # get chapter part of url.
+            except:
+                chapcount = 1
+            chapter = url.split('/',)[5]
+            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                           self.story.getMetadata('storyId'),
+                                           chapcount+1)
+            print('=Trying newer chapter: %s' % tryurl)
+            newdata = self._fetchUrl(tryurl)
+            if "Chapter not found. Please check to see you are not using an outdated url." \
+                    not in newdata:
+                print('=======Found newer chapter: %s' % tryurl)
+                soup = bs.BeautifulSoup(newdata)
+        except:
+            pass
+        
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -175,6 +197,12 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         a = soup.find('a', href='http://www.fictionratings.com/')
         self.story.setMetadata('rating',a.string)
 
+        # used below to get correct characters.
+        metatext = a.findNext(text=re.compile(r' - Reviews:'))
+        if metatext == None: # indicates there's no Reviews, look for id: instead.
+            metatext = a.findNext(text=re.compile(r' - id:'))
+        #print("========= metatext:\n%s"%metatext)
+
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
         if 'Complete' in a.findNext(text=re.compile(r'id:'+self.story.getMetadata('storyId'))):
@@ -183,22 +211,36 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             self.story.setMetadata('status', 'In-Progress')
 
         # Parse genre(s) from <meta name="description" content="..."
-        # <meta name="description" content="Chapter 1 of a Harry Potter  - Family/Friendship fanfiction. Dudley Dursley would be the first to say he lived a very normal life. But what happens when he gets invited to his cousin Harry Potter's wedding? Will Dudley get the courage to apologize for the torture he caused all those years ago? Harry/Ginny story..">
-        # <meta name="description" content="A Gundam Wing/AC and Gundam Seed  - Romance/Sci-Fi crossover fanfiction  with characters:  & Kira Y.. Story summary: One-Shoot dividido en dos partes. Kira va en camino a rescatar a Lacus, pero él no es el unico. Dos personajes de diferentes universos Gundams. SEED vs ZERO.">
-        # <meta name="description" content="Chapter 1 of a Alvin and the chipmunks and Alpha and Omega  crossover fanfiction  with characters: Alvin S. & Humphrey. You'll just have to read to find out... No Flames Plesae... and tell me what you want to see by PM'ing me....">
-        # genre is after first -, but before first 'fanfiction'.
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?)) (?:crossover )?fanfiction",
+        # <meta name="description" content="A Transformers/Beast Wars  - Humor fanfiction with characters Prowl & Sideswipe. Story summary: Sideswipe is bored. Prowl appears to be so, too  or at least, Sideswipe thinks he looks bored . So Sideswipe entertains them. After all, what's more fun than a race? Song-fic.">
+        # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
+        # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
+        # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
+        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?:[ ]+with characters (?P<char1>.*?\.?)(?: & (?P<char2>.*?\.?))?\. )?",
                      soup.find('meta',{'name':'description'})['content'])
         if m != None:
             genres=m.group('genres')
-            # Hurt/Comfort is one genre.
-            genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
-            for g in genres.split('/'):
-                self.story.addToList('genre',g)
+            if genres != None:
+                # Hurt/Comfort is one genre.
+                genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
+                for g in genres.split('/'):
+                    self.story.addToList('genre',g)
+
+            if m.group('char1') != None:
+                # At this point we've proven that there's character(s)
+                # We can't reliably parse characters out of meta name="description".
+                # There's no way to tell that "with characters Ichigo K. & Neliel T. O./Nel. " ends at "Nel.", not "T."
+                # But we can pull them from the reviewstext line, now that we know about existance of chars.
+                # reviewstext can take form of:
+                # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
+                # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
+                mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
+                              metatext)
+                chars = mc.group("chars")
+                for c in chars.split(' & '):
+                    self.story.addToList('characters',c)
         
         return
 
-
     def getChapterText(self, url):
         logging.debug('Getting chapter text from: %s' % url)
         time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -152,6 +152,12 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             for g in m.group(1).split(','):
                 self.story.addToList('genre',g)
         
+        m = re.match(r".*?Characters: (.*?) -.*?",metastr)
+        if m:
+            for g in m.group(1).split(','):
+                if g:
+                    self.story.addToList('characters',g)
+        
         m = re.match(r".*?Published: ([0-9/]+?) -.*?",metastr)
         if m:
             self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y/%m/%d"))
@@ -195,9 +201,6 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
         logging.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet tends to fail more if hit too fast.
-                        ## This is in additional to what ever the
-                        ## slow_down_sleep_time setting is.
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
 

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -20,6 +20,7 @@ import logging
 import re
 import urllib2
 import cookielib as cl
+import datetime
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
@@ -107,7 +108,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         for character in [character_icon['title'] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
             self.story.addToList("characters", character)
         for category in [category.text for category in soup.find("div", {"class":"categories"}).findAll("a")]:
-            self.story.addToList("category", category)
+            self.story.addToList("genre", category)
         self.story.addToList("category", "My Little Pony")
         
         
@@ -142,18 +143,27 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             pass
         self.story.setMetadata('description', description_soup.text)
         
-        # Unfortunately, nowhere on the page is the year mentioned. Because we would much rather update the story needlessly
-        # than miss an update, we hardcode the year of creation and update to be 2011.
+        # Unfortunately, nowhere on the page is the year mentioned.
+        # Best effort to deal with this:
+        # Use this year, if that's a date in the future, subtract one year.
+        # Their earliest story is Jun, so they'll probably change the date
+        # around then.
 
+        now = datetime.datetime.now()
+        
         # Get the date of creation from the first chapter
         datePublished_text = chapterDates[0]
         day, month = datePublished_text.split()
         day = re.sub(r"[^\d.]+", '', day)
-        datePublished = makeDate("2011"+month+day, "%Y%b%d")
+        datePublished = makeDate("%s%s%s"%(now.year,month,day), "%Y%b%d")
+        if datePublished > now :
+            datePublished = datePublished.replace(year=now.year-1)
         self.story.setMetadata("datePublished", datePublished)
         dateUpdated_soup = bs.BeautifulSoup(data).find("div", {"class":"calendar"})
         dateUpdated_soup.find('span').extract()
-        dateUpdated = makeDate("2011"+dateUpdated_soup.text, "%Y%b%d")
+        dateUpdated = makeDate("%s%s"%(now.year,dateUpdated_soup.text), "%Y%b%d")
+        if dateUpdated > now :
+            dateUpdated = datePublished.replace(year=now.year-1)
         self.story.setMetadata("dateUpdated", dateUpdated)
         
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -159,6 +159,11 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
                     for g in m.group(1).split(','):
                         self.story.addToList('genre',g)
         
+                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+                if m:
+                    for g in m.group(1).split(','):
+                        self.story.addToList('characters',g)
+        
                 m = re.match(r".*?Warnings: (.+).*?",metastr)
                 if m:
                     for w in m.group(1).split(','):

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -145,6 +145,16 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    if char == "!Snape and Harry (required)":
+                        self.story.addToList('characters',"Snape")
+                        self.story.addToList('characters',"Harry")
+                    else:
+                        self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                 genrestext = [genre.string for genre in genres]
@@ -166,7 +176,27 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfiction/'+a['href']
 
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -81,7 +81,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://")+r"(www\.)?"+re.escape("siye.co.uk/siye/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://")+r"(www\.)?siye\.co\.uk/(siye/)?"+re.escape("viewstory.php?sid=")+r"\d+$"
 
     # ## Login seems to be reasonably standard across eFiction sites.
     # def needToLoginCheck(self, data):
@@ -203,7 +203,10 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             if part.startswith("Characters:"):
                 part = part[part.find(':')+1:]
                 for item in part.split(','):
-                    if item.strip() != "None":
+                    if item.strip() == "Harry/Ginny":
+                        self.story.addToList('characters',"Harry")
+                        self.story.addToList('characters',"Ginny")
+                    elif item.strip() not in ("None","All"):
                         self.story.addToList('characters',item)
 
             if part.startswith("Genres:"):
@@ -225,15 +228,13 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             if part.startswith("Summary:"):
                 part = part[part.find(':')+1:]
                 self.story.setMetadata('description',part)
-
-        
-
+                
         # want to get the next tr of the table.
         #print("%s"%titlea.parent.parent.findNextSibling('tr'))
             
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
-        moremeta = stripHTML(titlea.parent.parent.findNextSibling('tr'))
+        moremeta = stripHTML(titlea.parent.parent.parent.find('div',{'class':'desc'}))
         for part in moremeta.replace(' - ','\n').split('\n'):
             #print("part:%s"%part)
             try:
@@ -256,7 +257,25 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             if name == 'Words':
                 self.story.setMetadata('numWords', value)
 
-        
+        try:
+            # Find Series name from series URL.
+            a = titlea.findPrevious('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass        
 
     # grab the text for an individual chapter.
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -178,6 +178,12 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                 genrestext = [genre.string for genre in genres]
@@ -199,6 +205,26 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
 
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_test1.py

@@ -84,7 +84,15 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         else:
             self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
         self.story.setMetadata('numWords','123456')
-        self.story.setMetadata('status','In-Completed')
+
+        idnum = int(self.story.getMetadata('storyId'))
+        if idnum % 2 == 1:
+            self.story.setMetadata('status','In-Progress')
+        else:
+            self.story.setMetadata('status','Completed')
+            
+        self.setSeries('The Great Test',idnum)
+        
         self.story.setMetadata('rating','Tweenie')
         
         self.story.setMetadata('authorId','98765')
@@ -96,7 +104,8 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('category','Harry Potter')
         self.story.addToList('category','Furbie')
         self.story.addToList('category','Crossover')
-        
+        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
+        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','SF')
         self.story.addToList('genre','Noir')

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -123,6 +123,11 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):] 
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
@@ -175,6 +180,12 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                 genrestext = [genre.string for genre in genres]
@@ -196,12 +207,37 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/library/'+a['href']
 
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -54,13 +54,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         return 'www.tthfanfic.org'
 
     def getSiteExampleURLs(self):
-        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm ttp://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
+        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm http://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
 
     # http://www.tthfanfic.org/T-526321777848988007890480555880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm
     # http://www.tthfanfic.org/Story-5583
     # http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm
+    # http://www.tthfanfic.org/story.php?no=26093
     def getSiteURLPattern(self):
-        return r"http://www.tthfanfic.org/(T-\d+/)?Story-(?P<id>\d+)(-\d+)?(/.*)?$"
+        return r"http://www.tthfanfic.org(/(T-\d+/)?Story-|/story.php\?no=)(?P<id>\d+)(-\d+)?(/.*)?$"
 
     # tth won't send you future updates if you aren't 'caught up'
     # on the story.  Login isn't required for F21, but logging in will
@@ -151,13 +152,13 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
 
         try:
             # going to pull part of the meta data from author list page.
-            logging.debug("author URL: "+self.story.getMetadata('authorUrl'))
+            logging.debug("**AUTHOR** URL: "+self.story.getMetadata('authorUrl'))
             authordata = self._fetchUrl(self.story.getMetadata('authorUrl'))
             authorsoup = bs.BeautifulSoup(authordata)
             # author can have several pages, scan until we find it.
             while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
                 nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
-                logging.debug("author nextpage URL: "+nextpage)
+                logging.debug("**AUTHOR** nextpage URL: "+nextpage)
                 authordata = self._fetchUrl(nextpage)
                 authorsoup = bs.BeautifulSoup(authordata)
         except urllib2.HTTPError, e:
@@ -213,6 +214,12 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                 self.chapterUrls.append((stripHTML(o),url))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        pseries = soup.find('p', {'style':'margin-top:0px'})
+        m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
+                     pseries.text)
+        if m:
+            self.setSeries(m.group('series'),m.group('num'))
         
         return
 

fanficdownloader/adapters/adapter_twilightednet.py

@@ -118,6 +118,12 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        # twilighted isn't writing <body> ??? wtf?
+        data = "<html><body>"+data[data.index("</head>"):] 
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
@@ -170,6 +176,12 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             ## twilighted.net doesn't use genre.
             # if 'Genre' in label:
             #     genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
@@ -192,12 +204,38 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+            
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+        
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
 
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        # twilighted isn't writing <body> ??? wtf?
+        data = "<html><body>"+data[data.index("</head>"):] 
+        
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -117,10 +117,13 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
 
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
+
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):] 
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
-        data = data[data.index("<body"):] # desperate--strip before <body
-        # in calibre plugin only, soup wasn't parsing the html properly.
         soup = bs.BeautifulSoup(data)
 
         ## Title
@@ -180,6 +183,12 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
                 genrestext = [genre.string for genre in genres]
@@ -208,14 +217,36 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                 value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
 
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
-        data = data[data.index("<body"):] # desperate--strip before <body
-        # in calibre plugin only, soup wasn't parsing the html properly.
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
         soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         

fanficdownloader/adapters/adapter_whoficcom.py

@@ -158,6 +158,15 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
         for g in genre.split(r', '):
             self.story.addToList('genre',g)
 
+        # line 3 is characters.
+        chars = metadatachunks[3]
+        charsearch="<i>Characters:</i>"
+        if charsearch in chars:
+            chars = chars[metadatachunks[3].index(charsearch)+len(charsearch):]
+            for c in chars.split(','):
+                if c.strip() != u'None':
+                    self.story.addToList('characters',c)
+            
         # the next line is stuff with ' - ' separators *and* names--with tags.
         moremeta = metadatachunks[5]
         moremeta = re.sub(r'<[^>]+>','',moremeta) # strip tags.
@@ -180,6 +189,26 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
             if name == 'Word Count':
                 self.story.setMetadata('numWords', value)
 
+        try:
+            # Find Series name from series URL.
+            a = metadata.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/base_adapter.py

@@ -40,6 +40,7 @@ except:
     #logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine")
 
 from ..story import Story
+from ..gziphttp import GZipProcessor
 from ..configurable import Configurable
 from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML
 from ..exceptions import InvalidStoryURL
@@ -72,7 +73,7 @@ class BaseSiteAdapter(Configurable):
         self.password = ""
         self.is_adult=False
         
-        self.opener = u2.build_opener(u2.HTTPCookieProcessor())
+        self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         self.storyDone = False
         self.metadataDone = False
         self.story = Story()
@@ -228,8 +229,34 @@ class BaseSiteAdapter(Configurable):
     def getChapterText(self, url):
         "Needs to be overriden in each adapter class."
         pass
-        
+
+    # Just for series, in case we choose to change how it's stored or represented later.
+    def setSeries(self,name,num):
+        if self.getConfig('collect_series'):
+            self.story.setMetadata('series','%s [%s]'%(name, num))
+
+fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"05",
+           "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
+           "November":"11", "December":"12" }
+
 def makeDate(string,format):
+    # Surprise!  Abstracting this turned out to be more useful than
+    # just saving bytes.
+
+    # fudge english month names for people who's locale is set to
+    # non-english.  All our current sites date in english, even if
+    # there's non-english content.
+    do_abbrev = "%b" in format
+        
+    if "%B" in format or do_abbrev:
+        format = format.replace("%B","%m").replace("%b","%m")
+        for (name,num) in fullmon.items():
+            if do_abbrev:
+                name = name[:3] # first three for abbrev
+            if name in string:
+                string = string.replace(name,num)
+                break
+            
     return datetime.datetime.strptime(string,format)
 
 acceptable_attributes = ['href','name']

fanficdownloader/gziphttp.py

@@ -0,0 +1,38 @@
+## Borrowed from http://techknack.net/python-urllib2-handlers/
+
+import urllib2
+from gzip import GzipFile
+from StringIO import StringIO
+
+class GZipProcessor(urllib2.BaseHandler):
+    """A handler to add gzip capabilities to urllib2 requests
+    """
+    def http_request(self, req):
+        req.add_header("Accept-Encoding", "gzip")
+        return req
+    https_request = http_request
+
+    def http_response(self, req, resp):
+        #print("Content-Encoding:%s"%resp.headers.get("Content-Encoding"))
+        if resp.headers.get("Content-Encoding") == "gzip":
+            gz = GzipFile(
+                        fileobj=StringIO(resp.read()),
+                        mode="r"
+                      )
+#            resp.read = gz.read
+#            resp.readlines = gz.readlines
+#            resp.readline = gz.readline
+#            resp.next = gz.next
+            old_resp = resp
+            resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        return resp
+    https_response = http_response
+
+# brave new world - 1:30 w/o, 1:10 with?  40 chapters, so 20s from sleeps.
+# with gzip, no sleep: 47.469
+# w/o gzip, no sleep: 47.736
+
+# I Am What I Am 67 chapters
+# w/o gzip: 57.168
+# w/ gzip: 40.692

fanficdownloader/story.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-import os
+import os, re
 
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 
@@ -25,7 +25,8 @@ class Story:
         try:
             self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
         except:
-            self.metadata = {'version':'4.2'}
+            self.metadata = {'version':'4.3'}
+        self.replacements = []
         self.chapters = [] # chapters will be tuples of (title,html)
         self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
 
@@ -36,6 +37,12 @@ class Story:
     def getMetadataRaw(self,key):
         if self.metadata.has_key(key):
             return self.metadata[key]
+
+    def doReplacments(self,value):
+        for (p,v) in self.replacements:
+            if (isinstance(value,str) or isinstance(value,unicode)) and re.match(p,value):
+                value = re.sub(p,v,value)                
+        return value;
         
     def getMetadata(self, key, removeallentities=False):
         value = None
@@ -50,21 +57,22 @@ class Story:
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
                  value = value.strftime("%Y-%m-%d")
-                 
+
+        value=self.doReplacments(value)
         if removeallentities and value != None:
             return removeAllEntities(value)
         else:
             return value
 
-    def getAllMetadata(self):
+    def getAllMetadata(self, removeallentities=False):
         '''
         All single value *and* list value metadata as strings.
         '''
         allmetadata = {}
         for k in self.metadata.keys():
-            allmetadata[k] = self.getMetadata(k)
+            allmetadata[k] = self.getMetadata(k, removeallentities)
         for l in self.listables.keys():
-            allmetadata[l] = self.getMetadata(l)
+            allmetadata[l] = self.getMetadata(l, removeallentities)
 
         return allmetadata
         
@@ -81,10 +89,14 @@ class Story:
     def getList(self,listname):
         if not self.listables.has_key(listname):
             return []
-        return self.listables[listname]
+        return filter( lambda x : x!=None and x!='' ,
+                       map(self.doReplacments,self.listables[listname]) )
 
     def getLists(self):
-        return self.listables
+        lsts = {}
+        for ln in self.listables.keys():
+            lsts[ln] = self.getList(ln)
+        return lsts
     
     def addChapter(self, title, html):
         self.chapters.append( (title,html) )
@@ -96,6 +108,12 @@ class Story:
     def __str__(self):
         return "Metadata: " +str(self.metadata) + "\nListables: " +str(self.listables) #+ "\nChapters: "+str(self.chapters)
 
+    def setReplace(self,replace):
+        for line in replace.splitlines():
+            if "=>" in line:
+                print("line:%s"%line)
+                self.replacements.append(map( lambda x: x.strip(), line.split("=>") ))
+    
 def commaGroups(s):
     groups = []
     while s and s[-1].isdigit():

fanficdownloader/writers/base_writer.py

@@ -46,10 +46,14 @@ class BaseStoryWriter(Configurable):
         
         self.adapter = adapter
         self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
+        
+        self.story.setReplace(self.getConfig('replace_metadata'))
+        
         self.validEntries = [
             'category',
             'genre',
             'characters',
+            'series',
             'status',
             'datePublished',
             'dateUpdated',
@@ -77,6 +81,7 @@ class BaseStoryWriter(Configurable):
             'category':'Category',
             'genre':'Genre',
             'status':'Status',
+            'series':'Series',
             'characters':'Characters',
             'datePublished':'Published',
             'dateUpdated':'Updated',
@@ -97,10 +102,13 @@ class BaseStoryWriter(Configurable):
             'formatname':'File Format',
             'formatext':'File Extension',
             'siteabbrev':'Site Abbrev',
-            'version':'FFD Version'
+            'version':'FFDL Version'
             }
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())
+        
+        for tag in self.getConfigList("extratags"):
+            self.story.addToList("extratags",tag)
 
     def getMetadata(self,key):
         return stripHTML(self.story.getMetadata(key))
@@ -184,13 +192,17 @@ class BaseStoryWriter(Configurable):
 
     # if no outstream is given, write to file.
     def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False):
-        for tag in self.getConfigList("extratags"):
-            self.story.addToList("extratags",tag)
 
         self.metaonly = metaonly
         if outfilename == None:
             outfilename=self.getOutputFileName()
 
+        # minor cheat, tucking css into metadata.
+        if self.getConfig("output_css"):
+            self.story.metadata["output_css"] = self.getConfig("output_css")
+        else:
+            self.story.metadata["output_css"] = ''
+            
         if not outstream:
             close=True
             logging.debug("Save directly to file: %s" % outfilename)
@@ -266,9 +278,6 @@ class BaseStoryWriter(Configurable):
             if name in self.getConfigList("include_subject_tags"):
                 for tag in lst:
                     subjectset.add(tag)
-                    
-        for tag in self.getConfigList("extratags"):
-            subjectset.add(tag)
 
         return list(subjectset)
             

fanficdownloader/writers/writer_epub.py

@@ -41,32 +41,7 @@ class EpubWriter(BaseStoryWriter):
     def __init__(self, config, story):
         BaseStoryWriter.__init__(self, config, story)
 
-        self.EPUB_CSS = string.Template('''
-body { margin: 2%;
-       text-align: justify;
-       background-color: #${background_color}; }
-pre { font-size: x-small; }
-sml { font-size: small; }
-h1 { text-align: center; }
-h2 { text-align: center; }
-h3 { text-align: center; }
-h4 { text-align: center; }
-h5 { text-align: center; }
-h6 { text-align: center; }
-.CI {
-    text-align:center;
-    margin-top:0px;
-    margin-bottom:0px;
-    padding:0px;
-    }
-.center   {text-align: center;}
-.cover    {text-align: center;}
-.full     {width: 100%; }
-.quarter  {width: 25%; }
-.smcap    {font-variant: small-caps;}
-.u        {text-decoration: underline;}
-.bold     {font-weight: bold;}
-''')
+        self.EPUB_CSS = string.Template('''${output_css}''')
         
         self.EPUB_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -361,7 +336,7 @@ h6 { text-align: center; }
         del tocncxdom
 
         # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute({"background_color":self.getConfig("background_color")})) 
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata)) 
 
         # write title page.
         if self.getConfig("titlepage_use_table"):

fanficdownloader/writers/writer_html.py

@@ -39,20 +39,7 @@ class HTMLWriter(BaseStoryWriter):
 <head>
 <title>${title} by ${author}</title>
 <style type="text/css">
-body { background-color: #${background_color}; }
-.CI {
-    text-align:center;
-    margin-top:0px;
-    margin-bottom:0px;
-    padding:0px;
-    }
-.center   {text-align: center;}
-.cover    {text-align: center;}
-.full     {width: 100%; }
-.quarter  {width: 25%; }
-.smcap    {font-variant: small-caps;}
-.u        {text-decoration: underline;}
-.bold     {font-weight: bold;}
+${output_css}
 </style>
 </head>
 <body>
@@ -95,9 +82,6 @@ body { background-color: #${background_color}; }
 
     def writeStoryImpl(self, out):
 
-        # minor cheat, tucking bg into metadata.
-        if self.getConfig("background_color"):
-            self.story.metadata["background_color"] = self.getConfig("background_color")
         self._write(out,self.HTML_FILE_START.substitute(self.story.metadata))
 
         self.writeTitlePage(out,

fanficdownloader/writers/writer_mobi.py

@@ -41,7 +41,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -64,7 +63,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -91,7 +89,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <div>
@@ -113,7 +110,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${chapter}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3>${chapter}</h3>

index-ajax.html

@@ -4,7 +4,7 @@
 		<link href="css/index.css" rel="stylesheet" type="text/css">
 		<link type="text/css" href="http://jqueryui.com/latest/themes/base/ui.all.css" rel="stylesheet" />
 	 
-		<title>Fanfiction Downloader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
+		<title>FanFictionDownLoader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
 		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
 		<script src="/js/jquery-1.3.2.js"></script>
 		<script src="/js/fdownloader.js"></script>
@@ -16,7 +16,7 @@
 	<body>
 		<div id='main'>
 			<h1>
-				<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+				<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
 			</h1>
 				
 <!--			<form action="/fdown" method="post"> -->
@@ -91,8 +91,8 @@
 					<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 					alt="Powered by Google App Engine" />
 					<br/><br/>
-					FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-					Copyright &copy; <a href="http://twitter.com/sigizmund">Roman Kirillov</a>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; Fanficdownloader team
 				</div>
 <!--			</form> -->
 		</div>

index.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -26,7 +26,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a> <g:plusone size="medium"></g:plusone>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a> <g:plusone size="medium"></g:plusone>
       </h1>
 
       <div style="text-align: center">
@@ -50,27 +50,27 @@
       <form action="/fdown" method="post">
 	<div id='urlbox'>
 	  <div id='greeting'>
-	    <p>Hi, {{ nickname }}! This is a fan fiction downloader, which makes reading stories from various websites
+	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>This is the Official Multithreading Version</h3>
+	  <h3>Support for Custom CSS</h3>
 	  <p>
-	    This version of the application uses Python 2.7 and
-	    multithreading to try and reduce our usage.  Google
-	    considers Python 2.7 Experimental still, so there may be issues.
+	    The CSS included in the HTML and EPUB output formats is now a customizable parameter.
+	  </p>
+	  <h3>Support for Custom Replacement of Metadata</h3>
+	  <p>
+	    There's now a customizable parameter to include a list of regular expressions to replace metadata as you see fit.
 	  </p>
 	  <p>
-	    <b>Changed Site</b><br />
-	    fanfic.castletv.net changed to castlefans.org.
+	    Examples of how to use both new features can be found in the
+	    <a href="http://www.mobileread.com/forums/showthread.php?p=1962034#post1962034">plugin forum</a>.
 	  </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
-	    the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
-	      Downloader Google Group</a>.  The
-	    <a href="http://4-0-7.fanfictiondownloader.appspot.com">Previous
-	      Version</a> is also available for you to use if necessary.
+	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
+	    <a href="http://4-3-1.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -106,14 +106,33 @@
       <div id='urlbox'>
 	<div id='greeting'>
 	  <p>
-	    This is a fan fiction downloader, which makes reading stories from various websites much easier. Before you
-	    can start downloading fanfics, you need to login, so downloader can remember your fanfics and store them.
+	    This is a FanFictionDownLoader, which makes reading stories from various websites much easier. Before you
+	    can start downloading fanfics, you need to login, so FanFictionDownLoader can remember your fanfics and store them.
 	  </p>
 	  <p><a href="{{ login_url }}">Login using Google account</a></p>
 	</div>
       </div>
       {% endif %}
       
+      <div id='typebox'>
+	  <p>
+	    <b>FanFictionDownLoader calibre Plugin</b>
+	    <br /><br />
+	    
+	    There's now a version of this downloader that runs
+	    entirely inside the
+	    popular <a href="http://calibre-ebook.com/">calibre</a>
+	    ebook management package as a plugin.
+	    
+	    <br /><br />
+	    
+	    Once you have calibre installed and running, inside
+	    calibre, you can go to 'Get plugins to enhance calibre' or
+	    'Get new plugins' and
+	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
+	    
+	  </p>
+      </div>
       <div id='helpbox'>
 	<dl>
 	  <dt>fictionalley.org</dt>
@@ -214,6 +233,12 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://www.siye.co.uk/siye/viewstory.php?sid=123">http://www.siye.co.uk/siye/viewstory.php?sid=123</a>.
 	  </dd>
+	  <dt>archiveofourown.org</dt>
+	  <dd>
+	    Use the URL of the story, or one of it's chapters, such as
+	    <br /><a href="http://archiveofourown.org/works/76366">http://archiveofourown.org/works/76366</a>.
+	    <br /><a href="http://archiveofourown.org/works/76366/chapters/101584">http://archiveofourown.org/works/76366/chapters/101584</a>.
+	  </dd>
 	</dl>	
 
 	
@@ -250,8 +275,8 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; FanFictionDownLoader team
       </div>
 
       <div style="margin-top: 1em; text-align: center'">

login.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Login Needed Fanfiction Downloader</title>
+    <title>Login Needed FanFictionDownLoader</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -22,7 +22,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
 
       <div style="text-align: center">
@@ -88,8 +88,8 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; FanFictionDownLoader team
       </div>
 
       <div style="margin-top: 1em; text-align: center'">

makeplugin.py

@@ -27,7 +27,7 @@ if __name__=="__main__":
     exclude=['*.pyc','*~','*.xcf']
     # from top dir. 'w' for overwrite
     createZipFile(filename,"w",
-                  ['plugin-defaults.ini','example.ini','epubmerge.py','fanficdownloader'],
+                  ['plugin-defaults.ini','plugin-example.ini','epubmerge.py','fanficdownloader'],
                   exclude=exclude)
     #from calibre-plugin dir. 'a' for append
     os.chdir('calibre-plugin')

plugin-defaults.ini

@@ -18,6 +18,11 @@
 ## [defaults] section applies to all formats and sites but may be
 ## overridden at several levels
 
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 ## All available titlepage_entries and the label used for them:
 ## <entryname>_label:<label>
 ## Labels may be customized.
@@ -37,6 +42,7 @@ formatext_label:File Extension
 category_label:Category
 genre_label:Genre
 characters_label:Characters
+series_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -61,12 +67,19 @@ authorId_label:Author ID
 extratags_label:Extra Tags
 ## The version of fanficdownloader
 ##
-version_label:FFD Version
+version_label:FFDL Version
 
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+collect_series: true
 
 ## include title page as first page.
 include_titlepage: true
@@ -92,28 +105,69 @@ extratags: FanFiction
 
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
-## Primarily for commandline.
 #slow_down_sleep_time:0.5
 
 ## output background color--only used by html and epub (and ignored in
 ## epub by many readers).  Must be hex code, # will be added.
 background_color: ffffff
 
+## Use regular expressions to find and replace (or remove) metadata.
+## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
+## etc.  See http://docs.python.org/library/re.html (look for re.sub)
+## for regexp details.
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+#replace_metadata:
+# Sci-Fi=>SF
+# Puella Magi Madoka Magica.* => Madoka
+# Comedy=>Humor
+# Crossover: (.*)=>\1
+# (.*)Great(.*)=>\1Moderate\2
+# .*-Centered=>
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css:
+ body { background-color: #%(background_color)s; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+
+## use \r\n for line endings, the windows convention.  text output only.
+windows_eol: true
+
+[txt]
+## Add URLs since there aren't links.
+titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
 
 [epub]
 
-## epub is already a zip file.
-zip_output: false
-
 ## epub carries the TOC in metadata.
 ## mobi generated from epub will have a TOC at the end.
 include_tocpage: false
@@ -124,6 +178,40 @@ titlepage_use_table: false
 ## When using tables, make these span both columns.
 wide_titlepage_entries: description, storyUrl, author URL
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css: 
+ body { background-color: #%(background_color)s;
+        text-align: justify;
+        margin: 2%%; }
+ pre { font-size: x-small; }
+ sml { font-size: small; }
+ h1 { text-align: center; }
+ h2 { text-align: center; }
+ h3 { text-align: center; }
+ h4 { text-align: center; }
+ h5 { text-align: center; }
+ h6 { text-align: center; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
@@ -167,6 +255,9 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+## twilighted.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.twiwrite.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -175,6 +266,9 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+## twiwrite.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.whofic.com]
 
 [www.mediaminer.org]
@@ -192,6 +286,9 @@ extratags:
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -212,9 +309,6 @@ extratags:
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## fictionally.org storyIds are not unique.  Combine with authorId.
-output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
-
 [www.harrypotterfanfiction.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -232,6 +326,12 @@ output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
 
 [overrides]
 ## It may sometimes be useful to override all of the specific format,

plugin-example.ini

@@ -0,0 +1,74 @@
+## This is an example of what your personal configuration might look
+## like.
+
+[defaults]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+## Turn off in [defaults] or [overrides] to prevent all sites from
+## updating series column.
+## default is true
+#collect_series: false
+
+## Most common, I expect will be using this to save username/passwords
+## for different sites.
+[www.twilighted.net]
+#username:YourPenname
+#password:YourPassword
+## default is false
+#collect_series: true
+
+[www.ficwad.com]
+#username:YourUsername
+#password:YourPassword
+
+[www.twiwrite.net]
+#username:YourName
+#password:yourpassword
+## default is false
+#collect_series: true
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.
+#is_adult:true
+
+[www.thewriterscoffeeshop.com]
+#username:YourName
+#password:yourpassword
+#is_adult:true
+## default is false
+#collect_series: true
+
+[www.fictionalley.org]
+#is_adult:true
+
+[www.harrypotterfanfiction.com]
+#is_adult:true
+
+[www.fimfiction.net]
+#is_adult:true
+
+[www.tthfanfic.org]
+#is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
+
+
+## This section will override anything in the system defaults or other
+## sections here.
+[overrides]
+## default varies by site.  Set true here to force all sites to
+## collect series.
+#collect_series: true

recent.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
+    <title>FanFictionDownLoader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <script type="text/javascript">
 
@@ -21,7 +21,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
       
       <script type="text/javascript"><!--
@@ -45,7 +45,7 @@
 	  <p><a href="/clearrecent">Clear your Recent Downloads List</a></p>
 	</div>
       </div>
-      
+
       <div id='helpbox'>
 	{% for fic in fics %}
 	<p>

status.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>{% if fic.completed %} Finished {% else %} {% if fic.failure %} Failed {% else %} Working... {% endif %} {% endif %} - Fanfiction Downloader</title>
+    <title>{% if fic.completed %} Finished {% else %} {% if fic.failure %} Failed {% else %} Working... {% endif %} {% endif %} - FanFictionDownLoader</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     {% if not fic.completed and not fic.failure %}
@@ -25,7 +25,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
       <div style="text-align: center">
         <script type="text/javascript"><!--
@@ -67,11 +67,12 @@
 	{% endif %}
 	  <p>See your personal list of <a href="/recent">previously downloaded fanfics</a>.</p>
       </div>
+
       <div style='text-align: center'>
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
 	Copyright &copy; Fanficdownloader team
       </div>