Better date kludge for fimfiction.net. They don't give the year.

& plugin.  Save plugin bump to 1.3.1.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
-application: ffd-retief-hrd
-version: 4-1-1
+application: fanfictiondownloader
+version: 4-3-1
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -11,7 +11,9 @@ __docformat__ = 'restructuredtext en'
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase
 
-class InterfacePluginDemo(InterfaceActionBase):
+## Apparently the name for this class doesn't matter--it was still
+## 'demo' for the first few versions.
+class FanFictionDownLoaderBase(InterfaceActionBase):
     '''
     This class is a simple wrapper that provides information about the
     actual plugin class. The actual interface plugin class is called
@@ -21,15 +23,13 @@ class InterfacePluginDemo(InterfaceActionBase):
     The reason for having two classes is that it allows the command line
     calibre utilities to run without needing to load the GUI libraries.
     '''
-    name                = 'FanFictionDownLoader Plugin'
+    name                = 'FanFictionDownLoader'
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 0, 2)
+    version             = (1, 3, 3)
     minimum_calibre_version = (0, 8, 30)
 
-#    action_menu_clone_qaction = True
-
     #: This field defines the GUI plugin class that contains all the code
     #: that actually does something. Its format is module_path:class_name
     #: The specified class must be defined in the specified module.
@@ -65,7 +65,7 @@ class InterfacePluginDemo(InterfaceActionBase):
         # GUI libraries to be loaded, which we do not want when using calibre
         # from the command line
         from calibre_plugins.fanfictiondownloader_plugin.config import ConfigWidget
-        return ConfigWidget()
+        return ConfigWidget(self.actual_plugin_)
 
     def save_settings(self, config_widget):
         '''

calibre-plugin/about.txt

@@ -1,11 +1,20 @@
-<p>FanFictionDownLoader Plugin</p>
 <hr />
 
-<p><a href="http://code.google.com/p/fanficdownloader/">http://code.google.com/p/fanficdownloader/</a></p>
+<p>Created by Jim Miller, borrowing heavily from Grant Drake's
+'<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
+'<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
+'<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
+plugins.</p>
 
-<p>Created by Jim Miller, borrowing heavily from
-Kovid Goyal's 'The InterfacePlugin Demo' and
-Grant Drake's 'Count Pages' plugins.</p>
+<p>
+Calibre officially distributes plugins from the mobileread.com forum site.
+The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>
+</p>
 
-<p>Requires calibre >= 0.7.53</p>
+<p> I also monitor the
+<a href="http://groups.google.com/group/fanfic-downloader">general users
+group</a> for the downloader.  That covers the web application and CLI, too.
+</p>
 
+The source for this plugin is available
+<a href="http://code.google.com/p/fanficdownloader/source/checkout">here</a>.

calibre-plugin/common_utils.py

@@ -0,0 +1,447 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import os
+from PyQt4 import QtGui
+from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime)
+from calibre.constants import iswindows
+from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME
+from calibre.gui2.actions import menu_action_unique_name
+from calibre.gui2.keyboard import ShortcutConfig
+from calibre.utils.config import config_dir
+from calibre.utils.date import now, format_date, qt_to_dt, UNDEFINED_DATE
+
+# Global definition of our plugin name. Used for common functions that require this.
+plugin_name = None
+# Global definition of our plugin resources. Used to share between the xxxAction and xxxBase
+# classes if you need any zip images to be displayed on the configuration dialog.
+plugin_icon_resources = {}
+
+
+def set_plugin_icon_resources(name, resources):
+    '''
+    Set our global store of plugin name and icon resources for sharing between
+    the InterfaceAction class which reads them and the ConfigWidget
+    if needed for use on the customization dialog for this plugin.
+    '''
+    global plugin_icon_resources, plugin_name
+    plugin_name = name
+    plugin_icon_resources = resources
+
+
+def get_icon(icon_name):
+    '''
+    Retrieve a QIcon for the named image from the zip file if it exists,
+    or if not then from Calibre's image cache.
+    '''
+    if icon_name:
+        pixmap = get_pixmap(icon_name)
+        if pixmap is None:
+            # Look in Calibre's cache for the icon
+            return QIcon(I(icon_name))
+        else:
+            return QIcon(pixmap)
+    return QIcon()
+
+
+def get_pixmap(icon_name):
+    '''
+    Retrieve a QPixmap for the named image
+    Any icons belonging to the plugin must be prefixed with 'images/'
+    '''
+    global plugin_icon_resources, plugin_name
+
+    if not icon_name.startswith('images/'):
+        # We know this is definitely not an icon belonging to this plugin
+        pixmap = QPixmap()
+        pixmap.load(I(icon_name))
+        return pixmap
+
+    # Check to see whether the icon exists as a Calibre resource
+    # This will enable skinning if the user stores icons within a folder like:
+    # ...\AppData\Roaming\calibre\resources\images\Plugin Name\
+    if plugin_name:
+        local_images_dir = get_local_images_dir(plugin_name)
+        local_image_path = os.path.join(local_images_dir, icon_name.replace('images/', ''))
+        if os.path.exists(local_image_path):
+            pixmap = QPixmap()
+            pixmap.load(local_image_path)
+            return pixmap
+
+    # As we did not find an icon elsewhere, look within our zip resources
+    if icon_name in plugin_icon_resources:
+        pixmap = QPixmap()
+        pixmap.loadFromData(plugin_icon_resources[icon_name])
+        return pixmap
+    return None
+
+
+def get_local_images_dir(subfolder=None):
+    '''
+    Returns a path to the user's local resources/images folder
+    If a subfolder name parameter is specified, appends this to the path
+    '''
+    images_dir = os.path.join(config_dir, 'resources/images')
+    if subfolder:
+        images_dir = os.path.join(images_dir, subfolder)
+    if iswindows:
+        images_dir = os.path.normpath(images_dir)
+    return images_dir
+
+
+def create_menu_item(ia, parent_menu, menu_text, image=None, tooltip=None,
+                     shortcut=(), triggered=None, is_checked=None):
+    '''
+    Create a menu action with the specified criteria and action
+    Note that if no shortcut is specified, will not appear in Preferences->Keyboard
+    This method should only be used for actions which either have no shortcuts,
+    or register their menus only once. Use create_menu_action_unique for all else.
+    '''
+    if shortcut is not None:
+        if len(shortcut) == 0:
+            shortcut = ()
+        else:
+            shortcut = _(shortcut)
+    ac = ia.create_action(spec=(menu_text, None, tooltip, shortcut),
+        attr=menu_text)
+    if image:
+        ac.setIcon(get_icon(image))
+    if triggered is not None:
+        ac.triggered.connect(triggered)
+    if is_checked is not None:
+        ac.setCheckable(True)
+        if is_checked:
+            ac.setChecked(True)
+
+    parent_menu.addAction(ac)
+    return ac
+
+
+def create_menu_action_unique(ia, parent_menu, menu_text, image=None, tooltip=None,
+                       shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
+                       unique_name=None):
+    '''
+    Create a menu action with the specified criteria and action, using the new
+    InterfaceAction.create_menu_action() function which ensures that regardless of
+    whether a shortcut is specified it will appear in Preferences->Keyboard
+    '''
+    orig_shortcut = shortcut
+    kb = ia.gui.keyboard
+    if unique_name is None:
+        unique_name = menu_text
+    if not shortcut == False:
+        full_unique_name = menu_action_unique_name(ia, unique_name)
+        if full_unique_name in kb.shortcuts:
+            shortcut = False
+        else:
+            if shortcut is not None and not shortcut == False:
+                if len(shortcut) == 0:
+                    shortcut = None
+                else:
+                    shortcut = _(shortcut)
+
+    if shortcut_name is None:
+        shortcut_name = menu_text.replace('&','')
+
+    ac = ia.create_menu_action(parent_menu, unique_name, menu_text, icon=None, shortcut=shortcut,
+        description=tooltip, triggered=triggered, shortcut_name=shortcut_name)
+    if shortcut == False and not orig_shortcut == False:
+        if ac.calibre_shortcut_unique_name in ia.gui.keyboard.shortcuts:
+            kb.replace_action(ac.calibre_shortcut_unique_name, ac)
+    if image:
+        ac.setIcon(get_icon(image))
+    if is_checked is not None:
+        ac.setCheckable(True)
+        if is_checked:
+            ac.setChecked(True)
+    return ac
+
+
+def swap_author_names(author):
+    if author.find(',') == -1:
+        return author
+    name_parts = author.strip().partition(',')
+    return name_parts[2].strip() + ' ' + name_parts[0]
+
+
+def get_library_uuid(db):
+    try:
+        library_uuid = db.library_id
+    except:
+        library_uuid = ''
+    return library_uuid
+
+
+class ImageLabel(QLabel):
+
+    def __init__(self, parent, icon_name, size=16):
+        QLabel.__init__(self, parent)
+        pixmap = get_pixmap(icon_name)
+        self.setPixmap(pixmap)
+        self.setMaximumSize(size, size)
+        self.setScaledContents(True)
+
+
+class ImageTitleLayout(QHBoxLayout):
+    '''
+    A reusable layout widget displaying an image followed by a title
+    '''
+    def __init__(self, parent, icon_name, title):
+        QHBoxLayout.__init__(self)
+        title_image_label = QLabel(parent)
+        pixmap = get_pixmap(icon_name)
+        if pixmap is None:
+            pixmap = get_pixmap('library.png')
+            # error_dialog(parent, _('Restart required'),
+            #              _('You must restart Calibre before using this plugin!'), show=True)
+        else:
+            title_image_label.setPixmap(pixmap)
+        title_image_label.setMaximumSize(32, 32)
+        title_image_label.setScaledContents(True)
+        self.addWidget(title_image_label)
+
+        title_font = QFont()
+        title_font.setPointSize(16)
+        shelf_label = QLabel(title, parent)
+        shelf_label.setFont(title_font)
+        self.addWidget(shelf_label)
+        self.insertStretch(-1)
+
+
+class SizePersistedDialog(QDialog):
+    '''
+    This dialog is a base class for any dialogs that want their size/position
+    restored when they are next opened.
+    '''
+    def __init__(self, parent, unique_pref_name):
+        QDialog.__init__(self, parent)
+        self.unique_pref_name = unique_pref_name
+        self.geom = gprefs.get(unique_pref_name, None)
+        self.finished.connect(self.dialog_closing)
+
+    def resize_dialog(self):
+        if self.geom is None:
+            self.resize(self.sizeHint())
+        else:
+            self.restoreGeometry(self.geom)
+
+    def dialog_closing(self, result):
+        geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = geom
+
+
+class ReadOnlyTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, text):
+        if text is None:
+            text = ''
+        QTableWidgetItem.__init__(self, text, QtGui.QTableWidgetItem.UserType)
+        self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+
+
+class RatingTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, rating, is_read_only=False):
+        QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType)
+        self.setData(Qt.DisplayRole, rating)
+        if is_read_only:
+            self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+
+
+class DateTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, date_read, is_read_only=False, default_to_today=False):
+        if date_read == UNDEFINED_DATE and default_to_today:
+            date_read = now()
+        if is_read_only:
+            QTableWidgetItem.__init__(self, format_date(date_read, None), QtGui.QTableWidgetItem.UserType)
+            self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+        else:
+            QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType)
+            self.setData(Qt.DisplayRole, QDateTime(date_read))
+
+
+class NoWheelComboBox(QComboBox):
+
+    def wheelEvent (self, event):
+        # Disable the mouse wheel on top of the combo box changing selection as plays havoc in a grid
+        event.ignore()
+
+
+class CheckableTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, checked=False, is_tristate=False):
+        QTableWidgetItem.__init__(self, '')
+        self.setFlags(Qt.ItemFlags(Qt.ItemIsSelectable | Qt.ItemIsUserCheckable | Qt.ItemIsEnabled ))
+        if is_tristate:
+            self.setFlags(self.flags() | Qt.ItemIsTristate)
+        if checked:
+            self.setCheckState(Qt.Checked)
+        else:
+            if is_tristate and checked is None:
+                self.setCheckState(Qt.PartiallyChecked)
+            else:
+                self.setCheckState(Qt.Unchecked)
+
+    def get_boolean_value(self):
+        '''
+        Return a boolean value indicating whether checkbox is checked
+        If this is a tristate checkbox, a partially checked value is returned as None
+        '''
+        if self.checkState() == Qt.PartiallyChecked:
+            return None
+        else:
+            return self.checkState() == Qt.Checked
+
+
+class TextIconWidgetItem(QTableWidgetItem):
+
+    def __init__(self, text, icon):
+        QTableWidgetItem.__init__(self, text)
+        if icon:
+            self.setIcon(icon)
+
+
+class ReadOnlyTextIconWidgetItem(ReadOnlyTableWidgetItem):
+
+    def __init__(self, text, icon):
+        ReadOnlyTableWidgetItem.__init__(self, text)
+        if icon:
+            self.setIcon(icon)
+
+
+class ReadOnlyLineEdit(QLineEdit):
+
+    def __init__(self, text, parent):
+        if text is None:
+            text = ''
+        QLineEdit.__init__(self, text, parent)
+        self.setEnabled(False)
+
+
+class KeyValueComboBox(QComboBox):
+
+    def __init__(self, parent, values, selected_key):
+        QComboBox.__init__(self, parent)
+        self.values = values
+        self.populate_combo(selected_key)
+
+    def populate_combo(self, selected_key):
+        self.clear()
+        selected_idx = idx = -1
+        for key, value in self.values.iteritems():
+            idx = idx + 1
+            self.addItem(value)
+            if key == selected_key:
+                selected_idx = idx
+        self.setCurrentIndex(selected_idx)
+
+    def selected_key(self):
+        for key, value in self.values.iteritems():
+            if value == unicode(self.currentText()).strip():
+                return key
+
+
+class CustomColumnComboBox(QComboBox):
+
+    def __init__(self, parent, custom_columns, selected_column, initial_items=['']):
+        QComboBox.__init__(self, parent)
+        self.populate_combo(custom_columns, selected_column, initial_items)
+
+    def populate_combo(self, custom_columns, selected_column, initial_items=['']):
+        self.clear()
+        self.column_names = initial_items
+        if len(initial_items) > 0:
+            self.addItems(initial_items)
+        selected_idx = 0
+        for idx, value in enumerate(initial_items):
+            if value == selected_column:
+                selected_idx = idx
+        for key in sorted(custom_columns.keys()):
+            self.column_names.append(key)
+            self.addItem('%s (%s)'%(key, custom_columns[key]['name']))
+            if key == selected_column:
+                selected_idx = len(self.column_names) - 1
+        self.setCurrentIndex(selected_idx)
+
+    def get_selected_column(self):
+        return self.column_names[self.currentIndex()]
+
+
+class KeyboardConfigDialog(SizePersistedDialog):
+    '''
+    This dialog is used to allow editing of keyboard shortcuts.
+    '''
+    def __init__(self, gui, group_name):
+        SizePersistedDialog.__init__(self, gui, 'Keyboard shortcut dialog')
+        self.gui = gui
+        self.setWindowTitle('Keyboard shortcuts')
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+
+        self.keyboard_widget = ShortcutConfig(self)
+        layout.addWidget(self.keyboard_widget)
+        self.group_name = group_name
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.commit)
+        button_box.rejected.connect(self.reject)
+        layout.addWidget(button_box)
+
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.initialize()
+
+    def initialize(self):
+        self.keyboard_widget.initialize(self.gui.keyboard)
+        self.keyboard_widget.highlight_group(self.group_name)
+
+    def commit(self):
+        self.keyboard_widget.commit()
+        self.accept()
+
+
+class DateDelegate(QStyledItemDelegate):
+    '''
+    Delegate for dates. Because this delegate stores the
+    format as an instance variable, a new instance must be created for each
+    column. This differs from all the other delegates.
+    '''
+    def __init__(self, parent):
+        QStyledItemDelegate.__init__(self, parent)
+        self.format = 'dd MMM yyyy'
+
+    def displayText(self, val, locale):
+        d = val.toDateTime()
+        if d <= UNDEFINED_QDATETIME:
+            return ''
+        return format_date(qt_to_dt(d, as_utc=False), self.format)
+
+    def createEditor(self, parent, option, index):
+        qde = QStyledItemDelegate.createEditor(self, parent, option, index)
+        qde.setDisplayFormat(self.format)
+        qde.setMinimumDateTime(UNDEFINED_QDATETIME)
+        qde.setSpecialValueText(_('Undefined'))
+        qde.setCalendarPopup(True)
+        return qde
+
+    def setEditorData(self, editor, index):
+        val = index.model().data(index, Qt.DisplayRole).toDateTime()
+        if val is None or val == UNDEFINED_QDATETIME:
+            val = now()
+        editor.setDateTime(val)
+
+    def setModelData(self, editor, model, index):
+        val = editor.dateTime()
+        if val <= UNDEFINED_QDATETIME:
+            model.setData(index, UNDEFINED_QDATETIME, Qt.EditRole)
+        else:
+            model.setData(index, QDateTime(val), Qt.EditRole)

calibre-plugin/config.py

@@ -7,41 +7,201 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import (QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QTextEdit,
-                      QComboBox, QCheckBox)
+import traceback, copy
 
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit,
+                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant)
+
+from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
+from calibre.gui2.ui import get_gui
 
-from calibre_plugins.fanfictiondownloader_plugin.dialogs import (OVERWRITE, ADDNEW, SKIP,CALIBREONLY,UPDATE)
+from calibre_plugins.fanfictiondownloader_plugin.dialogs \
+    import (SKIP, ADDNEW, UPDATE, UPDATEALWAYS, OVERWRITE, OVERWRITEALWAYS,
+             CALIBREONLY,collision_order)
+
+from calibre_plugins.fanfictiondownloader_plugin.common_utils \
+    import ( get_library_uuid, KeyboardConfigDialog )
+
+from calibre.gui2.complete import MultiCompleteLineEdit
 
 # This is where all preferences for this plugin will be stored
 # Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
 # in a global namespace, so make it as unique as possible.
 # You should always prefix your config file name with plugins/,
 # so as to ensure you dont accidentally clobber a calibre config file
-prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+all_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
 
-# urlsfrompriority values
-CLIP = 'Clipboard'
-SELECTED = 'Selected Stories'
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+all_prefs.defaults['personal.ini'] = get_resources('plugin-example.ini')
+all_prefs.defaults['updatemeta'] = True
+all_prefs.defaults['keeptags'] = False
+all_prefs.defaults['urlsfromclip'] = True
+all_prefs.defaults['updatedefault'] = True
+all_prefs.defaults['fileform'] = 'epub'
+all_prefs.defaults['collision'] = OVERWRITE
+all_prefs.defaults['deleteotherforms'] = False
+all_prefs.defaults['send_lists'] = ''
+all_prefs.defaults['read_lists'] = ''
+all_prefs.defaults['addtolists'] = False
+all_prefs.defaults['addtoreadlists'] = False
+all_prefs.defaults['addtolistsonread'] = False
+all_prefs.defaults['custom_cols'] = {}
 
-# Set defaults
-prefs.defaults['personal.ini'] = get_resources('example.ini')
-prefs.defaults['updatemeta'] = True
-prefs.defaults['onlyoverwriteifnewer'] = False
-prefs.defaults['urlsfromclip'] = True
-prefs.defaults['urlsfromselected'] = True
-prefs.defaults['urlsfrompriority'] = SELECTED
-prefs.defaults['fileform'] = 'epub'
-prefs.defaults['collision'] = OVERWRITE
+# The list of settings to copy from all_prefs or the previous library
+# when config is called for the first time on a library.
+copylist = ['personal.ini',
+            'updatemeta',
+            'keeptags',
+            'urlsfromclip',
+            'updatedefault',
+            'fileform',
+            'collision',
+            'deleteotherforms',
+            'addtolists',
+            'addtoreadlists',
+            'addtolistsonread']
 
+# fake out so I don't have to change the prefs calls anywhere.  The
+# Java programmer in me is offended by op-overloading, but it's very
+# tidy.
+class PrefsFacade():
+    def __init__(self,all_prefs):
+        self.all_prefs = all_prefs
+        self.lastlibid = None
+
+    def _get_copylist_prefs(self,frompref):
+        return filter( lambda x : x[0] in copylist, frompref.items() )
+        
+    def _get_prefs(self):
+        libraryid = get_library_uuid(get_gui().current_db)
+        if libraryid not in self.all_prefs:
+            if self.lastlibid == None:
+                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs))
+            else:
+                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs[self.lastlibid]))
+            self.lastlibid = libraryid
+            
+        return self.all_prefs[libraryid]
+
+    def _save_prefs(self,prefs):
+        libraryid = get_library_uuid(get_gui().current_db)
+        self.all_prefs[libraryid] = prefs
+        
+    def __getitem__(self,k):            
+        prefs = self._get_prefs()
+        if k not in prefs:
+            # pulls from all_prefs.defaults automatically if not set
+            # in all_prefs
+            return self.all_prefs[k]
+        return prefs[k]
+
+    def __setitem__(self,k,v):
+        prefs = self._get_prefs()
+        prefs[k]=v
+        self._save_prefs(prefs)
+
+    # to be avoided--can cause unexpected results as possibly ancient
+    # all_pref settings may be pulled.
+    def __delitem__(self,k):
+        prefs = self._get_prefs()
+        del prefs[k]
+        self._save_prefs(prefs)
+
+prefs = PrefsFacade(all_prefs)
+    
 class ConfigWidget(QWidget):
 
-    def __init__(self):
+    def __init__(self, plugin_action):
         QWidget.__init__(self)
+        self.plugin_action = plugin_action
+        
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
+        tab_widget = QTabWidget(self)
+        self.l.addWidget(tab_widget)
+
+        self.basic_tab = BasicTab(self, plugin_action)
+        tab_widget.addTab(self.basic_tab, 'Basic')
+
+        self.personalini_tab = PersonalIniTab(self, plugin_action)
+        tab_widget.addTab(self.personalini_tab, 'personal.ini')
+        
+        self.list_tab = ListTab(self, plugin_action)
+        tab_widget.addTab(self.list_tab, 'Reading Lists')
+        if 'Reading List' not in plugin_action.gui.iactions:
+            self.list_tab.setEnabled(False)
+
+        self.columns_tab = ColumnsTab(self, plugin_action)
+        tab_widget.addTab(self.columns_tab, 'Custom Columns')
+
+        self.other_tab = OtherTab(self, plugin_action)
+        tab_widget.addTab(self.other_tab, 'Other')
+
+
+    def save_settings(self):
+
+        # basic
+        prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
+        prefs['collision'] = unicode(self.basic_tab.collision.currentText())
+        prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
+        prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
+        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
+        prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
+
+        if self.list_tab:
+            # lists
+            prefs['send_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.list_tab.send_lists_box.text()).split(','))))
+            prefs['read_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.list_tab.read_lists_box.text()).split(','))))
+            # print("send_lists: %s"%prefs['send_lists'])
+            # print("read_lists: %s"%prefs['read_lists'])
+            prefs['addtolists'] = self.list_tab.addtolists.isChecked()
+            prefs['addtoreadlists'] = self.list_tab.addtoreadlists.isChecked()
+            prefs['addtolistsonread'] = self.list_tab.addtolistsonread.isChecked()
+
+        # personal.ini
+        ini = unicode(self.personalini_tab.ini.toPlainText())
+        if ini:
+            prefs['personal.ini'] = ini
+        else:
+            # if they've removed everything, reset to default.
+            prefs['personal.ini'] = get_resources('plugin-example.ini')
+
+        # Custom Columns tab
+        colsmap = {}
+        for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems():
+            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            if val != 'none':
+                colsmap[col] = val
+                #print("colsmap[%s]:%s"%(col,colsmap[col]))
+        prefs['custom_cols'] = colsmap
+        
+    def edit_shortcuts(self):
+        self.save_settings()
+        # Force the menus to be rebuilt immediately, so we have all our actions registered
+        self.plugin_action.rebuild_menus()
+        d = KeyboardConfigDialog(self.plugin_action.gui, self.plugin_action.action_spec[0])
+        if d.exec_() == d.Accepted:
+            self.plugin_action.gui.keyboard.finalize()
+
+class BasicTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
         horz = QHBoxLayout()
         label = QLabel('Default Output &Format:')
         horz.addWidget(label)
@@ -52,6 +212,7 @@ class ConfigWidget(QWidget):
         self.fileform.addItem('txt')
         self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
         self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
         horz.addWidget(self.fileform)
         self.l.addLayout(horz)
@@ -61,50 +222,73 @@ class ConfigWidget(QWidget):
         label.setToolTip("What to do if there's already an existing story with the same title and author.")
         horz.addWidget(label)
         self.collision = QComboBox(self)
-        self.collision.addItem(OVERWRITE)
-        self.collision.addItem(UPDATE)
-        self.collision.addItem(ADDNEW)
-        self.collision.addItem(SKIP)
-        self.collision.addItem(CALIBREONLY)
-        self.collision.setCurrentIndex(self.collision.findText(prefs['collision']))
-        self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
         label.setBuddy(self.collision)
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
 
         self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?')
+        self.updatemeta.setToolTip('Update title, author, URL, tags, custom columns, etc for story in Calibre from web site.')
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
-        self.onlyoverwriteifnewer = QCheckBox('Default Only Overwrite Story if Newer',self)
-        self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer or from the same day.")
-        self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer'])
-        self.l.addWidget(self.onlyoverwriteifnewer)
-        
+        self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
+        self.keeptags.setToolTip('Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.')
+        self.keeptags.setChecked(prefs['keeptags'])
+        self.l.addWidget(self.keeptags)
+
         self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
-        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when starting plugin?')
+        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
         self.urlsfromclip.setChecked(prefs['urlsfromclip'])
         self.l.addWidget(self.urlsfromclip)
 
-        self.urlsfromselected = QCheckBox('Take URLs from Selected Stories?',self)
-        self.urlsfromselected.setToolTip('Prefill URLs from valid URLs in selected stories when starting plugin?')
-        self.urlsfromselected.setChecked(prefs['urlsfromselected'])
-        self.l.addWidget(self.urlsfromselected)
+        self.updatedefault = QCheckBox('Default to Update when books selected?',self)
+        self.updatedefault.setToolTip('The top FanFictionDownLoader plugin button will start Update if\n'+
+                                      'books are selected.  If unchecked, it will always bring up \'Add New\'.')
+        self.updatedefault.setChecked(prefs['updatedefault'])
+        self.l.addWidget(self.updatedefault)
 
-        horz = QHBoxLayout()
-        label = QLabel('Take URLs from which first:')
-        label.setToolTip("If both clipbaord and selected enabled and populated, which is used?")
-        horz.addWidget(label)
-        self.urlsfrompriority = QComboBox(self)
-        self.urlsfrompriority.addItem(SELECTED)
-        self.urlsfrompriority.addItem(CLIP)
-        self.urlsfrompriority.setCurrentIndex(self.urlsfrompriority.findText(prefs['urlsfrompriority']))
-        self.urlsfrompriority.setToolTip('If both clipbaord and selected enabled and populated, which is used?')
-        label.setBuddy(self.urlsfrompriority)
-        horz.addWidget(self.urlsfrompriority)
-        self.l.addLayout(horz)
+        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
+        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
+        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
+        self.l.addWidget(self.deleteotherforms)
+        
+        self.l.insertStretch(-1)
+        
+    def set_collisions(self):
+        prev=self.collision.currentText()
+        self.collision.clear()
+        for o in collision_order:
+            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
+                self.collision.addItem(o)
+        i = self.collision.findText(prev)
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        
+    def show_defaults(self):
+        text = get_resources('plugin-defaults.ini')
+        ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+        
+class PersonalIniTab(QWidget):
 
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set is_adult and user/password for different sites.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
         self.label = QLabel('personal.ini:')
         self.l.addWidget(self.label)
 
@@ -112,22 +296,251 @@ class ConfigWidget(QWidget):
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(prefs['personal.ini'])
         self.l.addWidget(self.ini)
-        
-    def save_settings(self):
-        prefs['fileform'] = unicode(self.fileform.currentText())
-        prefs['collision'] = unicode(self.collision.currentText())
-        prefs['updatemeta'] = self.updatemeta.isChecked()
-        prefs['urlsfrompriority'] = unicode(self.urlsfrompriority.currentText())
-        prefs['urlsfromclip'] = self.urlsfromclip.isChecked()
-        prefs['urlsfromselected'] = self.urlsfromselected.isChecked()
-        prefs['onlyoverwriteifnewer'] = self.onlyoverwriteifnewer.isChecked()
-        
-        ini = unicode(self.ini.toPlainText())
-        if ini:
-            prefs['personal.ini'] = ini
-        else:
-            # if they've removed everything, clear it so they get the
-            # default next time.
-            del prefs['personal.ini']
-        
 
+        self.defaults = QPushButton('View Defaults', self)
+        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
+        self.defaults.clicked.connect(self.show_defaults)
+        self.l.addWidget(self.defaults)
+        
+        # self.l.insertStretch(-1)
+        # let edit box fill the space.
+        
+    def show_defaults(self):
+        text = get_resources('plugin-defaults.ini')
+        ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+
+class ShowDefaultsIniDialog(QDialog):
+
+    def __init__(self, icon, text, parent=None):
+        QDialog.__init__(self, parent)
+        self.resize(600, 500)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.label = QLabel("Plugin Defaults (Read-Only)")
+        self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        self.setWindowTitle(_('Plugin Defaults'))
+        self.setWindowIcon(icon)
+        self.l.addWidget(self.label)
+        
+        self.ini = QTextEdit(self)
+        self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        self.ini.setLineWrapMode(QTextEdit.NoWrap)
+        self.ini.setText(text)
+        self.ini.setReadOnly(True)
+        self.l.addWidget(self.ini)
+        
+        self.ok_button = QPushButton('OK', self)
+        self.ok_button.clicked.connect(self.hide)
+        self.l.addWidget(self.ok_button)
+        
+class ListTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        try:
+            rl_plugin = plugin_action.gui.iactions['Reading List']
+            reading_lists = rl_plugin.get_list_names()
+        except KeyError:
+            reading_lists= []
+            
+        label = QLabel('These settings provide integration with the Reading List Plugin.  Reading List can automatically send to devices and change custom columns.  You have to create and configure the lists in Reading List to be useful.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.addtolists = QCheckBox('Add new/updated stories to "Send to Device" Reading List(s).',self)
+        self.addtolists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.')
+        self.addtolists.setChecked(prefs['addtolists'])
+        self.l.addWidget(self.addtolists)
+            
+        horz = QHBoxLayout()
+        label = QLabel('"Send to Device" Reading Lists')
+        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        horz.addWidget(label)        
+        self.send_lists_box = MultiCompleteLineEdit(self)
+        self.send_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.send_lists_box.update_items_cache(reading_lists)
+        self.send_lists_box.setText(prefs['send_lists'])
+        horz.addWidget(self.send_lists_box)
+        self.l.addLayout(horz)
+        
+        self.addtoreadlists = QCheckBox('Add new/updated stories to "To Read" Reading List(s).',self)
+        self.addtoreadlists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.\nAlso offers menu option to remove stories from the "To Read" lists.')
+        self.addtoreadlists.setChecked(prefs['addtoreadlists'])
+        self.l.addWidget(self.addtoreadlists)
+            
+        horz = QHBoxLayout()
+        label = QLabel('"To Read" Reading Lists')
+        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        horz.addWidget(label)        
+        self.read_lists_box = MultiCompleteLineEdit(self)
+        self.read_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.read_lists_box.update_items_cache(reading_lists)
+        self.read_lists_box.setText(prefs['read_lists'])
+        horz.addWidget(self.read_lists_box)
+        self.l.addLayout(horz)
+        
+        self.addtolistsonread = QCheckBox('Add stories back to "Send to Device" Reading List(s) when marked "Read".',self)
+        self.addtolistsonread.setToolTip('Menu option to remove from "To Read" lists will also add stories back to "Send to Device" Reading List(s)')
+        self.addtolistsonread.setChecked(prefs['addtolistsonread'])
+        self.l.addWidget(self.addtolistsonread)
+            
+        self.l.insertStretch(-1)
+        
+class OtherTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        keyboard_shortcuts_button = QPushButton('Keyboard shortcuts...', self)
+        keyboard_shortcuts_button.setToolTip(_(
+                    'Edit the keyboard shortcuts associated with this plugin'))
+        keyboard_shortcuts_button.clicked.connect(parent_dialog.edit_shortcuts)
+        self.l.addWidget(keyboard_shortcuts_button)
+
+        reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self)
+        reset_confirmation_button.setToolTip(_(
+                    'Reset all show me again dialogs for the FanFictionDownLoader plugin'))
+        reset_confirmation_button.clicked.connect(self.reset_dialogs)
+        self.l.addWidget(reset_confirmation_button)
+        
+        self.l.insertStretch(-1)
+        
+    def reset_dialogs(self):
+        for key in dynamic.keys():
+            if key.startswith('fanfictiondownloader_') and key.endswith('_again') \
+                                                  and dynamic[key] is False:
+                dynamic[key] = True
+        info_dialog(self, _('Done'),
+                _('Confirmation dialogs have all been reset'), show=True)
+
+permitted_values = {
+    'int' : ['numWords','numChapters'],
+    'float' : ['numWords','numChapters'],
+    'bool' : ['status-C','status-I'],
+    'datetime' : ['datePublished', 'dateUpdated', 'dateCreated'],
+    'series' : ['series'],
+    'enumeration' : ['category',
+                     'genre',
+                     'series',
+                     'characters',
+                     'status',
+                     'datePublished',
+                     'dateUpdated',
+                     'dateCreated',
+                     'rating',
+                     'warnings',
+                     'numChapters',
+                     'numWords',
+                     'site',
+                     'storyId',
+                     'authorId',
+                     'extratags',
+                     'title',
+                     'storyUrl',
+                     'description',
+                     'author',
+                     'authorUrl',
+                     'formatname'
+                     #,'formatext'   # not useful information.
+                     #,'siteabbrev'
+                     #,'version'
+                     ]
+    }
+# no point copying the whole list.
+permitted_values['text'] = permitted_values['enumeration']
+permitted_values['comments'] = permitted_values['enumeration']
+
+titleLabels = {
+    'category':'Category',
+    'genre':'Genre',
+    'status':'Status',
+    'status-C':'Status:Completed',
+    'status-I':'Status:In-Progress',
+    'series':'Series',
+    'characters':'Characters',
+    'datePublished':'Published',
+    'dateUpdated':'Updated',
+    'dateCreated':'Packaged',
+    'rating':'Rating',
+    'warnings':'Warnings',
+    'numChapters':'Chapters',
+    'numWords':'Words',
+    'site':'Site',
+    'storyId':'Story ID',
+    'authorId':'Author ID',
+    'extratags':'Extra Tags',
+    'title':'Title',
+    'storyUrl':'Story URL',
+    'description':'Summary',
+    'author':'Author',
+    'authorUrl':'Author URL',
+    'formatname':'File Format',
+    'formatext':'File Extension',
+    'siteabbrev':'Site Abbrev',
+    'version':'FFDL Version'
+    }
+
+class ColumnsTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.custcol_dropdowns = {}
+
+        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
+
+        for key, column in custom_columns.iteritems():
+
+            if column['datatype'] in permitted_values:
+                # print("\n============== %s ===========\n"%key)
+                # for (k,v) in column.iteritems():
+                #     print("column['%s'] => %s"%(k,v))
+                horz = QHBoxLayout()
+                label = QLabel('%s(%s)'%(column['name'],key))
+                label.setToolTip("Update this %s column with..."%column['datatype'])
+                horz.addWidget(label)
+                dropdown = QComboBox(self)
+                dropdown.addItem('',QVariant('none'))
+                for md in permitted_values[column['datatype']]:
+                    dropdown.addItem(titleLabels[md],QVariant(md))
+                self.custcol_dropdowns[key] = dropdown
+                if key in prefs['custom_cols']:
+                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
+                if column['datatype'] == 'enumeration':
+                    dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
+                else:
+                    dropdown.setToolTip("Metadata values valid for this type of column.")
+
+                horz.addWidget(dropdown)
+                self.l.addLayout(horz)
+        
+        self.l.insertStretch(-1)
+
+        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])

calibre-plugin/dcsource.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+from zipfile import ZipFile
+
+from xml.dom.minidom import parseString
+
+def get_dcsource(inputio):
+    epub = ZipFile(inputio, 'r')
+
+    ## Find the .opf file.
+    container = epub.read("META-INF/container.xml")
+    containerdom = parseString(container)
+    rootfilenodelist = containerdom.getElementsByTagName("rootfile")
+    rootfilename = rootfilenodelist[0].getAttribute("full-path")
+
+    metadom = parseString(epub.read(rootfilename))
+    firstmetadom = metadom.getElementsByTagName("metadata")[0]
+    try:
+        source=firstmetadom.getElementsByTagName("dc:source")[0].firstChild.data.encode("utf-8")
+    except:
+        source=None
+
+    return source

calibre-plugin/dialogs.py

@@ -9,28 +9,68 @@ __docformat__ = 'restructuredtext en'
 
 import traceback
 
-from PyQt4.Qt import (QDialog, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, 
-                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, 
-                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt )
+from PyQt4 import QtGui
+from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
+                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, QTextCursor,
+                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
+                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt,QAbstractItemView )
 
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
+from calibre.gui2.dialogs.confirm_delete import confirm
+
+from calibre import confirm_config_name
+from calibre.gui2 import dynamic
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+from calibre_plugins.fanfictiondownloader_plugin.common_utils \
+    import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
+            ImageTitleLayout, get_icon)
 
-OVERWRITE='Overwrite'
-UPDATE='Update EPUB'
-ADDNEW='Add New'
 SKIP='Skip'
+ADDNEW='Add New Book'
+UPDATE='Update EPUB if New Chapters'
+UPDATEALWAYS='Update EPUB Always'
+OVERWRITE='Overwrite if Newer'
+OVERWRITEALWAYS='Overwrite Always'
 CALIBREONLY='Update Calibre Metadata Only'
+collision_order=[SKIP,
+                 ADDNEW,
+                 UPDATE,
+                 UPDATEALWAYS,
+                 OVERWRITE,
+                 OVERWRITEALWAYS,
+                 CALIBREONLY,]
+        
+class NotGoingToDownload(Exception):
+    def __init__(self,error,icon='dialog_error.png'):
+        self.error=error
+        self.icon=icon
 
-class DownloadDialog(QDialog):
+    def __str__(self):
+        return self.error
 
-    def __init__(self, gui, prefs, icon, url_list_text, do_user_config, start_downloads):
-        QDialog.__init__(self, gui)
+class DroppableQTextEdit(QTextEdit):
+    def __init__(self,parent):
+        QTextEdit.__init__(self,parent)
+        
+    def canInsertFromMimeData(self, source):
+        if source.hasUrls():
+            return True;
+        else:
+            return QTextEdit.canInsertFromMimeData(self,source)
+
+    def insertFromMimeData(self, source):
+        if source.hasUrls():
+            for u in source.urls():
+                self.append(u.toString())
+        else:
+            return QTextEdit.insertFromMimeData(self, source)
+                            
+class AddNewDialog(SizePersistedDialog):
+
+    def __init__(self, gui, prefs, icon, url_list_text):
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
         self.gui = gui
-        self.do_user_config = do_user_config
-        self.start_downloads = start_downloads
 
         self.setMinimumWidth(300)
         self.l = QVBoxLayout()
@@ -40,21 +80,12 @@ class DownloadDialog(QDialog):
         self.setWindowIcon(icon)
 
         self.l.addWidget(QLabel('Story URL(s), one per line:'))
-        self.url = QTextEdit(self)
-        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from selected books or clipboard, but only valid URLs.')
+        self.url = DroppableQTextEdit(self)
+        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.')
         self.url.setLineWrapMode(QTextEdit.NoWrap)
         self.url.setText(url_list_text)
         self.l.addWidget(self.url)
         
-        self.ffdl_button = QPushButton(
-            'Download Stories', self)
-        self.ffdl_button.setToolTip('Start download(s).')
-        self.ffdl_button.clicked.connect(self.ffdl)
-        # if there's already URL(s), focus 'go' button
-        if url_list_text:
-            self.ffdl_button.setFocus()
-        self.l.addWidget(self.ffdl_button)
-
         horz = QHBoxLayout()
         label = QLabel('Output &Format:')
         horz.addWidget(label)
@@ -65,6 +96,8 @@ class DownloadDialog(QDialog):
         self.fileform.addItem('txt')
         self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
         self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.activated.connect(self.set_collisions)
+        
         label.setBuddy(self.fileform)
         horz.addWidget(self.fileform)
         self.l.addLayout(horz)
@@ -74,17 +107,16 @@ class DownloadDialog(QDialog):
         label.setToolTip("What to do if there's already an existing story with the same title and author.")
         horz.addWidget(label)
         self.collision = QComboBox(self)
-        self.collision.addItem(OVERWRITE)
-        self.collision.addItem(UPDATE)
-        self.collision.addItem(ADDNEW)
-        self.collision.addItem(SKIP)
-        self.collision.addItem(CALIBREONLY)
-        self.collision.setCurrentIndex(self.collision.findText(prefs['collision']))
-        self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+
-                                  UPDATE+' will download new chapters only and add to existing EPUB.\n'+
-                                  ADDNEW+' will create a new story with the same title and author.\n'+
-                                  SKIP+' will not download existing stories.\n'+
-                                  CALIBREONLY+' will not download stories, but will update Calibre metadata.')
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        # self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+
+        #                           UPDATE+' will download new chapters only and add to existing EPUB.\n'+
+        #                           ADDNEW+' will create a new story with the same title and author.\n'+
+        #                           SKIP+' will not download existing stories.\n'+
+        #                           CALIBREONLY+' will not download stories, but will update Calibre metadata.')
         label.setBuddy(self.collision)
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
@@ -94,50 +126,37 @@ class DownloadDialog(QDialog):
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
-        self.onlyoverwriteifnewer = QCheckBox('Only Overwrite Story if Newer',self)
-        self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer.\n"+
-                                             "From the same day counts as 'newer' because the sites don't give update time.")
-        self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer'])
-        self.l.addWidget(self.onlyoverwriteifnewer)
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
         
-        horz = QHBoxLayout()
-        self.about_button = QPushButton('About', self)
-        self.about_button.clicked.connect(self.about)
-        horz.addWidget(self.about_button)
-        self.conf_button = QPushButton(
-                'Configure this plugin', self)
-        self.conf_button.clicked.connect(self.config)
-        horz.addWidget(self.conf_button)
-        self.l.addLayout(horz)
+        if url_list_text:
+            button_box.button(QDialogButtonBox.Ok).setFocus()
 
-        self.resize(self.sizeHint())
+        # restore saved size.
+        self.resize_dialog()
+        #self.resize(self.sizeHint())
 
-    def about(self):
-        # Get the about text from a file inside the plugin zip file
-        # The get_resources function is a builtin function defined for all your
-        # plugin code. It loads files from the plugin zip file. It returns
-        # the bytes from the specified file.
-        #
-        # Note that if you are loading more than one file, for performance, you
-        # should pass a list of names to get_resources. In this case,
-        # get_resources will return a dictionary mapping names to bytes. Names that
-        # are not found in the zip file will not be in the returned dictionary.
-        text = get_resources('about.txt')
-#        QMessageBox.about(self, 'About the FanFictionDownLoader Plugin',
- #               text.decode('utf-8'))
-        AboutDialog(self.windowIcon(),text,self).exec_()
-
-    def ffdl(self):
-        self.start_downloads(unicode(self.url.toPlainText()),
-                             unicode(self.fileform.currentText()),
-                             unicode(self.collision.currentText()),
-                             self.updatemeta.isChecked(),
-                             self.onlyoverwriteifnewer.isChecked())
-        self.hide()
-
-    def config(self):
-        self.do_user_config(parent=self)
+    def set_collisions(self):
+        prev=self.collision.currentText()
+        self.collision.clear()
+        for o in collision_order:
+            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
+                self.collision.addItem(o)
+        i = self.collision.findText(prev)
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        
+    def get_ffdl_options(self):
+        return {
+            'fileform': unicode(self.fileform.currentText()),
+            'collision': unicode(self.collision.currentText()),
+            'updatemeta': self.updatemeta.isChecked(),
+            }
 
+    def get_urlstext(self):
+        return unicode(self.url.toPlainText())
         
 class UserPassDialog(QDialog):
     '''
@@ -160,6 +179,7 @@ class UserPassDialog(QDialog):
    
         self.l.addWidget(QLabel("Password:"),2,0)
         self.passwd = QLineEdit(self)
+        self.passwd.setEchoMode(QLineEdit.Password)
         self.l.addWidget(self.passwd,2,1)
    
         self.ok_button = QPushButton('OK', self)
@@ -180,23 +200,28 @@ class UserPassDialog(QDialog):
         self.status=False
         self.hide()
 
-class MetadataProgressDialog(QProgressDialog):
+class LoopProgressDialog(QProgressDialog):
     '''
     ProgressDialog displayed while fetching metadata for each story.
     '''
-    def __init__(self, gui, loop_list, fileform, getadapter_function, download_list_function, db):
+    def __init__(self, gui,
+                 book_list,
+                 foreach_function,
+                 finish_function,
+                 init_label="Fetching metadata for stories...",
+                 win_title="Downloading metadata for stories",
+                 status_prefix="Fetched metadata for"):
         QProgressDialog.__init__(self,
-                                 "Fetching metadata for stories...",
-                                 QString(), 0, len(loop_list), gui)
-        self.setWindowTitle("Downloading metadata for stories")
+                                 init_label,
+                                 QString(), 0, len(book_list), gui)
+        self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
         self.gui = gui
-        self.db = db
-        self.loop_list = loop_list
-        self.fileform = fileform
-        self.getadapter_function = getadapter_function
-        self.download_list_function = download_list_function
-        self.i, self.loop_bad, self.loop_good = 0, [], []
+        self.book_list = book_list
+        self.foreach_function = foreach_function
+        self.finish_function = finish_function
+        self.status_prefix = status_prefix
+        self.i = 0
         
         ## self.do_loop does QTimer.singleShot on self.do_loop also.
         ## A weird way to do a loop, but that was the example I had.
@@ -204,59 +229,49 @@ class MetadataProgressDialog(QProgressDialog):
         self.exec_()
 
     def updateStatus(self):
-        self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.loop_list)))
+        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
         self.setValue(self.i+1)
         print(self.labelText())
 
     def do_loop(self):
-        print("self.i:%d"%self.i)
 
         if self.i == 0:
             self.setValue(0)
 
-        if self.i >= len(self.loop_list) or self.wasCanceled():
-            return self.do_when_finished()
-
-        else:
-            current = self.loop_list[self.i]
-            try:
-                ## collision spec passed into getadapter by partial from ffdl_plugin
-                ## no retval only if it exists, but collision is SKIP
-                retval = self.getadapter_function(current,self.fileform)
-                self.loop_good.append((current,retval))
-            except Exception as e:
-                print("%s:%s"%(current,e))
-                self.loop_bad.append((current,e))
-                traceback.print_exc()
+        book = self.book_list[self.i]
+        try:
+            ## collision spec passed into getadapter by partial from ffdl_plugin
+            ## no retval only if it exists, but collision is SKIP
+            self.foreach_function(book)
             
-            self.updateStatus()
-            self.i += 1
+        except NotGoingToDownload as d:
+            book['good']=False
+            book['comment']=unicode(d)
+            book['icon'] = d.icon
+
+        except Exception as e:
+            book['good']=False
+            book['comment']=unicode(e)
+            print("Exception: %s:%s"%(book,unicode(e)))
+            traceback.print_exc()
+            
+        self.updateStatus()
+        self.i += 1
+            
+        if self.i >= len(self.book_list) or self.wasCanceled():
+            return self.do_when_finished()
+        else:
             QTimer.singleShot(0, self.do_loop)
 
     def do_when_finished(self):
         self.hide()
-        
-        # Queues a job to process these ePub/Mobi books in the background.
-        self.download_list_function(self.loop_good,self.fileform)
-        
-        if self.loop_bad != []:
-            res = []
-            for j in self.loop_bad:
-                res.append('%s : %s'%j)
-            msg = '%s' % '\n'.join(res)
-            warning_dialog(self.gui, _('Not going to download some stories'),
-                _('Not going to download %d of %d stories.') %
-                (len(self.loop_bad), len(self.loop_list)),
-                msg).exec_()
-        # else:
-        #     info_dialog(self.gui, "Starting Downloads",
-        #                 "Got metadata and started download for %d stories."%len(self.loop_good),
-        #                 show_copy_button=False).exec_()
-        self.gui = None
+        self.gui = None        
+        # Queues a job to process these books in the background.
+        self.finish_function(self.book_list)
 
 class AboutDialog(QDialog):
 
-    def __init__(self, icon, text, parent=None):
+    def __init__(self, parent, icon, text):
         QDialog.__init__(self, parent)
         self.resize(400, 250)
         self.l = QGridLayout()
@@ -267,7 +282,7 @@ class AboutDialog(QDialog):
         self.label = QLabel(text)
         self.label.setOpenExternalLinks(True)
         self.label.setWordWrap(True)
-        self.setWindowTitle(_('Update available!'))
+        self.setWindowTitle(_('About FanFictionDownLoader'))
         self.setWindowIcon(icon)
         self.l.addWidget(self.logo, 0, 0)
         self.l.addWidget(self.label, 0, 1)
@@ -277,10 +292,355 @@ class AboutDialog(QDialog):
         self.l.addWidget(self.bb, 2, 0, 1, -1)
         self.bb.accepted.connect(self.accept)
 
-        
-class NotGoingToDownload(Exception):
-    def __init__(self,error):
-        self.error=error
+class IconWidgetItem(ReadOnlyTextIconWidgetItem):
+    def __init__(self, text, icon, sort_key):
+        ReadOnlyTextIconWidgetItem.__init__(self, text, icon)
+        self.sort_key = sort_key
 
-    def __str__(self):
-        return self.error
+    #Qt uses a simple < check for sorting items, override this to use the sortKey
+    def __lt__(self, other):
+        return self.sort_key < other.sort_key
+
+class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):
+    def __init__(self, text, sort_key):
+        ReadOnlyTableWidgetItem.__init__(self, text)
+        self.sort_key = sort_key
+
+    #Qt uses a simple < check for sorting items, override this to use the sortKey
+    def __lt__(self, other):
+        return self.sort_key < other.sort_key
+
+class UpdateExistingDialog(SizePersistedDialog):
+    def __init__(self, gui, header, prefs, icon, books,
+                 save_size_name='fanfictiondownloader_plugin:update list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        self.gui = gui
+        
+        self.setWindowTitle(header)
+        self.setWindowIcon(icon)
+        
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, 'images/icon.png',
+                                        header)
+        layout.addLayout(title_layout)
+        books_layout = QHBoxLayout()
+        layout.addLayout(books_layout)
+
+        self.books_table = StoryListTableWidget(self)
+        books_layout.addWidget(self.books_table)
+
+        button_layout = QVBoxLayout()
+        books_layout.addLayout(button_layout)
+        # self.move_up_button = QtGui.QToolButton(self)
+        # self.move_up_button.setToolTip('Move selected books up the list')
+        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
+        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
+        # button_layout.addWidget(self.move_up_button)
+        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem)
+        self.remove_button = QtGui.QToolButton(self)
+        self.remove_button.setToolTip('Remove selected books from the list')
+        self.remove_button.setIcon(get_icon('list_remove.png'))
+        self.remove_button.clicked.connect(self.remove_from_list)
+        button_layout.addWidget(self.remove_button)
+        spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem1)
+        # self.move_down_button = QtGui.QToolButton(self)
+        # self.move_down_button.setToolTip('Move selected books down the list')
+        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
+        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
+        # button_layout.addWidget(self.move_down_button)
+
+        options_layout = QHBoxLayout()
+
+        label = QLabel('Output &Format:')
+        options_layout.addWidget(label)
+        self.fileform = QComboBox(self)
+        self.fileform.addItem('epub')
+        self.fileform.addItem('mobi')
+        self.fileform.addItem('html')
+        self.fileform.addItem('txt')
+        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.activated.connect(self.set_collisions)
+        label.setBuddy(self.fileform)
+        options_layout.addWidget(self.fileform)
+        
+        label = QLabel('Update Mode:')
+        label.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
+        options_layout.addWidget(label)
+        self.collision = QComboBox(self)
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
+        label.setBuddy(self.collision)
+        options_layout.addWidget(self.collision)
+
+        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
+        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?  May set default from plugin configuration.')
+        self.updatemeta.setChecked(prefs['updatemeta'])
+        options_layout.addWidget(self.updatemeta)
+                
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        options_layout.addWidget(button_box)
+        
+        layout.addLayout(options_layout)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.books_table.populate_table(books)
+
+    def set_collisions(self):
+        prev=self.collision.currentText()
+        self.collision.clear()
+        for o in collision_order:
+            if o not in [ADDNEW,SKIP] and \
+                    (self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]):
+                self.collision.addItem(o)        
+        i = self.collision.findText(prev)
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        
+    def remove_from_list(self):
+        self.books_table.remove_selected_rows()
+
+    def get_books(self):
+        return self.books_table.get_books()
+
+    def get_ffdl_options(self):
+        return {
+            'fileform': unicode(self.fileform.currentText()),
+            'collision': unicode(self.collision.currentText()),
+            'updatemeta': self.updatemeta.isChecked(),
+            }
+
+def display_story_list(gui, header, prefs, icon, books,
+                       label_text='',
+                       save_size_name='fanfictiondownloader_plugin:display list dialog',
+                       offer_skip=False):
+    all_good = True
+    for b in books:
+        if not b['good']:
+            all_good=False
+            break
+        
+    ## 
+    if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
+        return True
+        pass
+            ## fake accept?
+    d = DisplayStoryListDialog(gui, header, prefs, icon, books,
+                               label_text,
+                               save_size_name,
+                               offer_skip and all_good)
+    d.exec_()
+    return d.result() == d.Accepted
+
+class DisplayStoryListDialog(SizePersistedDialog):
+    def __init__(self, gui, header, prefs, icon, books,
+                 label_text='',
+                 save_size_name='fanfictiondownloader_plugin:display list dialog',
+                 offer_skip=False):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        self.name = save_size_name
+        self.gui = gui
+        
+        self.setWindowTitle(header)
+        self.setWindowIcon(icon)
+        
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, 'images/icon.png',
+                                        header)
+        layout.addLayout(title_layout)
+
+        self.books_table = StoryListTableWidget(self)
+        layout.addWidget(self.books_table)
+
+        options_layout = QHBoxLayout()
+        self.label = QLabel(label_text)
+        #self.label.setOpenExternalLinks(True)
+        #self.label.setWordWrap(True)
+        options_layout.addWidget(self.label)
+        
+        if offer_skip:
+            spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
+            options_layout.addItem(spacerItem1)
+            self.again = QCheckBox('Show this again?',self)
+            self.again.setChecked(True)
+            self.again.stateChanged.connect(self.toggle)
+            self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
+            options_layout.addWidget(self.again)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+
+        options_layout.addWidget(button_box)
+        
+        layout.addLayout(options_layout)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.books_table.populate_table(books)
+
+    def get_books(self):
+        return self.books_table.get_books()
+
+    def toggle(self, *args):
+        dynamic[confirm_config_name(self.name)] = self.again.isChecked()
+
+    
+
+class StoryListTableWidget(QTableWidget):
+
+    def __init__(self, parent):
+        QTableWidget.__init__(self, parent)
+        self.setSelectionBehavior(QAbstractItemView.SelectRows)
+
+    def populate_table(self, books):
+        self.clear()
+        self.setAlternatingRowColors(True)
+        self.setRowCount(len(books))
+        header_labels = ['','Title', 'Author', 'URL', 'Comment']
+        self.setColumnCount(len(header_labels))
+        self.setHorizontalHeaderLabels(header_labels)
+        self.horizontalHeader().setStretchLastSection(True)
+        #self.verticalHeader().setDefaultSectionSize(24)
+        self.verticalHeader().hide()
+
+        self.books={}
+        for row, book in enumerate(books):
+            self.populate_table_row(row, book)
+            self.books[row] = book
+
+        # turning True breaks up/down.  Do we need either sorting or up/down?
+        self.setSortingEnabled(True)
+        self.resizeColumnsToContents()
+        self.setMinimumColumnWidth(1, 100)
+        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumColumnWidth(3, 100)
+        self.setMinimumSize(300, 0)
+        # if len(books) > 0:
+        #     self.selectRow(0)
+        self.sortItems(1)
+        self.sortItems(0)
+
+    def setMinimumColumnWidth(self, col, minimum):
+        if self.columnWidth(col) < minimum:
+            self.setColumnWidth(col, minimum)
+
+    def populate_table_row(self, row, book):
+        if book['good']:
+            icon = get_icon('ok.png')
+            val = 0
+        else:
+            icon = get_icon('minus.png')
+            val = 1
+        if 'icon' in book:
+            icon = get_icon(book['icon'])
+
+        status_cell = IconWidgetItem(None,icon,val)
+        status_cell.setData(Qt.UserRole, QVariant(val))
+        self.setItem(row, 0, status_cell)
+        
+        title_cell = ReadOnlyTableWidgetItem(book['title'])
+        title_cell.setData(Qt.UserRole, QVariant(row))
+        self.setItem(row, 1, title_cell)
+        
+        self.setItem(row, 2, AuthorTableWidgetItem(book['author'], book['author_sort']))
+        
+        url_cell = ReadOnlyTableWidgetItem(book['url'])
+        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
+        self.setItem(row, 3, url_cell)
+        
+        comment_cell = ReadOnlyTableWidgetItem(book['comment'])
+        #comment_cell.setData(Qt.UserRole, QVariant(book))
+        self.setItem(row, 4, comment_cell)
+
+    def get_books(self):
+        books = []
+        #print("=========================\nbooks:%s"%self.books)
+        for row in range(self.rowCount()):
+            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
+            book = self.books[rnum]
+            books.append(book)
+        return books
+
+    def remove_selected_rows(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        message = '<p>Are you sure you want to remove this book from the list?'
+        if len(rows) > 1:
+            message = '<p>Are you sure you want to remove the selected %d books from the list?'%len(rows)
+        if not confirm(message,'fanfictiondownloader_delete_item', self):
+            return
+        first_sel_row = self.currentRow()
+        for selrow in reversed(rows):
+            self.removeRow(selrow.row())
+        if first_sel_row < self.rowCount():
+            self.select_and_scroll_to_row(first_sel_row)
+        elif self.rowCount() > 0:
+            self.select_and_scroll_to_row(first_sel_row - 1)
+
+    def select_and_scroll_to_row(self, row):
+        self.selectRow(row)
+        self.scrollToItem(self.currentItem())
+
+    def move_rows_up(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        first_sel_row = rows[0].row()
+        if first_sel_row <= 0:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in selrows:
+            self.swap_row_widgets(selrow - 1, selrow + 1)
+        scroll_to_row = first_sel_row - 1
+        if scroll_to_row > 0:
+            scroll_to_row = scroll_to_row - 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def move_rows_down(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        last_sel_row = rows[-1].row()
+        if last_sel_row == self.rowCount() - 1:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in reversed(selrows):
+            self.swap_row_widgets(selrow + 2, selrow)
+        scroll_to_row = last_sel_row + 1
+        if scroll_to_row < self.rowCount() - 1:
+            scroll_to_row = scroll_to_row + 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def swap_row_widgets(self, src_row, dest_row):
+        self.blockSignals(True)
+        self.insertRow(dest_row)
+        for col in range(0, self.columnCount()):
+            self.setItem(dest_row, col, self.takeItem(src_row, col))
+        self.removeRow(src_row)
+        self.blockSignals(False)

calibre-plugin/jobs.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Jim Miller'
+__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import time, os, traceback
+
+from ConfigParser import SafeConfigParser
+from StringIO import StringIO
+#from itertools import izip
+#from threading import Event
+
+#from calibre.gui2.convert.single import sort_formats_by_preference
+from calibre.utils.ipc.server import Server
+from calibre.utils.ipc.job import ParallelJob
+from calibre.utils.logging import Log
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
+    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge
+
+# ------------------------------------------------------------------------------
+#
+#              Functions to perform downloads using worker jobs
+#
+# ------------------------------------------------------------------------------
+
+def do_download_worker(book_list, options,
+                       cpus, notification=lambda x,y:x):
+    '''
+    Master job, to launch child jobs to extract ISBN for a set of books
+    This is run as a worker job in the background to keep the UI more
+    responsive and get around the memory leak issues as it will launch
+    a child job for each book as a worker process
+    '''
+    server = Server(pool_size=cpus)
+
+    print(options['version'])
+    total = 0
+    # Queue all the jobs
+    print("Adding jobs for URLs:")
+    for book in book_list:
+        if book['good']:
+            print("%s"%book['url'])
+            total += 1
+            args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
+                    'do_download_for_worker',
+                    (book,options)]
+            job = ParallelJob('arbitrary',
+                              "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
+                              done=None,
+                              args=args)
+            job._book = book
+            # job._book_id = book_id
+            # job._title = title
+            # job._modified_date = modified_date
+            # job._existing_isbn = existing_isbn
+            server.add_job(job)
+    
+    # This server is an arbitrary_n job, so there is a notifier available.
+    # Set the % complete to a small number to avoid the 'unavailable' indicator
+    notification(0.01, 'Downloading FanFiction Stories')
+
+    # dequeue the job results as they arrive, saving the results
+    count = 0
+    while True:
+        job = server.changed_jobs_queue.get()
+        # A job can 'change' when it is not finished, for example if it
+        # produces a notification. Ignore these.
+        job.update()
+        if not job.is_finished:
+            continue
+        # A job really finished. Get the information.
+        output_book = job.result
+        #print("output_book:%s"%output_book)
+        book_list.remove(job._book)
+        book_list.append(job.result)
+        book_id = job._book['calibre_id']
+        #title = job._title
+        count = count + 1
+        notification(float(count)/total, 'Downloaded Story')
+        # Add this job's output to the current log
+        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        print(job.details)
+
+        if count >= total:
+            # All done!
+            break
+
+    server.close()
+    
+    # return the book list as the job result
+    return book_list
+
+def do_download_for_worker(book,options):
+    '''
+    Child job, to extract isbn from formats for this specific book,
+    when run as a worker job
+    '''
+    try:
+        book['comment'] = 'Download started...'
+
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        ffdlconfig.readfp(StringIO(options['personal.ini']))
+        
+        adapter = adapters.getAdapter(ffdlconfig,book['url'])
+        adapter.is_adult = book['is_adult'] 
+        adapter.username = book['username'] 
+        adapter.password = book['password']
+        
+        story = adapter.getStoryMetadataOnly()
+        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+
+        outfile = book['outfile']
+
+        ## No need to download at all.  Shouldn't ever get down here.
+        if options['collision'] in (CALIBREONLY):
+            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            book['comment'] = 'Metadata collected.'
+            
+        ## checks were done earlier, it's new or not dup or newer--just write it.
+        elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
+                ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
+            print("write to %s"%outfile)
+            writer.writeStory(outfilename=outfile, forceOverwrite=True)
+            book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
+            
+        ## checks were done earlier, just update it.
+        elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
+            
+            urlchaptercount = int(story.getMetadata('numChapters'))
+            ## First, get existing epub with titlepage and tocpage stripped.
+            updateio = StringIO()
+            (epuburl,chaptercount) = doMerge(updateio,
+                                             [book['epub_for_update']],
+                                             titlenavpoints=False,
+                                             striptitletoc=True,
+                                             forceunique=False)
+            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+            print("write to %s"%outfile)
+
+            ## Get updated title page/metadata by itself in an epub.
+            ## Even if the title page isn't included, this carries the metadata.
+            titleio = StringIO()
+            writer.writeStory(outstream=titleio,metaonly=True)
+
+            newchaptersio = None
+            if urlchaptercount > chaptercount :
+                ## Go get the new chapters
+                newchaptersio = StringIO()
+                adapter.setChaptersRange(chaptercount+1,urlchaptercount)
+
+                adapter.config.set("overrides",'include_tocpage','false')
+                adapter.config.set("overrides",'include_titlepage','false')
+                writer.writeStory(outstream=newchaptersio)
+
+            ## Merge the three epubs together.
+            doMerge(outfile,
+                    [titleio,updateio,newchaptersio],
+                    fromfirst=True,
+                    titlenavpoints=False,
+                    striptitletoc=False,
+                    forceunique=False)
+            
+            book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
+                (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
+        
+    except NotGoingToDownload as d:
+        book['good']=False
+        book['comment']=unicode(d)
+        book['icon'] = d.icon
+
+    except Exception as e:
+        book['good']=False
+        book['comment']=unicode(e)
+        book['icon']='dialog_error.png'
+        print("Exception: %s:%s"%(book,unicode(e)))
+        traceback.print_exc()
+
+    #time.sleep(10)
+    return book

defaults.ini

@@ -1,4 +1,4 @@
-# Copyright 2011 Fanficdownloader team
+# Copyright 2012 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,6 +37,7 @@ formatext_label:File Extension
 category_label:Category
 genre_label:Genre
 characters_label:Characters
+series_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -61,12 +62,19 @@ authorId_label:Author ID
 extratags_label:Extra Tags
 ## The version of fanficdownloader
 ##
-version_label:FFD Version
+version_label:FFDL Version
 
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+collect_series: true
 
 ## include title page as first page.
 include_titlepage: true
@@ -88,8 +96,10 @@ include_tocpage: true
 #output_filename: books/${title}-${siteabbrev}_${storyId}${formatext}
 #output_filename: books/${formatname}/${siteabbrev}/${authorId}/${title}-${siteabbrev}_${storyId}${formatext}
 output_filename: ${title}-${siteabbrev}_${storyId}${formatext}
+
 ## Make directories as needed.
 make_directories: true
+
 ## Always overwrite output files.  Otherwise, the downloader checks
 ## the timestamp on the existing file and only overwrites if the story
 ## has been updated more recently.  Command line version only
@@ -97,6 +107,7 @@ make_directories: true
 
 ## put output (with output_filename) in a zip file zip_filename.
 zip_output: false
+
 ## Can include directories.  .zip will be added if not in name somewhere
 zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 
@@ -126,12 +137,13 @@ background_color: ffffff
 ## values are available, plus output_filename.
 #post_process_cmd: addbook -f "${output_filename}" -t "${title}"
 
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
@@ -194,6 +206,9 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+## twilighted.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.twiwrite.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -202,6 +217,9 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+## twiwrite.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.whofic.com]
 
 [www.mediaminer.org]
@@ -219,6 +237,9 @@ extratags:
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -259,6 +280,12 @@ output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
 
 [overrides]
 ## It may sometimes be useful to override all of the specific format,

downloader.py

@@ -84,14 +84,17 @@ def main():
 
    conflist = []
    homepath = join(expanduser("~"),".fanficdownloader")
+   
    if isfile(join(homepath,"defaults.ini")):
        conflist.append(join(homepath,"defaults.ini"))
-   if isfile(join(homepath,"personal.ini")):
-       conflist.append(join(homepath,"personal.ini"))
    if isfile("defaults.ini"):
        conflist.append("defaults.ini")
+       
+   if isfile(join(homepath,"personal.ini")):
+       conflist.append(join(homepath,"personal.ini"))
    if isfile("personal.ini"):
        conflist.append("personal.ini")
+       
    if options.configfile:
        conflist.extend(options.configfile)
       
@@ -127,21 +130,22 @@ def main():
            url = args[0]
 
        adapter = adapters.getAdapter(config,url)
-           
-       try:
-           adapter.getStoryMetadataOnly()
-       except exceptions.FailedToLogin:
-           print "Login Failed, Need Username/Password."
-           sys.stdout.write("Username: ")
-           adapter.username = sys.stdin.readline().strip()
-           adapter.password = getpass.getpass(prompt='Password: ')
-           #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
-           adapter.getStoryMetadataOnly()
-       except exceptions.AdultCheckRequired:
-           print "Please confirm you are an adult in your locale: (y/n)?"
-           if sys.stdin.readline().strip().lower().startswith('y'):
-               adapter.is_adult=True
-           adapter.getStoryMetadataOnly()
+
+       ## three tries, that's enough if both user/pass & is_adult needed,
+       ## or a couple tries of one or the other
+       for x in range(0,2):
+           try:
+               adapter.getStoryMetadataOnly()
+           except exceptions.FailedToLogin:
+               print "Login Failed, Need Username/Password."
+               sys.stdout.write("Username: ")
+               adapter.username = sys.stdin.readline().strip()
+               adapter.password = getpass.getpass(prompt='Password: ')
+               #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
+           except exceptions.AdultCheckRequired:
+               print "Please confirm you are an adult in your locale: (y/n)?"
+               if sys.stdin.readline().strip().lower().startswith('y'):
+                   adapter.is_adult=True
 
        if options.update and not options.force:
            urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))

editconfig.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -22,7 +22,7 @@
   <body>
     <div id='main' style="width: 80%; margin-left: 10%;">
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
 
       <div style="text-align: center">
@@ -67,7 +67,7 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
 	Copyright &copy; Fanficdownloader team
       </div>
 

epubmerge.py

@@ -224,10 +224,21 @@ def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None,
                             pass # Skip missing files.
                 
         for itemref in metadom.getElementsByTagName("itemref"):
+
             if not striptitletoc or not re.match(r'(title|toc)_page', itemref.getAttribute("idref")):
                 itemrefs.append(bookid+itemref.getAttribute("idref"))
 
         booknum=booknum+1;
+        if not forceunique:
+            # If not forceunique, it's an epub update.
+            # If there's a "calibre_bookmarks.txt", it's from reading
+            # in Calibre and should be preserved.
+            try:
+                fn = "META-INF/calibre_bookmarks.txt"
+                outputepub.writestr(fn,epub.read(fn))
+            except:
+                pass
+        
 
     ## create content.opf file. 
     uniqueid="epubmerge-uid-%d" % time() # real sophisticated uid scheme.
@@ -355,7 +366,7 @@ def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None,
     ## during TOC generation to save loops.
     outputepub.writestr("content.opf",contentdom.toxml('utf-8'))
     outputepub.writestr("toc.ncx",tocncxdom.toxml('utf-8'))
-    
+
     # declares all the files created by Windows.  otherwise, when
     # it runs in appengine, windows unzips the files as 000 perms.
     for zf in outputepub.filelist:

example.ini

@@ -1,6 +1,12 @@
 ## This is an example of what your personal configuration might look
 ## like.
 
+[defaults]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Most common, I expect will be using this to save username/passwords
 ## for different sites.
 [www.twilighted.net]

fanficdownloader/adapters/__init__.py

@@ -1,95 +1,97 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os, re, sys, glob, types
-from os.path import dirname, basename, normpath
-import logging
-import urlparse as up
-
-from .. import exceptions as exceptions
-
-## must import each adapter here.
-
-import adapter_test1
-import adapter_fanfictionnet
-import adapter_fanficcastletvnet
-import adapter_fanfictionnet
-import adapter_fictionalleyorg
-import adapter_fictionpresscom
-import adapter_ficwadcom
-import adapter_fimfictionnet
-import adapter_harrypotterfanfictioncom
-import adapter_mediaminerorg
-import adapter_potionsandsnitchesnet
-import adapter_tenhawkpresentscom
-import adapter_adastrafanficcom
-import adapter_thewriterscoffeeshopcom
-import adapter_tthfanficorg
-import adapter_twilightednet
-import adapter_twiwritenet
-import adapter_whoficcom
-
-## This bit of complexity allows adapters to be added by just adding
-## importing.  It eliminates the long if/else clauses we used to need
-## to pick out the adapter.
-    
-## List of registered site adapters.
-__class_list = []
-
-def imports():
-    for name, val in globals().items():
-        if isinstance(val, types.ModuleType):
-            yield val.__name__
-
-for x in imports():
-    if "fanficdownloader.adapters.adapter_" in x:
-        #print x
-        __class_list.append(sys.modules[x].getClass())
-            
-def getAdapter(config,url):
-    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
-    if not fixedurl.startswith("http"):
-        fixedurl = "http://%s"%url
-    ## remove any trailing '#' locations.
-    fixedurl = re.sub(r"#.*$","",fixedurl)
-    
-    ## remove any trailing '&' parameters--?sid=999 will be left.
-    ## that's all that any of the current adapters need or want.
-    fixedurl = re.sub(r"&.*$","",fixedurl)
-    
-    parsedUrl = up.urlparse(fixedurl)
-    domain = parsedUrl.netloc.lower()
-    if( domain != parsedUrl.netloc ):
-        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
-
-    logging.debug("site:"+domain)
-    cls = getClassFor(domain)
-    if not cls:
-        logging.debug("trying site:www."+domain)
-        cls = getClassFor("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
-    if cls:
-        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        return adapter
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getClassFor(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, re, sys, glob, types
+from os.path import dirname, basename, normpath
+import logging
+import urlparse as up
+
+from .. import exceptions as exceptions
+
+## must import each adapter here.
+
+import adapter_test1
+import adapter_fanfictionnet
+import adapter_castlefansorg
+import adapter_fanfictionnet
+import adapter_fictionalleyorg
+import adapter_fictionpresscom
+import adapter_ficwadcom
+import adapter_fimfictionnet
+import adapter_harrypotterfanfictioncom
+import adapter_mediaminerorg
+import adapter_potionsandsnitchesnet
+import adapter_tenhawkpresentscom
+import adapter_adastrafanficcom
+import adapter_thewriterscoffeeshopcom
+import adapter_tthfanficorg
+import adapter_twilightednet
+import adapter_twiwritenet
+import adapter_whoficcom
+import adapter_siyecouk
+import adapter_archiveofourownorg
+
+## This bit of complexity allows adapters to be added by just adding
+## importing.  It eliminates the long if/else clauses we used to need
+## to pick out the adapter.
+    
+## List of registered site adapters.
+__class_list = []
+
+def imports():
+    for name, val in globals().items():
+        if isinstance(val, types.ModuleType):
+            yield val.__name__
+
+for x in imports():
+    if "fanficdownloader.adapters.adapter_" in x:
+        #print x
+        __class_list.append(sys.modules[x].getClass())
+            
+def getAdapter(config,url):
+    ## fix up leading protocol.
+    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
+    if not fixedurl.startswith("http"):
+        fixedurl = "http://%s"%url
+    ## remove any trailing '#' locations.
+    fixedurl = re.sub(r"#.*$","",fixedurl)
+    
+    ## remove any trailing '&' parameters--?sid=999 will be left.
+    ## that's all that any of the current adapters need or want.
+    fixedurl = re.sub(r"&.*$","",fixedurl)
+    
+    parsedUrl = up.urlparse(fixedurl)
+    domain = parsedUrl.netloc.lower()
+    if( domain != parsedUrl.netloc ):
+        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
+
+    logging.debug("site:"+domain)
+    cls = getClassFor(domain)
+    if not cls:
+        logging.debug("trying site:www."+domain)
+        cls = getClassFor("www."+domain)
+        fixedurl = fixedurl.replace("http://","http://www.")
+    if cls:
+        adapter = cls(config,fixedurl) # raises InvalidStoryURL
+        return adapter
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getClassFor(domain):
+    for cls in __class_list:
+        if cls.matchesSite(domain):
+            return cls

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -79,6 +79,11 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
         if "Content is only suitable for mature adults. May contain explicit language and adult themes. Equivalent of NC-17." in data:
             raise exceptions.AdultCheckRequired(self.url)
             
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
@@ -142,6 +147,12 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 genrestext = [genre.string for genre in genres]
@@ -163,19 +174,45 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.strip(), "%m/%d/%Y"))
+                self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
             
             if 'Updated' in label:
                 # there's a stray [ at the end.
                 #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%m/%d/%Y"))
+                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
 
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -0,0 +1,268 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+
+def getClass():
+    return ArchiveOfOurOwnOrgAdapter
+
+
+class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+							   
+							   
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ao3')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%b-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.archiveofourown.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/works/123456"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            addurl = "?view_adult=true"
+        else:
+            addurl=""
+
+        meta = self.url+addurl
+        url = self.url+'/navigate'+addurl
+        logging.debug("URL: "+meta)
+
+        try:
+            data = self._fetchUrl(url)
+            meta = self._fetchUrl(meta)
+
+            if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        metasoup = bs.BeautifulSoup(meta)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r"^/works/\w+"))
+        self.story.setMetadata('title',a.string)
+		
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.text)
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
+        self.story.setMetadata('numChapters',len(chapters))
+        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        for x in range(0,len(chapters)):
+            # just in case there's tags, like <i> in chapter titles.
+            chapter=chapters[x]
+            if len(chapters)==1:
+                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
+            else:
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
+
+
+
+        a = metasoup.find('blockquote',{'class':'userstuff'})
+        if a != None:
+            self.story.setMetadata('description',a.text)
+		
+        a = metasoup.find('dd',{'class':"rating tags"})
+        if a != None:
+            self.story.setMetadata('rating',stripHTML(a.text))
+		
+        a = metasoup.find('dd',{'class':"fandom tags"})
+        fandoms = a.findAll('a',{'class':"tag"})
+        fandomstext = [fandom.string for fandom in fandoms]
+        for fandom in fandomstext:
+            self.story.addToList('category',fandom.string)
+		
+        a = metasoup.find('dd',{'class':"warning tags"})
+        if a != None:
+            warnings = a.findAll('a',{'class':"tag"})
+            warningstext = [warning.string for warning in warnings]
+            for warning in warningstext:
+                if warning.string == "Author Chose Not To Use Archive Warnings":
+                    warning.string = "No Archive Warnings Apply"
+                if warning.string != "No Archive Warnings Apply":
+                    self.story.addToList('warnings',warning.string)
+		
+        a = metasoup.find('dd',{'class':"freeform tags"})
+        if a != None:
+            genres = a.findAll('a',{'class':"tag"})
+            genrestext = [genre.string for genre in genres]
+            for genre in genrestext:
+                self.story.addToList('genre',genre.string)
+        a = metasoup.find('dd',{'class':"category tags"})
+        if a != None:
+            genres = a.findAll('a',{'class':"tag"})
+            genrestext = [genre.string for genre in genres]
+            for genre in genrestext:
+                if genre != "Gen":
+                    self.story.addToList('genre',genre.string)
+		
+        a = metasoup.find('dd',{'class':"character tags"})
+        if a != None:
+            chars = a.findAll('a',{'class':"tag"})
+            charstext = [char.string for char in chars]
+            for char in charstext:
+                self.story.addToList('characters',char.string)
+        a = metasoup.find('dd',{'class':"relationship tags"})
+        if a != None:
+            chars = a.findAll('a',{'class':"tag"})
+            charstext = [char.string for char in chars]
+            for char in charstext:
+                self.story.addToList('characters',char.string)
+		
+
+        stats = metasoup.find('dl',{'class':'stats'})
+        dt = stats.findAll('dt')
+        dd = stats.findAll('dd')
+        for x in range(0,len(dt)):
+            label = dt[x].text
+            value = dd[x].text
+
+            if 'Words:' in label:
+                self.story.setMetadata('numWords', value)
+				
+            if 'Chapters:' in label:
+                if value.split('/')[0] == value.split('/')[1]:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+				
+            if 'Completed' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+		
+        try:
+            # Find Series name from series URL.
+            a = metasoup.find('dd',{'class':"series"})
+            b = a.find('a', href=re.compile(r"/series/\d+"))
+            series_name = b.string
+            series_url = 'http://'+self.host+'/fanfic/'+b['href']
+            series_index = int(a.text.split(' ')[1])
+            self.setSeries(series_name, series_index)
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr'))
+		
+        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
+        if headnotes != None:
+            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+            if headnotes != None:
+                chapter.append(bs.BeautifulSoup("<b>Author's Note:</b>"))
+                chapter.append(headnotes)
+        
+        chapsumm = soup.find('div', {'id' : "summary"})
+        if chapsumm != None:
+            chapsumm = chapsumm.find('blockquote')
+            chapter.append(bs.BeautifulSoup("<b>Summary for the Chapter:</b>"))
+            chapter.append(chapsumm)
+        chapnotes = soup.find('div', {'id' : "notes"})
+        if chapnotes != None:
+            chapnotes = chapnotes.find('blockquote')
+            if chapnotes != None:
+                chapter.append(bs.BeautifulSoup("<b>Notes for the Chapter:</b>"))
+                chapter.append(chapnotes)
+        
+        footnotes = soup.find('div', {'id' : "work_endnotes"})
+        chapfoot = soup.find('div', {'class' : "end notes module"})
+		
+        soup = soup.find('div', {'class' : "userstuff module"})
+        chtext = soup.find('h3', {'class' : "landmark heading"})
+        if chtext:
+            chtext.extract()
+        chapter.append(soup)
+		
+        if chapfoot != None:
+            chapfoot = chapfoot.find('blockquote')
+            chapter.append(bs.BeautifulSoup("<b>Notes for the Chapter:</b>"))
+            chapter.append(chapfoot)
+		
+        if footnotes != None:
+            footnotes = footnotes.find('blockquote')
+            chapter.append(bs.BeautifulSoup("<b>Author's Note:</b>"))
+            chapter.append(footnotes)
+			
+        if None == soup:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return utf8FromSoup(chapter)

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -51,11 +51,11 @@ from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 # updated to reflect the class below it.  That, plus getSiteDomain()
 # take care of 'Registering'.
 def getClass():
-    return FanficCastleTVNetAdapter # XXX
+    return CastleFansOrgAdapter # XXX
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
+class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -74,10 +74,10 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
         logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/fanfic/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','csltv') # XXX
+        self.story.setMetadata('siteabbrev','cslf') # XXX
 
         # If all stories from the site fall into the same category,
         # the site itself isn't likely to label them as such, so we
@@ -91,13 +91,13 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'fanfic.castletv.net' # XXX
+        return 'castlefans.org' # XXX
 
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -120,7 +120,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
         params['cookiecheck'] = '1'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfic/user.php?action=login'
         logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
@@ -192,7 +192,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfic/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -232,6 +232,12 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             ## Not all sites use Genre, but there's no harm to
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
@@ -266,6 +272,26 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfic/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -70,7 +70,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         url = self.origurl
         logging.debug("URL: "+url)
-        
+
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
             data = self._fetchUrl(url)
@@ -86,7 +86,29 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             
         if "Chapter not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
-            
+
+        try:
+            # rather nasty way to check for a newer chapter.  ffnet has a
+            # tendency to send out update notices in email before all
+            # their servers are showing the update on the first chapter.
+            try:
+                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+            # get chapter part of url.
+            except:
+                chapcount = 1
+            chapter = url.split('/',)[5]
+            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                           self.story.getMetadata('storyId'),
+                                           chapcount+1)
+            print('=Trying newer chapter: %s' % tryurl)
+            newdata = self._fetchUrl(tryurl)
+            if "Chapter not found. Please check to see you are not using an outdated url." \
+                    not in newdata:
+                print('=======Found newer chapter: %s' % tryurl)
+                soup = bs.BeautifulSoup(newdata)
+        except:
+            pass
+        
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -175,6 +197,12 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         a = soup.find('a', href='http://www.fictionratings.com/')
         self.story.setMetadata('rating',a.string)
 
+        # used below to get correct characters.
+        metatext = a.findNext(text=re.compile(r' - Reviews:'))
+        if metatext == None: # indicates there's no Reviews, look for id: instead.
+            metatext = a.findNext(text=re.compile(r' - id:'))
+        #print("========= metatext:\n%s"%metatext)
+
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
         if 'Complete' in a.findNext(text=re.compile(r'id:'+self.story.getMetadata('storyId'))):
@@ -183,22 +211,36 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             self.story.setMetadata('status', 'In-Progress')
 
         # Parse genre(s) from <meta name="description" content="..."
-        # <meta name="description" content="Chapter 1 of a Harry Potter  - Family/Friendship fanfiction. Dudley Dursley would be the first to say he lived a very normal life. But what happens when he gets invited to his cousin Harry Potter's wedding? Will Dudley get the courage to apologize for the torture he caused all those years ago? Harry/Ginny story..">
-        # <meta name="description" content="A Gundam Wing/AC and Gundam Seed  - Romance/Sci-Fi crossover fanfiction  with characters:  & Kira Y.. Story summary: One-Shoot dividido en dos partes. Kira va en camino a rescatar a Lacus, pero él no es el unico. Dos personajes de diferentes universos Gundams. SEED vs ZERO.">
-        # <meta name="description" content="Chapter 1 of a Alvin and the chipmunks and Alpha and Omega  crossover fanfiction  with characters: Alvin S. & Humphrey. You'll just have to read to find out... No Flames Plesae... and tell me what you want to see by PM'ing me....">
-        # genre is after first -, but before first 'fanfiction'.
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?)) (?:crossover )?fanfiction",
+        # <meta name="description" content="A Transformers/Beast Wars  - Humor fanfiction with characters Prowl & Sideswipe. Story summary: Sideswipe is bored. Prowl appears to be so, too  or at least, Sideswipe thinks he looks bored . So Sideswipe entertains them. After all, what's more fun than a race? Song-fic.">
+        # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
+        # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
+        # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
+        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?:[ ]+with characters (?P<char1>.*?\.?)(?: & (?P<char2>.*?\.?))?\. )?",
                      soup.find('meta',{'name':'description'})['content'])
         if m != None:
             genres=m.group('genres')
-            # Hurt/Comfort is one genre.
-            genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
-            for g in genres.split('/'):
-                self.story.addToList('genre',g)
+            if genres != None:
+                # Hurt/Comfort is one genre.
+                genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
+                for g in genres.split('/'):
+                    self.story.addToList('genre',g)
+
+            if m.group('char1') != None:
+                # At this point we've proven that there's character(s)
+                # We can't reliably parse characters out of meta name="description".
+                # There's no way to tell that "with characters Ichigo K. & Neliel T. O./Nel. " ends at "Nel.", not "T."
+                # But we can pull them from the reviewstext line, now that we know about existance of chars.
+                # reviewstext can take form of:
+                # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
+                # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
+                mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
+                              metatext)
+                chars = mc.group("chars")
+                for c in chars.split(' & '):
+                    self.story.addToList('characters',c)
         
         return
 
-
     def getChapterText(self, url):
         logging.debug('Getting chapter text from: %s' % url)
         time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -152,6 +152,12 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             for g in m.group(1).split(','):
                 self.story.addToList('genre',g)
         
+        m = re.match(r".*?Characters: (.*?) -.*?",metastr)
+        if m:
+            for g in m.group(1).split(','):
+                if g:
+                    self.story.addToList('characters',g)
+        
         m = re.match(r".*?Published: ([0-9/]+?) -.*?",metastr)
         if m:
             self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y/%m/%d"))

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -20,6 +20,7 @@ import logging
 import re
 import urllib2
 import cookielib as cl
+import datetime
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
@@ -87,8 +88,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             raise exceptions.AdultCheckRequired(self.url)
         
         soup = bs.BeautifulSoup(data).find("div", {"class":"content_box post_content_box"})
-        
-        title, author = [link.text for link in soup.find("h2").findAll("a")]
+
+        titleheader = soup.find("h2")
+        title = titleheader.find("a", href=re.compile(r'^/story/')).text
+        author = titleheader.find("a", href=re.compile(r'^/user/')).text
         self.story.setMetadata("title", title)
         self.story.setMetadata("author", author)
         self.story.setMetadata("authorId", author) # The author's name will be unique
@@ -105,7 +108,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         for character in [character_icon['title'] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
             self.story.addToList("characters", character)
         for category in [category.text for category in soup.find("div", {"class":"categories"}).findAll("a")]:
-            self.story.addToList("category", category)
+            self.story.addToList("genre", category)
         self.story.addToList("category", "My Little Pony")
         
         
@@ -126,7 +129,9 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         status = status_bar.text.split("|")[0].strip().replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
         self.story.setMetadata('status', status)
         self.story.setMetadata('rating', status_bar.span.text)
-        self.story.setMetadata('numWords', status_bar.div.b.text)
+        # This way is less elegant, perhaps, but more robust in face of format changes.
+        numWords = status_bar.find("div",{"class":"word_count"}).b.text
+        self.story.setMetadata('numWords', numWords)
         
         description_soup = soup.find("div", {"class":"description"})
         # Sometimes the description has an expanding element
@@ -138,18 +143,27 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             pass
         self.story.setMetadata('description', description_soup.text)
         
-        # Unfortunately, nowhere on the page is the year mentioned. Because we would much rather update the story needlessly
-        # than miss an update, we hardcode the year of creation and update to be 2011.
+        # Unfortunately, nowhere on the page is the year mentioned.
+        # Best effort to deal with this:
+        # Use this year, if that's a date in the future, subtract one year.
+        # Their earliest story is Jun, so they'll probably change the date
+        # around then.
 
+        now = datetime.datetime.now()
+        
         # Get the date of creation from the first chapter
         datePublished_text = chapterDates[0]
         day, month = datePublished_text.split()
         day = re.sub(r"[^\d.]+", '', day)
-        datePublished = makeDate("2011"+month+day, "%Y%b%d")
+        datePublished = makeDate("%s%s%s"%(now.year,month,day), "%Y%b%d")
+        if datePublished > now :
+            datePublished = datePublished.replace(year=now.year-1)
         self.story.setMetadata("datePublished", datePublished)
         dateUpdated_soup = bs.BeautifulSoup(data).find("div", {"class":"calendar"})
         dateUpdated_soup.find('span').extract()
-        dateUpdated = makeDate("2011"+dateUpdated_soup.text, "%Y%b%d")
+        dateUpdated = makeDate("%s%s"%(now.year,dateUpdated_soup.text), "%Y%b%d")
+        if dateUpdated > now :
+            dateUpdated = datePublished.replace(year=now.year-1)
         self.story.setMetadata("dateUpdated", dateUpdated)
         
     def getChapterText(self, url):
@@ -158,4 +172,4 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return utf8FromSoup(soup)
-        
+        

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -159,6 +159,11 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
                     for g in m.group(1).split(','):
                         self.story.addToList('genre',g)
         
+                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+                if m:
+                    for g in m.group(1).split(','):
+                        self.story.addToList('characters',g)
+        
                 m = re.match(r".*?Warnings: (.+).*?",metastr)
                 if m:
                     for w in m.group(1).split(','):

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -145,6 +145,16 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    if char == "!Snape and Harry (required)":
+                        self.story.addToList('characters',"Snape")
+                        self.story.addToList('characters',"Harry")
+                    else:
+                        self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                 genrestext = [genre.string for genre in genres]
@@ -166,7 +176,27 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfiction/'+a['href']
 
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return SiyeCoUkAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8",]# 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        # self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        # self.password = ""
+        # self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','siye') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.siye.co.uk' # XXX
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.siye.co.uk','siye.co.uk']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+r"(www\.)?siye\.co\.uk/(siye/)?"+re.escape("viewstory.php?sid=")+r"\d+$"
+
+    # ## Login seems to be reasonably standard across eFiction sites.
+    # def needToLoginCheck(self, data):
+    #     if 'Registered Users Only' in data \
+    #             or 'There is no such account on our website' in data \
+    #             or "That password doesn't match the one in our database" in data:
+    #         return True
+    #     else:
+    #         return False
+        
+    # def performLogin(self, url):
+    #     params = {}
+
+    #     if self.password:
+    #         params['penname'] = self.username
+    #         params['password'] = self.password
+    #     else:
+    #         params['penname'] = self.getConfig("username")
+    #         params['password'] = self.getConfig("password")
+    #     params['cookiecheck'] = '1'
+    #     params['submit'] = 'Submit'
+    
+    #     loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+    #     logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+    #                                                           params['penname']))
+    
+    #     d = self._fetchUrl(loginUrl, params)
+    
+    #     if "Member Account" not in d : #Member Account
+    #         logging.info("Failed to login to URL %s as %s" % (loginUrl,
+    #                                                           params['penname']))
+    #         raise exceptions.FailedToLogin(url,params['penname'])
+    #         return False
+    #     else:
+    #         return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # if self.is_adult or self.getConfig("is_adult"):
+        #     # Weirdly, different sites use different warning numbers.
+        #     # If the title search below fails, there's a good chance
+        #     # you need a different number.  print data at that point
+        #     # and see what the 'click here to continue' url says.
+        #     addurl = "&ageconsent=ok&warning=4" # XXX
+        # else:
+        #     addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        # Except it doesn't this time. :-/
+        url = self.url #+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # if self.needToLoginCheck(data):
+        #     # need to log in for this one.
+        #     self.performLogin(url)
+        #     data = self._fetchUrl(url)
+
+        # # The actual text that is used to announce you need to be an
+        # # adult varies from site to site.  Again, print data before
+        # # the title search to troubleshoot.
+        # if "Age Consent Required" in data: # XXX 
+        #     raise exceptions.AdultCheckRequired(self.url)
+            
+        # if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+        #     raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/siye/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # need(or easier) to pull other metadata from the author's list page.
+        authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+
+        ## Title
+        titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',titlea.string)
+        
+        # Find the chapters (from soup, not authsoup):
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/siye/'+chapter['href']))
+
+        if self.chapterUrls:
+            self.story.setMetadata('numChapters',len(self.chapterUrls))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+            self.story.setMetadata('numChapters',1)
+
+        # The stuff we can get from the chapter list/one-shot page are
+        # in the first table with 95% width.
+        metatable = soup.find('table',{'width':'95%'})
+
+        # Categories
+        cat_as = metatable.findAll('a', href=re.compile(r'categories.php'))
+        for cat_a in cat_as:
+            self.story.addToList('category',stripHTML(cat_a))
+
+        moremetaparts = stripHTML(metatable).split('\n')
+        for part in moremetaparts:
+            part = part.strip()
+            if part.startswith("Characters:"):
+                part = part[part.find(':')+1:]
+                for item in part.split(','):
+                    if item.strip() == "Harry/Ginny":
+                        self.story.addToList('characters',"Harry")
+                        self.story.addToList('characters',"Ginny")
+                    elif item.strip() not in ("None","All"):
+                        self.story.addToList('characters',item)
+
+            if part.startswith("Genres:"):
+                part = part[part.find(':')+1:]
+                for item in part.split(','):
+                    if item.strip() != "None":
+                        self.story.addToList('genre',item)
+
+            if part.startswith("Warnings:"):
+                part = part[part.find(':')+1:]
+                for item in part.split(','):
+                    if item.strip() != "None":
+                        self.story.addToList('warnings',item)
+
+            if part.startswith("Rating:"):
+                part = part[part.find(':')+1:]
+                self.story.setMetadata('rating',part)
+
+            if part.startswith("Summary:"):
+                part = part[part.find(':')+1:]
+                self.story.setMetadata('description',part)
+                
+        # want to get the next tr of the table.
+        #print("%s"%titlea.parent.parent.findNextSibling('tr'))
+            
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        moremeta = stripHTML(titlea.parent.parent.parent.find('div',{'class':'desc'}))
+        for part in moremeta.replace(' - ','\n').split('\n'):
+            #print("part:%s"%part)
+            try:
+                (name,value) = part.split(': ')
+            except:
+                # not going to worry about fancier processing for the bits
+                # that don't match.
+                continue
+            name=name.strip()
+            value=value.strip()
+            if name == 'Published':
+                self.story.setMetadata('datePublished', makeDate(value, self.dateformat))
+            if name == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat))
+            if name == 'Completed':
+                if value == 'Yes':
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+            if name == 'Words':
+                self.story.setMetadata('numWords', value)
+
+        try:
+            # Find Series name from series URL.
+            a = titlea.findPrevious('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass        
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        # soup = bs.BeautifulSoup(self._fetchUrl(url))
+        # BeautifulSoup objects to <p> inside <span>, which
+        # technically isn't allowed.
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        # not the most unique thing in the world, but it appears to be
+        # the best we can do here.
+        story = soup.find('span', {'style' : 'font-size: 100%;'})
+        
+        if None == story:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return utf8FromSoup(story)

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -178,6 +178,12 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                 genrestext = [genre.string for genre in genres]
@@ -199,6 +205,26 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
 
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_test1.py

@@ -67,9 +67,11 @@ class TestSiteAdapter(BaseSiteAdapter):
             raise exceptions.FailedToLogin(self.url,self.username)
 
         if self.story.getMetadata('storyId') == '664':
-            self.story.setMetadata(u'title',"Test Story Title "+self.crazystring)
+            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring)
+            self.story.setMetadata('author','Test Author aa bare amp(&) quote(') amp(&)')
         else:
             self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
+            self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('storyUrl',self.url)
         self.story.setMetadata('description',u'Description '+self.crazystring+u''' Done
 
@@ -82,10 +84,17 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         else:
             self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
         self.story.setMetadata('numWords','123456')
-        self.story.setMetadata('status','In-Completed')
+
+        idnum = int(self.story.getMetadata('storyId'))
+        if idnum % 2 == 1:
+            self.story.setMetadata('status','In-Progress')
+        else:
+            self.story.setMetadata('status','Completed')
+            
+        self.setSeries('The Great Test',idnum)
+        
         self.story.setMetadata('rating','Tweenie')
         
-        self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('authorId','98765')
         self.story.setMetadata('authorUrl','http://author/url')
 

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -123,6 +123,11 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):] 
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
@@ -175,6 +180,12 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                 genrestext = [genre.string for genre in genres]
@@ -196,12 +207,37 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/library/'+a['href']
 
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -54,13 +54,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         return 'www.tthfanfic.org'
 
     def getSiteExampleURLs(self):
-        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm ttp://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
+        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm http://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
 
     # http://www.tthfanfic.org/T-526321777848988007890480555880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm
     # http://www.tthfanfic.org/Story-5583
     # http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm
+    # http://www.tthfanfic.org/story.php?no=26093
     def getSiteURLPattern(self):
-        return r"http://www.tthfanfic.org/(T-\d+/)?Story-(?P<id>\d+)(-\d+)?(/.*)?$"
+        return r"http://www.tthfanfic.org(/(T-\d+/)?Story-|/story.php\?no=)(?P<id>\d+)(-\d+)?(/.*)?$"
 
     # tth won't send you future updates if you aren't 'caught up'
     # on the story.  Login isn't required for F21, but logging in will
@@ -151,13 +152,13 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
 
         try:
             # going to pull part of the meta data from author list page.
-            logging.debug("author URL: "+self.story.getMetadata('authorUrl'))
+            logging.debug("**AUTHOR** URL: "+self.story.getMetadata('authorUrl'))
             authordata = self._fetchUrl(self.story.getMetadata('authorUrl'))
             authorsoup = bs.BeautifulSoup(authordata)
             # author can have several pages, scan until we find it.
             while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
                 nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
-                logging.debug("author nextpage URL: "+nextpage)
+                logging.debug("**AUTHOR** nextpage URL: "+nextpage)
                 authordata = self._fetchUrl(nextpage)
                 authorsoup = bs.BeautifulSoup(authordata)
         except urllib2.HTTPError, e:
@@ -213,6 +214,12 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                 self.chapterUrls.append((stripHTML(o),url))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        pseries = soup.find('p', {'style':'margin-top:0px'})
+        m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
+                     pseries.text)
+        if m:
+            self.setSeries(m.group('series'),m.group('num'))
         
         return
 

fanficdownloader/adapters/adapter_twilightednet.py

@@ -118,6 +118,12 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        # twilighted isn't writing <body> ??? wtf?
+        data = "<html><body>"+data[data.index("</head>"):] 
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
@@ -170,6 +176,12 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             ## twilighted.net doesn't use genre.
             # if 'Genre' in label:
             #     genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
@@ -192,12 +204,38 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+            
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+        
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
 
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        # twilighted isn't writing <body> ??? wtf?
+        data = "<html><body>"+data[data.index("</head>"):] 
+        
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -84,7 +84,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
@@ -117,7 +117,12 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
 
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
+
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):] 
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
@@ -178,6 +183,12 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
                 genrestext = [genre.string for genre in genres]
@@ -206,12 +217,37 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                 value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
 
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})

fanficdownloader/adapters/adapter_whoficcom.py

@@ -158,6 +158,15 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
         for g in genre.split(r', '):
             self.story.addToList('genre',g)
 
+        # line 3 is characters.
+        chars = metadatachunks[3]
+        charsearch="<i>Characters:</i>"
+        if charsearch in chars:
+            chars = chars[metadatachunks[3].index(charsearch)+len(charsearch):]
+            for c in chars.split(','):
+                if c.strip() != u'None':
+                    self.story.addToList('characters',c)
+            
         # the next line is stuff with ' - ' separators *and* names--with tags.
         moremeta = metadatachunks[5]
         moremeta = re.sub(r'<[^>]+>','',moremeta) # strip tags.
@@ -180,6 +189,26 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
             if name == 'Word Count':
                 self.story.setMetadata('numWords', value)
 
+        try:
+            # Find Series name from series URL.
+            a = metadata.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/base_adapter.py

@@ -68,6 +68,10 @@ class BaseSiteAdapter(Configurable):
         self.addConfigSection(self.getSiteDomain())
         self.addConfigSection("overrides")
         
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
         self.opener = u2.build_opener(u2.HTTPCookieProcessor())
         self.storyDone = False
         self.metadataDone = False
@@ -224,8 +228,34 @@ class BaseSiteAdapter(Configurable):
     def getChapterText(self, url):
         "Needs to be overriden in each adapter class."
         pass
-        
+
+    # Just for series, in case we choose to change how it's stored or represented later.
+    def setSeries(self,name,num):
+        if self.getConfig('collect_series'):
+            self.story.setMetadata('series','%s [%s]'%(name, num))
+
+fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"05",
+           "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
+           "November":"11", "December":"12" }
+
 def makeDate(string,format):
+    # Surprise!  Abstracting this turned out to be more useful than
+    # just saving bytes.
+
+    # fudge english month names for people who's locale is set to
+    # non-english.  All our current sites date in english, even if
+    # there's non-english content.
+    do_abbrev = "%b" in format
+        
+    if "%B" in format or do_abbrev:
+        format = format.replace("%B","%m").replace("%b","%m")
+        for (name,num) in fullmon.items():
+            if do_abbrev:
+                name = name[:3] # first three for abbrev
+            if name in string:
+                string = string.replace(name,num)
+                break
+            
     return datetime.datetime.strptime(string,format)
 
 acceptable_attributes = ['href','name']

fanficdownloader/story.py

@@ -17,7 +17,7 @@
 
 import os
 
-from htmlcleanup import conditionalRemoveEntities
+from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 
 class Story:
     
@@ -25,20 +25,22 @@ class Story:
         try:
             self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
         except:
-            self.metadata = {'version':'4.1'}
+            self.metadata = {'version':'4.3'}
         self.chapters = [] # chapters will be tuples of (title,html)
         self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
 
     def setMetadata(self, key, value):
+        ## still keeps < < and &
         self.metadata[key]=conditionalRemoveEntities(value)
 
     def getMetadataRaw(self,key):
         if self.metadata.has_key(key):
             return self.metadata[key]
         
-    def getMetadata(self, key):
+    def getMetadata(self, key, removeallentities=False):
+        value = None
         if self.getLists().has_key(key):
-            return ', '.join(self.getList(key))
+            value = ', '.join(self.getList(key))
         if self.metadata.has_key(key):
             value = self.metadata[key]
             if value:
@@ -48,16 +50,33 @@ class Story:
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
                  value = value.strftime("%Y-%m-%d")
+                 
+        if removeallentities and value != None:
+            return removeAllEntities(value)
+        else:
             return value
 
+    def getAllMetadata(self, removeallentities=False):
+        '''
+        All single value *and* list value metadata as strings.
+        '''
+        allmetadata = {}
+        for k in self.metadata.keys():
+            allmetadata[k] = self.getMetadata(k, removeallentities)
+        for l in self.listables.keys():
+            allmetadata[l] = self.getMetadata(l, removeallentities)
+
+        return allmetadata
+        
     def addToList(self,listname,value):
         if value==None:
             return
+        value = conditionalRemoveEntities(value)
         if not self.listables.has_key(listname):
             self.listables[listname]=[]
         # prevent duplicates.
         if not value in self.listables[listname]:
-            self.listables[listname].append(conditionalRemoveEntities(value))
+            self.listables[listname].append(value)
 
     def getList(self,listname):
         if not self.listables.has_key(listname):

fanficdownloader/writers/base_writer.py

@@ -50,6 +50,7 @@ class BaseStoryWriter(Configurable):
             'category',
             'genre',
             'characters',
+            'series',
             'status',
             'datePublished',
             'dateUpdated',
@@ -77,6 +78,7 @@ class BaseStoryWriter(Configurable):
             'category':'Category',
             'genre':'Genre',
             'status':'Status',
+            'series':'Series',
             'characters':'Characters',
             'datePublished':'Published',
             'dateUpdated':'Updated',
@@ -97,10 +99,13 @@ class BaseStoryWriter(Configurable):
             'formatname':'File Format',
             'formatext':'File Extension',
             'siteabbrev':'Site Abbrev',
-            'version':'FFD Version'
+            'version':'FFDL Version'
             }
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())
+        
+        for tag in self.getConfigList("extratags"):
+            self.story.addToList("extratags",tag)
 
     def getMetadata(self,key):
         return stripHTML(self.story.getMetadata(key))
@@ -118,7 +123,7 @@ class BaseStoryWriter(Configurable):
         return self.formatFileName(self.getConfig('zip_filename'))
 
     def formatFileName(self,template):
-        values = self.story.metadata
+        values = origvalues = self.story.getAllMetadata()
         # fall back default:
         if not template:
             template="${title}-${siteabbrev}_${storyId}${formatext}"
@@ -126,7 +131,7 @@ class BaseStoryWriter(Configurable):
         if not self.getConfig('allow_unsafe_filename'):
             values={}
             pattern = re.compile(r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+")
-            for k in self.story.metadata.keys():
+            for k in origvalues.keys():
                 values[k]=re.sub(pattern,'_', removeAllEntities(self.story.getMetadata(k)))
 
         return string.Template(template).substitute(values).encode('utf8')
@@ -183,12 +188,11 @@ class BaseStoryWriter(Configurable):
             self._write(out,END.substitute(self.story.metadata))
 
     # if no outstream is given, write to file.
-    def writeStory(self,outstream=None,metaonly=False):
-        for tag in self.getConfigList("extratags"):
-            self.story.addToList("extratags",tag)
+    def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False):
 
         self.metaonly = metaonly
-        outfilename=self.getOutputFileName()
+        if outfilename == None:
+            outfilename=self.getOutputFileName()
 
         if not outstream:
             close=True
@@ -202,7 +206,7 @@ class BaseStoryWriter(Configurable):
                         os.mkdir(path) ## os.makedirs() doesn't work in 2.5.2?
 
             ## Check for output file date vs updated date here
-            if not self.getConfig('always_overwrite'):
+            if not (self.getConfig('always_overwrite') or forceOverwrite):
                 if os.path.exists(outfilename):
                     ## date() truncs off time, which files have, but sites don't report.
                     lastupdated=self.story.getMetadataRaw('dateUpdated').date()
@@ -248,21 +252,25 @@ class BaseStoryWriter(Configurable):
     def getTags(self):
         # set to avoid duplicates subject tags.
         subjectset = set()
+        
+        if self.story.getMetadataRaw('dateUpdated'):
+            # Last Update tags for Bill.
+            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update Year/Month: %Y/%m"))
+            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update: %Y/%m/%d"))
+        
         for entry in self.validEntries:
             if entry in self.getConfigList("include_subject_tags") and \
                     entry not in self.story.getLists() and \
                     self.story.getMetadata(entry):
                 subjectset.add(self.getMetadata(entry))
+                
         # listables all go into dc:subject tags, but only if they are configured.
         for (name,lst) in self.story.getLists().iteritems():
             if name in self.getConfigList("include_subject_tags"):
                 for tag in lst:
                     subjectset.add(tag)
-                    
-        for tag in self.getConfigList("extratags"):
-            subjectset.add(tag)
 
-        return subjectset
+        return list(subjectset)
             
     def writeStoryImpl(self, out):
         "Must be overriden by sub classes."

fanficdownloader/writers/writer_epub.py

@@ -249,9 +249,6 @@ h6 { text-align: center; }
             metadata.appendChild(newTag(contentdom,"meta",
                                         attrs={"name":"calibre:timestamp",
                                                "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")}))
-            # Last Update tags for Bill.
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update Year/Month: %Y/%m"))
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update: %Y/%m/%d"))
         
         if self.getMetadata('description'):
             metadata.appendChild(newTag(contentdom,"dc:description",text=

index-ajax.html

@@ -4,7 +4,7 @@
 		<link href="css/index.css" rel="stylesheet" type="text/css">
 		<link type="text/css" href="http://jqueryui.com/latest/themes/base/ui.all.css" rel="stylesheet" />
 	 
-		<title>Fanfiction Downloader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
+		<title>FanFictionDownLoader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
 		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
 		<script src="/js/jquery-1.3.2.js"></script>
 		<script src="/js/fdownloader.js"></script>
@@ -16,7 +16,7 @@
 	<body>
 		<div id='main'>
 			<h1>
-				<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+				<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
 			</h1>
 				
 <!--			<form action="/fdown" method="post"> -->
@@ -91,8 +91,8 @@
 					<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 					alt="Powered by Google App Engine" />
 					<br/><br/>
-					FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-					Copyright &copy; <a href="http://twitter.com/sigizmund">Roman Kirillov</a>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; Fanficdownloader team
 				</div>
 <!--			</form> -->
 		</div>

index.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -26,7 +26,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a> <g:plusone size="medium"></g:plusone>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a> <g:plusone size="medium"></g:plusone>
       </h1>
 
       <div style="text-align: center">
@@ -50,33 +50,38 @@
       <form action="/fdown" method="post">
 	<div id='urlbox'>
 	  <div id='greeting'>
-	    <p>Hi, {{ nickname }}! This is a fan fiction downloader, which makes reading stories from various websites
+	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>This is the Official Multithreading Version</h3>
+	  <h3>Support for 'Series'</h3>
 	  <p>
-	    This version of the application uses Python 2.7 and
-	    multithreading to try and reduce our usage.  Google
-	    considers Python 2.7 Experimental still, so there may be issues.
+	    We now collect 'Series' name and number for the sites:
+	      harrypotterfanfiction.com,
+	      potionsandsnitches.net,
+	      adastrafanfic.com,
+	      whofic.com,
+	      fanfiction.tenhawkpresents.com,
+	      castlefans.org,
+	      tthfanfic.org,
+	      www.siye.co.uk,
+	      twilighted.net*,
+	      twilighted.net* and 
+	      thewriterscoffeeshop.com*.
 	  </p>
 	  <p>
-	    <b>Good news!</b><br />
-	    The issue that was causing problems with downloading large stories
-	    has been fixed.
+	    * The last three use series as reading lists and stories collections as much as true story series,
+	    so they default to <i>not</i> collect series info.  You can turn it on in your User Configuration if you want.
 	  </p>
+	  <h3>New Site: <a href="http://archiveofourown.org">archiveofourown.org</a></h3>
 	  <p>
-	    <b>New Feature</b><br /> You can now set a custom
-	    parameter for background_color that will be used with html
-	    and epub output.  (Note: many epub readers ignore the bg color.)
+	    Thanks to Ida Leter for writing the code to support a new site: <a href="http://archiveofourown.org">archiveofourown.org</a>.
 	  </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
-	    the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
-	      Downloader Google Group</a>.  The
-	    <a href="http://4-0-7.fanfictiondownloader.appspot.com">Previous
-	      Version</a> is also available for you to use if necessary.
+	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
+	    <a href="http://4-3-0.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -112,14 +117,33 @@
       <div id='urlbox'>
 	<div id='greeting'>
 	  <p>
-	    This is a fan fiction downloader, which makes reading stories from various websites much easier. Before you
-	    can start downloading fanfics, you need to login, so downloader can remember your fanfics and store them.
+	    This is a FanFictionDownLoader, which makes reading stories from various websites much easier. Before you
+	    can start downloading fanfics, you need to login, so FanFictionDownLoader can remember your fanfics and store them.
 	  </p>
 	  <p><a href="{{ login_url }}">Login using Google account</a></p>
 	</div>
       </div>
       {% endif %}
       
+      <div id='typebox'>
+	  <p>
+	    <b>FanFictionDownLoader calibre Plugin</b>
+	    <br /><br />
+	    
+	    There's now a version of this downloader that runs
+	    entirely inside the
+	    popular <a href="http://calibre-ebook.com/">calibre</a>
+	    ebook management package as a plugin.
+	    
+	    <br /><br />
+	    
+	    Once you have calibre installed and running, inside
+	    calibre, you can go to 'Get plugins to enhance calibre' or
+	    'Get new plugins' and
+	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
+	    
+	  </p>
+      </div>
       <div id='helpbox'>
 	<dl>
 	  <dt>fictionalley.org</dt>
@@ -196,10 +220,10 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://fanfiction.tenhawkpresents.com/viewstory.php?sid=294">http://fanfiction.tenhawkpresents.com/viewstory.php?sid=294</a>.
 	  </dd>
-	  <dt>fanfic.castletv.net</dt>
+	  <dt>castlefans.org</dt>
 	  <dd>
 	    Use the URL of the story's chapter list, such as
-	    <br /><a href="http://fanfic.castletv.net/viewstory.php?sid=123">http://fanfic.castletv.net/viewstory.php?sid=123</a>.
+	    <br /><a href="http://castlefans.org/fanfic/viewstory.php?sid=123">http://castlefans.org/fanfic/viewstory.php?sid=123</a>.
 	  </dd>
 	  <dt>fimfiction.net</dt>
 	  <dd>
@@ -215,6 +239,17 @@
 	    <br /><a href="http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm">http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm</a>.
 	    <br /><a href="http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm">http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm</a>.
 	  </dd>
+	  <dt>www.siye.co.uk</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.siye.co.uk/siye/viewstory.php?sid=123">http://www.siye.co.uk/siye/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>archiveofourown.org</dt>
+	  <dd>
+	    Use the URL of the story, or one of it's chapters, such as
+	    <br /><a href="http://archiveofourown.org/works/76366">http://archiveofourown.org/works/76366</a>.
+	    <br /><a href="http://archiveofourown.org/works/76366/chapters/101584">http://archiveofourown.org/works/76366/chapters/101584</a>.
+	  </dd>
 	</dl>	
 
 	
@@ -251,8 +286,8 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; FanFictionDownLoader team
       </div>
 
       <div style="margin-top: 1em; text-align: center'">

login.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Login Needed Fanfiction Downloader</title>
+    <title>Login Needed FanFictionDownLoader</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -22,7 +22,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
 
       <div style="text-align: center">
@@ -88,8 +88,8 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; FanFictionDownLoader team
       </div>
 
       <div style="margin-top: 1em; text-align: center'">

makeplugin.py

@@ -23,11 +23,11 @@ from glob import glob
 from makezip import createZipFile
 
 if __name__=="__main__":
-    filename="FanFictionDownLoaderPlugin.zip"
+    filename="FanFictionDownLoader.zip"
     exclude=['*.pyc','*~','*.xcf']
     # from top dir. 'w' for overwrite
     createZipFile(filename,"w",
-                  ['defaults.ini','example.ini','epubmerge.py','fanficdownloader'],
+                  ['plugin-defaults.ini','plugin-example.ini','epubmerge.py','fanficdownloader'],
                   exclude=exclude)
     #from calibre-plugin dir. 'a' for append
     os.chdir('calibre-plugin')

makezip.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# epubmerge.py 1.0
+
+# Copyright 2011, Jim Miller
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, zipfile, sys
+from glob import glob
+
+def addFolderToZip(myZipFile,folder,exclude=[]):
+    folder = folder.encode('ascii') #convert path to ascii for ZipFile Method
+    excludelist=[]
+    for ex in exclude:
+        excludelist.extend(glob(folder+"/"+ex))
+    for file in glob(folder+"/*"):
+        if file in excludelist:
+            continue
+        if os.path.isfile(file):
+            #print file
+            myZipFile.write(file, file, zipfile.ZIP_DEFLATED)
+        elif os.path.isdir(file):
+            addFolderToZip(myZipFile,file,exclude=exclude)
+
+def createZipFile(filename,mode,files,exclude=[]):
+    myZipFile = zipfile.ZipFile( filename, mode ) # Open the zip file for writing
+    excludelist=[]
+    for ex in exclude:
+        excludelist.extend(glob(ex))
+    for file in files:
+        if file in excludelist:
+            continue
+        file = file.encode('ascii') #convert path to ascii for ZipFile Method
+        if os.path.isfile(file):
+            (filepath, filename) = os.path.split(file)
+            #print file
+            myZipFile.write( file, filename, zipfile.ZIP_DEFLATED )
+        if os.path.isdir(file):
+            addFolderToZip(myZipFile,file,exclude=exclude)
+    myZipFile.close()
+    return (1,filename)
+

plugin-defaults.ini

@@ -0,0 +1,263 @@
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[defaults]
+
+## [defaults] section applies to all formats and sites but may be
+## overridden at several levels
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## All available titlepage_entries and the label used for them:
+## <entryname>_label:<label>
+## Labels may be customized.
+title_label:Title
+storyUrl_label:Story URL
+description_label:Summary
+author_label:Author
+authorUrl_label:Author URL
+## epub, txt, html
+formatname_label:File Format
+## .epub, .txt, .html
+formatext_label:File Extension
+## Category and Genre have overlap, depending on the site.
+## Sometimes Harry Potter is a category and Fantasy a genre. (fanfiction.net)
+## Sometimes Fantasy is category *and* a genre (fictionpress.com)
+## Sometimes there are multiple categories and/or genres.
+category_label:Category
+genre_label:Genre
+characters_label:Characters
+series_label:Series
+## Completed/In-Progress
+status_label:Status
+## Dates story first published, last updated, and downloaded(last with time).
+datePublished_label:Published
+dateUpdated_label:Updated
+dateCreated_label:Packaged
+## Rating depends on the site.  Some use K,T,M,etc, and some PG,R,NC-17
+rating_label:Rating
+## Also depends on the site.  
+warnings_label:Warnings
+numChapters_label:Chapters
+numWords_label:Words
+## www.fanfiction.net, fictionalley.com, etc.
+site_label:Publisher
+## ffnet, fpcom, etc.
+siteabbrev_label:Site Abbrev
+## The site's unique story/author identifier.  Usually a number.
+storyId_label:Story ID
+authorId_label:Author ID
+## Primarily to put specific values in dc:subject tags for epub.  Will
+## show up in Calibre as tags.  Also carried into mobi when converted.
+extratags_label:Extra Tags
+## The version of fanficdownloader
+##
+version_label:FFDL Version
+
+## items to include in the title page
+## Empty entries will *not* appear, even if in the list.
+## All current formats already include title and author.
+titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+collect_series: true
+
+## include title page as first page.
+include_titlepage: true
+
+## include a TOC page before the story text
+include_tocpage: true
+
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+#website_encodings: auto, utf8, Windows-1252
+
+## entries to make epub subjects and calibre tags
+## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
+include_subject_tags: extratags, genre, category, characters, status
+
+## extra tags (comma separated) to include, primarily for epub.
+extratags: FanFiction
+
+## number of seconds to sleep between calls to the story site.  May by
+## useful if pulling large numbers of stories or if the site is slow.
+#slow_down_sleep_time:0.5
+
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Must be hex code, # will be added.
+background_color: ffffff
+
+## Each output format has a section that overrides [defaults]
+[html]
+
+[txt]
+## Add URLs since there aren't links.
+titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+
+## use \r\n for line endings, the windows convention.  text output only.
+windows_eol: true
+
+[epub]
+
+## epub carries the TOC in metadata.
+## mobi generated from epub will have a TOC at the end.
+include_tocpage: false
+
+## epub->mobi conversions typically don't like tables.
+titlepage_use_table: false
+
+## When using tables, make these span both columns.
+wide_titlepage_entries: description, storyUrl, author URL
+
+[mobi]
+## mobi TOC cannot be turned off right now.
+#include_tocpage: true
+
+
+## Each site has a section that overrides [defaults] *and* the format
+## sections test1.com specifically is not a real story site.  Instead,
+## it is a fake site for testing configuration and output.  It uses
+## URLs like: http://test1.com?sid=12345
+[test1.com]
+extratags: FanFiction,Testing
+
+## If necessary, you can define [<site>:<format>] sections to
+## customize the formats differently for the same site.  Overrides
+## defaults, format and site.
+[test1.com:txt]
+extratags: FanFiction,Testing,Text
+
+[test1.com:html]
+extratags: FanFiction,Testing,HTML
+
+[www.fanfiction.net]
+
+[www.fictionpress.com]
+## Clear FanFiction from defaults, fictionpress.com is original fiction.
+extratags:
+
+[www.ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.twilighted.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## twilighted.net (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.twiwrite.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## twiwrite.net (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.whofic.com]
+
+[www.mediaminer.org]
+
+[www.thewriterscoffeeshop.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.fictionalley.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.harrypotterfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
+
+[overrides]
+## It may sometimes be useful to override all of the specific format,
+## site and site:format sections in your private configuration.  For
+## example, this extratags param here would override all of the
+## extratags params in all other sections.  Only commandline options
+## beat overrides.
+#extratags:fanficdownloader

plugin-example.ini

@@ -0,0 +1,74 @@
+## This is an example of what your personal configuration might look
+## like.
+
+[defaults]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+## Turn off in [defaults] or [overrides] to prevent all sites from
+## updating series column.
+## default is true
+#collect_series: false
+
+## Most common, I expect will be using this to save username/passwords
+## for different sites.
+[www.twilighted.net]
+#username:YourPenname
+#password:YourPassword
+## default is false
+#collect_series: true
+
+[www.ficwad.com]
+#username:YourUsername
+#password:YourPassword
+
+[www.twiwrite.net]
+#username:YourName
+#password:yourpassword
+## default is false
+#collect_series: true
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.
+#is_adult:true
+
+[www.thewriterscoffeeshop.com]
+#username:YourName
+#password:yourpassword
+#is_adult:true
+## default is false
+#collect_series: true
+
+[www.fictionalley.org]
+#is_adult:true
+
+[www.harrypotterfanfiction.com]
+#is_adult:true
+
+[www.fimfiction.net]
+#is_adult:true
+
+[www.tthfanfic.org]
+#is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
+
+
+## This section will override anything in the system defaults or other
+## sections here.
+[overrides]
+## default varies by site.  Set true here to force all sites to
+## collect series.
+#collect_series: true

recent.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
+    <title>FanFictionDownLoader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <script type="text/javascript">
 
@@ -21,7 +21,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
       
       <script type="text/javascript"><!--
@@ -45,7 +45,7 @@
 	  <p><a href="/clearrecent">Clear your Recent Downloads List</a></p>
 	</div>
       </div>
-      
+
       <div id='helpbox'>
 	{% for fic in fics %}
 	<p>

status.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>{% if fic.completed %} Finished {% else %} {% if fic.failure %} Failed {% else %} Working... {% endif %} {% endif %} - Fanfiction Downloader</title>
+    <title>{% if fic.completed %} Finished {% else %} {% if fic.failure %} Failed {% else %} Working... {% endif %} {% endif %} - FanFictionDownLoader</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     {% if not fic.completed and not fic.failure %}
@@ -25,7 +25,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
       <div style="text-align: center">
         <script type="text/javascript"><!--
@@ -67,11 +67,12 @@
 	{% endif %}
 	  <p>See your personal list of <a href="/recent">previously downloaded fanfics</a>.</p>
       </div>
+
       <div style='text-align: center'>
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
 	Copyright &copy; Fanficdownloader team
       </div>