Plugin 1.1.4, fix Category title, keep calibre_bookmarks.txt on epub update.

Existing Completed, In-Progress and Last Update tags still removed.
Add plugin-defaults.ini to remove unusable options and default lastupdate off.

app.yaml

@@ -1,20 +1,22 @@
-# fanfictionloader ffd-retief
-application: fanfictionloader
-version: 4-0-7
-runtime: python
+# ffd-retief-hrd fanfictiondownloader
+application: fanfictiondownloader
+version: 4-2-0
+runtime: python27
 api_version: 1
+threadsafe: true
 
 handlers:
+
 - url: /r3m0v3r.*
-  script: utils/remover.py
+  script: utils.remover.app
   login: admin
 
 - url: /tally.*
-  script: utils/tally.py
+  script: utils.tally.app
   login: admin
 
 - url: /fdownloadtask
-  script: main.py
+  script: main.app
   login: admin
   
 - url: /css
@@ -31,7 +33,14 @@ handlers:
   upload: static/favicon\.ico
 
 - url: /.*
-  script: main.py
+  script: main.app
 
 builtins:
 - datastore_admin: on
+
+libraries:
+- name: django
+  version: "1.2"
+
+- name: PIL
+  version: "1.1.7"

calibre-plugin/__init__.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+# The class that all Interface Action plugin wrappers must inherit from
+from calibre.customize import InterfaceActionBase
+
+## Apparently the name for this class doesn't matter--it was still
+## 'demo' for the first few versions.
+class FanFictionDownLoaderBase(InterfaceActionBase):
+    '''
+    This class is a simple wrapper that provides information about the
+    actual plugin class. The actual interface plugin class is called
+    InterfacePlugin and is defined in the ffdl_plugin.py file, as
+    specified in the actual_plugin field below.
+
+    The reason for having two classes is that it allows the command line
+    calibre utilities to run without needing to load the GUI libraries.
+    '''
+    name                = 'FanFictionDownLoader'
+    description         = 'UI plugin to download FanFiction stories from various sites.'
+    supported_platforms = ['windows', 'osx', 'linux']
+    author              = 'Jim Miller'
+    version             = (1, 1, 4)
+    minimum_calibre_version = (0, 8, 30)
+
+    #: This field defines the GUI plugin class that contains all the code
+    #: that actually does something. Its format is module_path:class_name
+    #: The specified class must be defined in the specified module.
+    actual_plugin       = 'calibre_plugins.fanfictiondownloader_plugin.ffdl_plugin:FanFictionDownLoaderPlugin'
+
+    def is_customizable(self):
+        '''
+        This method must return True to enable customization via
+        Preferences->Plugins
+        '''
+        return True
+
+    def config_widget(self):
+        '''
+        Implement this method and :meth:`save_settings` in your plugin to
+        use a custom configuration dialog.
+
+        This method, if implemented, must return a QWidget. The widget can have
+        an optional method validate() that takes no arguments and is called
+        immediately after the user clicks OK. Changes are applied if and only
+        if the method returns True.
+
+        If for some reason you cannot perform the configuration at this time,
+        return a tuple of two strings (message, details), these will be
+        displayed as a warning dialog to the user and the process will be
+        aborted.
+
+        The base class implementation of this method raises NotImplementedError
+        so by default no user configuration is possible.
+        '''
+        # It is important to put this import statement here rather than at the
+        # top of the module as importing the config class will also cause the
+        # GUI libraries to be loaded, which we do not want when using calibre
+        # from the command line
+        from calibre_plugins.fanfictiondownloader_plugin.config import ConfigWidget
+        return ConfigWidget()
+
+    def save_settings(self, config_widget):
+        '''
+        Save the settings specified by the user with config_widget.
+
+        :param config_widget: The widget returned by :meth:`config_widget`.
+        '''
+        config_widget.save_settings()
+
+        # Apply the changes
+        ac = self.actual_plugin_
+        if ac is not None:
+            ac.apply_settings()
+
+# For testing, run from command line with this:
+# calibre-debug -e __init__.py
+# 
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    from calibre.gui2.preferences import test_widget
+    app = QApplication([])
+    test_widget('Advanced', 'Plugins')

calibre-plugin/about.txt

@@ -0,0 +1,21 @@
+<p>FanFictionDownLoader Plugin</p>
+<hr />
+
+<p>Created by Jim Miller, borrowing heavily from Grant Drake's
+'<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
+'<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
+'<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
+plugins.</p>
+
+<p>
+Calibre officially distributes plugins from the mobileread.com forum site.
+The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>
+</p>
+
+<p> However, I monitor the
+<a href="http://groups.google.com/group/fanfic-downloader">general users
+group</a> for the downloader more closely.  That also covers the web application and CLI.
+</p>
+
+The source project for this plugin is <a href="http://code.google.com/p/fanficdownloader/source/checkout">
+also available</a>.

calibre-plugin/common_utils.py

@@ -0,0 +1,447 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import os
+from PyQt4 import QtGui
+from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime)
+from calibre.constants import iswindows
+from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME
+from calibre.gui2.actions import menu_action_unique_name
+from calibre.gui2.keyboard import ShortcutConfig
+from calibre.utils.config import config_dir
+from calibre.utils.date import now, format_date, qt_to_dt, UNDEFINED_DATE
+
+# Global definition of our plugin name. Used for common functions that require this.
+plugin_name = None
+# Global definition of our plugin resources. Used to share between the xxxAction and xxxBase
+# classes if you need any zip images to be displayed on the configuration dialog.
+plugin_icon_resources = {}
+
+
+def set_plugin_icon_resources(name, resources):
+    '''
+    Set our global store of plugin name and icon resources for sharing between
+    the InterfaceAction class which reads them and the ConfigWidget
+    if needed for use on the customization dialog for this plugin.
+    '''
+    global plugin_icon_resources, plugin_name
+    plugin_name = name
+    plugin_icon_resources = resources
+
+
+def get_icon(icon_name):
+    '''
+    Retrieve a QIcon for the named image from the zip file if it exists,
+    or if not then from Calibre's image cache.
+    '''
+    if icon_name:
+        pixmap = get_pixmap(icon_name)
+        if pixmap is None:
+            # Look in Calibre's cache for the icon
+            return QIcon(I(icon_name))
+        else:
+            return QIcon(pixmap)
+    return QIcon()
+
+
+def get_pixmap(icon_name):
+    '''
+    Retrieve a QPixmap for the named image
+    Any icons belonging to the plugin must be prefixed with 'images/'
+    '''
+    global plugin_icon_resources, plugin_name
+
+    if not icon_name.startswith('images/'):
+        # We know this is definitely not an icon belonging to this plugin
+        pixmap = QPixmap()
+        pixmap.load(I(icon_name))
+        return pixmap
+
+    # Check to see whether the icon exists as a Calibre resource
+    # This will enable skinning if the user stores icons within a folder like:
+    # ...\AppData\Roaming\calibre\resources\images\Plugin Name\
+    if plugin_name:
+        local_images_dir = get_local_images_dir(plugin_name)
+        local_image_path = os.path.join(local_images_dir, icon_name.replace('images/', ''))
+        if os.path.exists(local_image_path):
+            pixmap = QPixmap()
+            pixmap.load(local_image_path)
+            return pixmap
+
+    # As we did not find an icon elsewhere, look within our zip resources
+    if icon_name in plugin_icon_resources:
+        pixmap = QPixmap()
+        pixmap.loadFromData(plugin_icon_resources[icon_name])
+        return pixmap
+    return None
+
+
+def get_local_images_dir(subfolder=None):
+    '''
+    Returns a path to the user's local resources/images folder
+    If a subfolder name parameter is specified, appends this to the path
+    '''
+    images_dir = os.path.join(config_dir, 'resources/images')
+    if subfolder:
+        images_dir = os.path.join(images_dir, subfolder)
+    if iswindows:
+        images_dir = os.path.normpath(images_dir)
+    return images_dir
+
+
+def create_menu_item(ia, parent_menu, menu_text, image=None, tooltip=None,
+                     shortcut=(), triggered=None, is_checked=None):
+    '''
+    Create a menu action with the specified criteria and action
+    Note that if no shortcut is specified, will not appear in Preferences->Keyboard
+    This method should only be used for actions which either have no shortcuts,
+    or register their menus only once. Use create_menu_action_unique for all else.
+    '''
+    if shortcut is not None:
+        if len(shortcut) == 0:
+            shortcut = ()
+        else:
+            shortcut = _(shortcut)
+    ac = ia.create_action(spec=(menu_text, None, tooltip, shortcut),
+        attr=menu_text)
+    if image:
+        ac.setIcon(get_icon(image))
+    if triggered is not None:
+        ac.triggered.connect(triggered)
+    if is_checked is not None:
+        ac.setCheckable(True)
+        if is_checked:
+            ac.setChecked(True)
+
+    parent_menu.addAction(ac)
+    return ac
+
+
+def create_menu_action_unique(ia, parent_menu, menu_text, image=None, tooltip=None,
+                       shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
+                       unique_name=None):
+    '''
+    Create a menu action with the specified criteria and action, using the new
+    InterfaceAction.create_menu_action() function which ensures that regardless of
+    whether a shortcut is specified it will appear in Preferences->Keyboard
+    '''
+    orig_shortcut = shortcut
+    kb = ia.gui.keyboard
+    if unique_name is None:
+        unique_name = menu_text
+    if not shortcut == False:
+        full_unique_name = menu_action_unique_name(ia, unique_name)
+        if full_unique_name in kb.shortcuts:
+            shortcut = False
+        else:
+            if shortcut is not None and not shortcut == False:
+                if len(shortcut) == 0:
+                    shortcut = None
+                else:
+                    shortcut = _(shortcut)
+
+    if shortcut_name is None:
+        shortcut_name = menu_text.replace('&','')
+
+    ac = ia.create_menu_action(parent_menu, unique_name, menu_text, icon=None, shortcut=shortcut,
+        description=tooltip, triggered=triggered, shortcut_name=shortcut_name)
+    if shortcut == False and not orig_shortcut == False:
+        if ac.calibre_shortcut_unique_name in ia.gui.keyboard.shortcuts:
+            kb.replace_action(ac.calibre_shortcut_unique_name, ac)
+    if image:
+        ac.setIcon(get_icon(image))
+    if is_checked is not None:
+        ac.setCheckable(True)
+        if is_checked:
+            ac.setChecked(True)
+    return ac
+
+
+def swap_author_names(author):
+    if author.find(',') == -1:
+        return author
+    name_parts = author.strip().partition(',')
+    return name_parts[2].strip() + ' ' + name_parts[0]
+
+
+def get_library_uuid(db):
+    try:
+        library_uuid = db.library_id
+    except:
+        library_uuid = ''
+    return library_uuid
+
+
+class ImageLabel(QLabel):
+
+    def __init__(self, parent, icon_name, size=16):
+        QLabel.__init__(self, parent)
+        pixmap = get_pixmap(icon_name)
+        self.setPixmap(pixmap)
+        self.setMaximumSize(size, size)
+        self.setScaledContents(True)
+
+
+class ImageTitleLayout(QHBoxLayout):
+    '''
+    A reusable layout widget displaying an image followed by a title
+    '''
+    def __init__(self, parent, icon_name, title):
+        QHBoxLayout.__init__(self)
+        title_image_label = QLabel(parent)
+        pixmap = get_pixmap(icon_name)
+        if pixmap is None:
+            pixmap = get_pixmap('library.png')
+            # error_dialog(parent, _('Restart required'),
+            #              _('You must restart Calibre before using this plugin!'), show=True)
+        else:
+            title_image_label.setPixmap(pixmap)
+        title_image_label.setMaximumSize(32, 32)
+        title_image_label.setScaledContents(True)
+        self.addWidget(title_image_label)
+
+        title_font = QFont()
+        title_font.setPointSize(16)
+        shelf_label = QLabel(title, parent)
+        shelf_label.setFont(title_font)
+        self.addWidget(shelf_label)
+        self.insertStretch(-1)
+
+
+class SizePersistedDialog(QDialog):
+    '''
+    This dialog is a base class for any dialogs that want their size/position
+    restored when they are next opened.
+    '''
+    def __init__(self, parent, unique_pref_name):
+        QDialog.__init__(self, parent)
+        self.unique_pref_name = unique_pref_name
+        self.geom = gprefs.get(unique_pref_name, None)
+        self.finished.connect(self.dialog_closing)
+
+    def resize_dialog(self):
+        if self.geom is None:
+            self.resize(self.sizeHint())
+        else:
+            self.restoreGeometry(self.geom)
+
+    def dialog_closing(self, result):
+        geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = geom
+
+
+class ReadOnlyTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, text):
+        if text is None:
+            text = ''
+        QTableWidgetItem.__init__(self, text, QtGui.QTableWidgetItem.UserType)
+        self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+
+
+class RatingTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, rating, is_read_only=False):
+        QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType)
+        self.setData(Qt.DisplayRole, rating)
+        if is_read_only:
+            self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+
+
+class DateTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, date_read, is_read_only=False, default_to_today=False):
+        if date_read == UNDEFINED_DATE and default_to_today:
+            date_read = now()
+        if is_read_only:
+            QTableWidgetItem.__init__(self, format_date(date_read, None), QtGui.QTableWidgetItem.UserType)
+            self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+        else:
+            QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType)
+            self.setData(Qt.DisplayRole, QDateTime(date_read))
+
+
+class NoWheelComboBox(QComboBox):
+
+    def wheelEvent (self, event):
+        # Disable the mouse wheel on top of the combo box changing selection as plays havoc in a grid
+        event.ignore()
+
+
+class CheckableTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, checked=False, is_tristate=False):
+        QTableWidgetItem.__init__(self, '')
+        self.setFlags(Qt.ItemFlags(Qt.ItemIsSelectable | Qt.ItemIsUserCheckable | Qt.ItemIsEnabled ))
+        if is_tristate:
+            self.setFlags(self.flags() | Qt.ItemIsTristate)
+        if checked:
+            self.setCheckState(Qt.Checked)
+        else:
+            if is_tristate and checked is None:
+                self.setCheckState(Qt.PartiallyChecked)
+            else:
+                self.setCheckState(Qt.Unchecked)
+
+    def get_boolean_value(self):
+        '''
+        Return a boolean value indicating whether checkbox is checked
+        If this is a tristate checkbox, a partially checked value is returned as None
+        '''
+        if self.checkState() == Qt.PartiallyChecked:
+            return None
+        else:
+            return self.checkState() == Qt.Checked
+
+
+class TextIconWidgetItem(QTableWidgetItem):
+
+    def __init__(self, text, icon):
+        QTableWidgetItem.__init__(self, text)
+        if icon:
+            self.setIcon(icon)
+
+
+class ReadOnlyTextIconWidgetItem(ReadOnlyTableWidgetItem):
+
+    def __init__(self, text, icon):
+        ReadOnlyTableWidgetItem.__init__(self, text)
+        if icon:
+            self.setIcon(icon)
+
+
+class ReadOnlyLineEdit(QLineEdit):
+
+    def __init__(self, text, parent):
+        if text is None:
+            text = ''
+        QLineEdit.__init__(self, text, parent)
+        self.setEnabled(False)
+
+
+class KeyValueComboBox(QComboBox):
+
+    def __init__(self, parent, values, selected_key):
+        QComboBox.__init__(self, parent)
+        self.values = values
+        self.populate_combo(selected_key)
+
+    def populate_combo(self, selected_key):
+        self.clear()
+        selected_idx = idx = -1
+        for key, value in self.values.iteritems():
+            idx = idx + 1
+            self.addItem(value)
+            if key == selected_key:
+                selected_idx = idx
+        self.setCurrentIndex(selected_idx)
+
+    def selected_key(self):
+        for key, value in self.values.iteritems():
+            if value == unicode(self.currentText()).strip():
+                return key
+
+
+class CustomColumnComboBox(QComboBox):
+
+    def __init__(self, parent, custom_columns, selected_column, initial_items=['']):
+        QComboBox.__init__(self, parent)
+        self.populate_combo(custom_columns, selected_column, initial_items)
+
+    def populate_combo(self, custom_columns, selected_column, initial_items=['']):
+        self.clear()
+        self.column_names = initial_items
+        if len(initial_items) > 0:
+            self.addItems(initial_items)
+        selected_idx = 0
+        for idx, value in enumerate(initial_items):
+            if value == selected_column:
+                selected_idx = idx
+        for key in sorted(custom_columns.keys()):
+            self.column_names.append(key)
+            self.addItem('%s (%s)'%(key, custom_columns[key]['name']))
+            if key == selected_column:
+                selected_idx = len(self.column_names) - 1
+        self.setCurrentIndex(selected_idx)
+
+    def get_selected_column(self):
+        return self.column_names[self.currentIndex()]
+
+
+class KeyboardConfigDialog(SizePersistedDialog):
+    '''
+    This dialog is used to allow editing of keyboard shortcuts.
+    '''
+    def __init__(self, gui, group_name):
+        SizePersistedDialog.__init__(self, gui, 'Keyboard shortcut dialog')
+        self.gui = gui
+        self.setWindowTitle('Keyboard shortcuts')
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+
+        self.keyboard_widget = ShortcutConfig(self)
+        layout.addWidget(self.keyboard_widget)
+        self.group_name = group_name
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.commit)
+        button_box.rejected.connect(self.reject)
+        layout.addWidget(button_box)
+
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.initialize()
+
+    def initialize(self):
+        self.keyboard_widget.initialize(self.gui.keyboard)
+        self.keyboard_widget.highlight_group(self.group_name)
+
+    def commit(self):
+        self.keyboard_widget.commit()
+        self.accept()
+
+
+class DateDelegate(QStyledItemDelegate):
+    '''
+    Delegate for dates. Because this delegate stores the
+    format as an instance variable, a new instance must be created for each
+    column. This differs from all the other delegates.
+    '''
+    def __init__(self, parent):
+        QStyledItemDelegate.__init__(self, parent)
+        self.format = 'dd MMM yyyy'
+
+    def displayText(self, val, locale):
+        d = val.toDateTime()
+        if d <= UNDEFINED_QDATETIME:
+            return ''
+        return format_date(qt_to_dt(d, as_utc=False), self.format)
+
+    def createEditor(self, parent, option, index):
+        qde = QStyledItemDelegate.createEditor(self, parent, option, index)
+        qde.setDisplayFormat(self.format)
+        qde.setMinimumDateTime(UNDEFINED_QDATETIME)
+        qde.setSpecialValueText(_('Undefined'))
+        qde.setCalendarPopup(True)
+        return qde
+
+    def setEditorData(self, editor, index):
+        val = index.model().data(index, Qt.DisplayRole).toDateTime()
+        if val is None or val == UNDEFINED_QDATETIME:
+            val = now()
+        editor.setDateTime(val)
+
+    def setModelData(self, editor, model, index):
+        val = editor.dateTime()
+        if val <= UNDEFINED_QDATETIME:
+            model.setData(index, UNDEFINED_QDATETIME, Qt.EditRole)
+        else:
+            model.setData(index, QDateTime(val), Qt.EditRole)

calibre-plugin/config.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit,
+                      QTextEdit, QComboBox, QCheckBox, QPushButton)
+
+from calibre.gui2 import dynamic, info_dialog
+from calibre.utils.config import JSONConfig
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs \
+    import (SKIP, ADDNEW, UPDATE, UPDATEALWAYS, OVERWRITE, OVERWRITEALWAYS,
+             CALIBREONLY,collision_order)
+
+# This is where all preferences for this plugin will be stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
+# Set defaults
+prefs.defaults['personal.ini'] = get_resources('example.ini')
+prefs.defaults['updatemeta'] = True
+prefs.defaults['keeptags'] = False
+#prefs.defaults['onlyoverwriteifnewer'] = False
+prefs.defaults['urlsfromclip'] = True
+prefs.defaults['updatedefault'] = True
+prefs.defaults['fileform'] = 'epub'
+prefs.defaults['collision'] = OVERWRITE
+prefs.defaults['deleteotherforms'] = False
+
+class ConfigWidget(QWidget):
+
+    def __init__(self):
+        QWidget.__init__(self)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        horz = QHBoxLayout()
+        label = QLabel('Default Output &Format:')
+        horz.addWidget(label)
+        self.fileform = QComboBox(self)
+        self.fileform.addItem('epub')
+        self.fileform.addItem('mobi')
+        self.fileform.addItem('html')
+        self.fileform.addItem('txt')
+        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.activated.connect(self.set_collisions)
+        label.setBuddy(self.fileform)
+        horz.addWidget(self.fileform)
+        self.l.addLayout(horz)
+
+        horz = QHBoxLayout()
+        label = QLabel('Default If Story Already Exists?')
+        label.setToolTip("What to do if there's already an existing story with the same title and author.")
+        horz.addWidget(label)
+        self.collision = QComboBox(self)
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
+        label.setBuddy(self.collision)
+        horz.addWidget(self.collision)
+        self.l.addLayout(horz)
+
+        self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
+        self.updatemeta.setToolTip('Update title, author, URL, tags, etc for story in Calibre from web site.')
+        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.l.addWidget(self.updatemeta)
+
+        self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
+        self.keeptags.setToolTip('Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.')
+        self.keeptags.setChecked(prefs['keeptags'])
+        self.l.addWidget(self.keeptags)
+
+        # self.onlyoverwriteifnewer = QCheckBox('Default Only Overwrite Story if Newer',self)
+        # self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer or from the same day.")
+        # self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer'])
+        # self.l.addWidget(self.onlyoverwriteifnewer)
+        
+        self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
+        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New?')
+        self.urlsfromclip.setChecked(prefs['urlsfromclip'])
+        self.l.addWidget(self.urlsfromclip)
+
+        self.updatedefault = QCheckBox('Default to Update when books selected?',self)
+        self.updatedefault.setToolTip('The top FanFictionDownLoader plugin button will start Update if\n'+
+                                      'books are selected.  If unchecked, it will always bring up \'Add New\'.')
+        self.updatedefault.setChecked(prefs['updatedefault'])
+        self.l.addWidget(self.updatedefault)
+
+        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
+        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
+        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
+        self.l.addWidget(self.deleteotherforms)
+        
+        self.label = QLabel('personal.ini:')
+        self.l.addWidget(self.label)
+
+        self.ini = QTextEdit(self)
+        self.ini.setLineWrapMode(QTextEdit.NoWrap)
+        self.ini.setText(prefs['personal.ini'])
+        self.l.addWidget(self.ini)
+
+        self.defaults = QPushButton('View Defaults', self)
+        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
+        self.defaults.clicked.connect(self.show_defaults)
+        self.l.addWidget(self.defaults)
+        
+        reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self)
+        reset_confirmation_button.setToolTip(_(
+                    'Reset all show me again dialogs for the FanFictionDownLoader plugin'))
+        reset_confirmation_button.clicked.connect(self.reset_dialogs)
+        self.l.addWidget(reset_confirmation_button)
+        
+    def set_collisions(self):
+        prev=self.collision.currentText()
+        self.collision.clear()
+        for o in collision_order:
+            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
+                self.collision.addItem(o)
+        i = self.collision.findText(prev)
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        
+    def save_settings(self):
+        prefs['fileform'] = unicode(self.fileform.currentText())
+        prefs['collision'] = unicode(self.collision.currentText())
+        prefs['updatemeta'] = self.updatemeta.isChecked()
+        prefs['keeptags'] = self.keeptags.isChecked()
+        prefs['urlsfromclip'] = self.urlsfromclip.isChecked()
+        prefs['updatedefault'] = self.updatedefault.isChecked()
+#        prefs['onlyoverwriteifnewer'] = self.onlyoverwriteifnewer.isChecked()
+        prefs['deleteotherforms'] = self.deleteotherforms.isChecked()
+        
+        ini = unicode(self.ini.toPlainText())
+        if ini:
+            prefs['personal.ini'] = ini
+        else:
+            # if they've removed everything, clear it so they get the
+            # default next time.
+            del prefs['personal.ini']
+        
+    def show_defaults(self):
+        text = get_resources('plugin-defaults.ini')
+        ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+
+    def reset_dialogs(self):
+        for key in dynamic.keys():
+            if key.startswith('fanfictiondownloader_') and key.endswith('_again') \
+                                                  and dynamic[key] is False:
+                dynamic[key] = True
+        info_dialog(self, _('Done'),
+                _('Confirmation dialogs have all been reset'), show=True)
+
+        
+class ShowDefaultsIniDialog(QDialog):
+
+    def __init__(self, icon, text, parent=None):
+        QDialog.__init__(self, parent)
+        self.resize(600, 500)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.label = QLabel("Plugin Defaults (Read-Only)")
+        self.label.setToolTip("These all of the plugin's configurable settings\nand their default settings.")
+        self.setWindowTitle(_('Plugin Defaults'))
+        self.setWindowIcon(icon)
+        self.l.addWidget(self.label)
+        
+        self.ini = QTextEdit(self)
+        self.ini.setToolTip("These all of the plugin's configurable settings\nand their default settings.")
+        self.ini.setLineWrapMode(QTextEdit.NoWrap)
+        self.ini.setText(text)
+        self.ini.setReadOnly(True)
+        self.l.addWidget(self.ini)
+        
+        self.ok_button = QPushButton('OK', self)
+        self.ok_button.clicked.connect(self.hide)
+        self.l.addWidget(self.ok_button)
+        

calibre-plugin/dialogs.py

@@ -0,0 +1,589 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import traceback
+
+from PyQt4 import QtGui
+from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
+                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon,
+                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
+                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt,QAbstractItemView )
+
+from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
+from calibre.gui2.dialogs.confirm_delete import confirm
+
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+from calibre_plugins.fanfictiondownloader_plugin.common_utils \
+    import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
+            ImageTitleLayout, get_icon)
+
+SKIP='Skip'
+ADDNEW='Add New Book'
+UPDATE='Update EPUB if New Chapters'
+UPDATEALWAYS='Update EPUB Always'
+OVERWRITE='Overwrite if Newer'
+OVERWRITEALWAYS='Overwrite Always'
+CALIBREONLY='Update Calibre Metadata Only'
+collision_order=[SKIP,
+                 ADDNEW,
+                 UPDATE,
+                 UPDATEALWAYS,
+                 OVERWRITE,
+                 OVERWRITEALWAYS,
+                 CALIBREONLY,]
+        
+class NotGoingToDownload(Exception):
+    def __init__(self,error,icon='dialog_error.png'):
+        self.error=error
+        self.icon=icon
+
+    def __str__(self):
+        return self.error
+    
+class AddNewDialog(SizePersistedDialog):
+
+    def __init__(self, gui, prefs, icon, url_list_text):
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
+        self.gui = gui
+
+        self.setMinimumWidth(300)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        self.setWindowTitle('FanFictionDownLoader')
+        self.setWindowIcon(icon)
+
+        self.l.addWidget(QLabel('Story URL(s), one per line:'))
+        self.url = QTextEdit(self)
+        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.')
+        self.url.setLineWrapMode(QTextEdit.NoWrap)
+        self.url.setText(url_list_text)
+        self.l.addWidget(self.url)
+        
+        horz = QHBoxLayout()
+        label = QLabel('Output &Format:')
+        horz.addWidget(label)
+        self.fileform = QComboBox(self)
+        self.fileform.addItem('epub')
+        self.fileform.addItem('mobi')
+        self.fileform.addItem('html')
+        self.fileform.addItem('txt')
+        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.activated.connect(self.set_collisions)
+        
+        label.setBuddy(self.fileform)
+        horz.addWidget(self.fileform)
+        self.l.addLayout(horz)
+
+        horz = QHBoxLayout()
+        label = QLabel('If Story Already Exists?')
+        label.setToolTip("What to do if there's already an existing story with the same title and author.")
+        horz.addWidget(label)
+        self.collision = QComboBox(self)
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        # self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+
+        #                           UPDATE+' will download new chapters only and add to existing EPUB.\n'+
+        #                           ADDNEW+' will create a new story with the same title and author.\n'+
+        #                           SKIP+' will not download existing stories.\n'+
+        #                           CALIBREONLY+' will not download stories, but will update Calibre metadata.')
+        label.setBuddy(self.collision)
+        horz.addWidget(self.collision)
+        self.l.addLayout(horz)
+
+        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
+        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?')
+        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.l.addWidget(self.updatemeta)
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
+        
+        if url_list_text:
+            button_box.button(QDialogButtonBox.Ok).setFocus()
+
+        # restore saved size.
+        self.resize_dialog()
+        #self.resize(self.sizeHint())
+
+    def set_collisions(self):
+        prev=self.collision.currentText()
+        self.collision.clear()
+        for o in collision_order:
+            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
+                self.collision.addItem(o)
+        i = self.collision.findText(prev)
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        
+    def get_ffdl_options(self):
+        return {
+            'fileform': unicode(self.fileform.currentText()),
+            'collision': unicode(self.collision.currentText()),
+            'updatemeta': unicode(self.updatemeta.isChecked()),
+            }
+
+    def get_urlstext(self):
+        return unicode(self.url.toPlainText())
+        
+class UserPassDialog(QDialog):
+    '''
+    Need to collect User/Pass for some sites.
+    '''
+    def __init__(self, gui, site):
+        QDialog.__init__(self, gui)
+        self.gui = gui
+        self.status=False
+        self.setWindowTitle('User/Password')
+
+        self.l = QGridLayout()
+        self.setLayout(self.l)
+
+        self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
+        
+        self.l.addWidget(QLabel("User:"),1,0)
+        self.user = QLineEdit(self)
+        self.l.addWidget(self.user,1,1)
+   
+        self.l.addWidget(QLabel("Password:"),2,0)
+        self.passwd = QLineEdit(self)
+        self.passwd.setEchoMode(QLineEdit.Password)
+        self.l.addWidget(self.passwd,2,1)
+   
+        self.ok_button = QPushButton('OK', self)
+        self.ok_button.clicked.connect(self.ok)
+        self.l.addWidget(self.ok_button,3,0)
+
+        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button.clicked.connect(self.cancel)
+        self.l.addWidget(self.cancel_button,3,1)
+
+        self.resize(self.sizeHint())
+
+    def ok(self):
+        self.status=True
+        self.hide()
+
+    def cancel(self):
+        self.status=False
+        self.hide()
+
+class MetadataProgressDialog(QProgressDialog):
+    '''
+    ProgressDialog displayed while fetching metadata for each story.
+    '''
+    def __init__(self, gui,
+                 book_list,
+                 options,
+                 metadata_function,
+                 startdownload_function):
+        QProgressDialog.__init__(self,
+                                 "Fetching metadata for stories...",
+                                 QString(), 0, len(book_list), gui)
+        self.setWindowTitle("Downloading metadata for stories")
+        self.setMinimumWidth(500)
+        self.gui = gui
+        self.book_list = book_list
+        self.options = options
+        self.metadata_function = metadata_function
+        self.startdownload_function = startdownload_function
+        self.i = 0
+        
+        ## self.do_loop does QTimer.singleShot on self.do_loop also.
+        ## A weird way to do a loop, but that was the example I had.
+        QTimer.singleShot(0, self.do_loop)
+        self.exec_()
+
+    def updateStatus(self):
+        self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.book_list)))
+        self.setValue(self.i+1)
+        print(self.labelText())
+
+    def do_loop(self):
+
+        if self.i == 0:
+            self.setValue(0)
+
+        book = self.book_list[self.i]
+        try:
+            ## collision spec passed into getadapter by partial from ffdl_plugin
+            ## no retval only if it exists, but collision is SKIP
+            self.metadata_function(book)
+            
+        except NotGoingToDownload as d:
+            book['good']=False
+            book['comment']=unicode(d)
+            book['icon'] = d.icon
+
+        except Exception as e:
+            book['good']=False
+            book['comment']=unicode(e)
+            print("Exception: %s:%s"%(book,unicode(e)))
+            traceback.print_exc()
+            
+        self.updateStatus()
+        self.i += 1
+            
+        if self.i >= len(self.book_list) or self.wasCanceled():
+            return self.do_when_finished()
+        else:
+            QTimer.singleShot(0, self.do_loop)
+
+    def do_when_finished(self):
+        self.hide()
+        self.gui = None        
+        # Queues a job to process these books in the background.
+        self.startdownload_function(self.book_list)
+
+class AboutDialog(QDialog):
+
+    def __init__(self, parent, icon, text):
+        QDialog.__init__(self, parent)
+        self.resize(400, 250)
+        self.l = QGridLayout()
+        self.setLayout(self.l)
+        self.logo = QLabel()
+        self.logo.setMaximumWidth(110)
+        self.logo.setPixmap(QPixmap(icon.pixmap(100,100)))
+        self.label = QLabel(text)
+        self.label.setOpenExternalLinks(True)
+        self.label.setWordWrap(True)
+        self.setWindowTitle(_('About FanFictionDownLoader'))
+        self.setWindowIcon(icon)
+        self.l.addWidget(self.logo, 0, 0)
+        self.l.addWidget(self.label, 0, 1)
+        self.bb = QDialogButtonBox(self)
+        b = self.bb.addButton(_('OK'), self.bb.AcceptRole)
+        b.setDefault(True)
+        self.l.addWidget(self.bb, 2, 0, 1, -1)
+        self.bb.accepted.connect(self.accept)
+
+class IconWidgetItem(ReadOnlyTextIconWidgetItem):
+    def __init__(self, text, icon, sort_key):
+        ReadOnlyTextIconWidgetItem.__init__(self, text, icon)
+        self.sort_key = sort_key
+
+    #Qt uses a simple < check for sorting items, override this to use the sortKey
+    def __lt__(self, other):
+        return self.sort_key < other.sort_key
+
+class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):
+    def __init__(self, text, sort_key):
+        ReadOnlyTableWidgetItem.__init__(self, text)
+        self.sort_key = sort_key
+
+    #Qt uses a simple < check for sorting items, override this to use the sortKey
+    def __lt__(self, other):
+        return self.sort_key < other.sort_key
+
+class UpdateExistingDialog(SizePersistedDialog):
+    def __init__(self, gui, header, prefs, icon, books,
+                 save_size_name='FanFictionDownLoader plugin:update list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        self.gui = gui
+        
+        self.setWindowTitle(header)
+        self.setWindowIcon(icon)
+        
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, 'images/icon.png',
+                                        header)
+        layout.addLayout(title_layout)
+        books_layout = QHBoxLayout()
+        layout.addLayout(books_layout)
+
+        self.books_table = StoryListTableWidget(self)
+        books_layout.addWidget(self.books_table)
+
+        button_layout = QVBoxLayout()
+        books_layout.addLayout(button_layout)
+        # self.move_up_button = QtGui.QToolButton(self)
+        # self.move_up_button.setToolTip('Move selected books up the list')
+        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
+        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
+        # button_layout.addWidget(self.move_up_button)
+        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem)
+        self.remove_button = QtGui.QToolButton(self)
+        self.remove_button.setToolTip('Remove selected books from the list')
+        self.remove_button.setIcon(get_icon('list_remove.png'))
+        self.remove_button.clicked.connect(self.remove_from_list)
+        button_layout.addWidget(self.remove_button)
+        spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem1)
+        # self.move_down_button = QtGui.QToolButton(self)
+        # self.move_down_button.setToolTip('Move selected books down the list')
+        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
+        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
+        # button_layout.addWidget(self.move_down_button)
+
+        options_layout = QHBoxLayout()
+
+        label = QLabel('Output &Format:')
+        options_layout.addWidget(label)
+        self.fileform = QComboBox(self)
+        self.fileform.addItem('epub')
+        self.fileform.addItem('mobi')
+        self.fileform.addItem('html')
+        self.fileform.addItem('txt')
+        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.activated.connect(self.set_collisions)
+        label.setBuddy(self.fileform)
+        options_layout.addWidget(self.fileform)
+        
+        label = QLabel('Update Mode:')
+        label.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
+        options_layout.addWidget(label)
+        self.collision = QComboBox(self)
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
+        label.setBuddy(self.collision)
+        options_layout.addWidget(self.collision)
+
+        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
+        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?  May set default from plugin configuration.')
+        self.updatemeta.setChecked(prefs['updatemeta'])
+        options_layout.addWidget(self.updatemeta)
+                
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        options_layout.addWidget(button_box)
+        
+        layout.addLayout(options_layout)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.books_table.populate_table(books)
+
+    def set_collisions(self):
+        prev=self.collision.currentText()
+        self.collision.clear()
+        for o in collision_order:
+            if o not in [ADDNEW,SKIP] and \
+                    (self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]):
+                self.collision.addItem(o)        
+        i = self.collision.findText(prev)
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        
+    def remove_from_list(self):
+        self.books_table.remove_selected_rows()
+
+    def get_books(self):
+        return self.books_table.get_books()
+
+    def get_ffdl_options(self):
+        return {
+            'fileform': unicode(self.fileform.currentText()),
+            'collision': unicode(self.collision.currentText()),
+            'updatemeta': unicode(self.updatemeta.isChecked()),
+            }
+
+class DisplayStoryListDialog(SizePersistedDialog):
+    def __init__(self, gui, header, prefs, icon, books,
+                 label_text='',
+                 save_size_name='FanFictionDownLoader plugin:display list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        # UpdateExistingDialog.__init__(self, gui, header, prefs, icon, books,
+        #                               save_size_name='FanFictionDownLoader plugin:display list dialog')
+
+        self.gui = gui
+        
+        self.setWindowTitle(header)
+        self.setWindowIcon(icon)
+        
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, 'images/icon.png',
+                                        header)
+        layout.addLayout(title_layout)
+
+        self.books_table = StoryListTableWidget(self)
+        layout.addWidget(self.books_table)
+
+        options_layout = QHBoxLayout()
+        self.label = QLabel(label_text)
+        #self.label.setOpenExternalLinks(True)
+        #self.label.setWordWrap(True)
+        options_layout.addWidget(self.label)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+
+        options_layout.addWidget(button_box)
+        
+        layout.addLayout(options_layout)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.books_table.populate_table(books)
+        
+    def get_books(self):
+        return self.books_table.get_books()
+
+class StoryListTableWidget(QTableWidget):
+
+    def __init__(self, parent):
+        QTableWidget.__init__(self, parent)
+        self.setSelectionBehavior(QAbstractItemView.SelectRows)
+
+    def populate_table(self, books):
+        self.clear()
+        self.setAlternatingRowColors(True)
+        self.setRowCount(len(books))
+        header_labels = ['','Title', 'Author', 'URL', 'Comment']
+        self.setColumnCount(len(header_labels))
+        self.setHorizontalHeaderLabels(header_labels)
+        self.horizontalHeader().setStretchLastSection(True)
+        #self.verticalHeader().setDefaultSectionSize(24)
+        self.verticalHeader().hide()
+
+        self.books={}
+        for row, book in enumerate(books):
+            self.populate_table_row(row, book)
+            self.books[row] = book
+
+        # turning True breaks up/down.  Do we need either sorting or up/down?
+        self.setSortingEnabled(True)
+        self.resizeColumnsToContents()
+        self.setMinimumColumnWidth(1, 100)
+        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumColumnWidth(3, 100)
+        self.setMinimumSize(300, 0)
+        # if len(books) > 0:
+        #     self.selectRow(0)
+        self.sortItems(1)
+        self.sortItems(0)
+
+    def setMinimumColumnWidth(self, col, minimum):
+        if self.columnWidth(col) < minimum:
+            self.setColumnWidth(col, minimum)
+
+    def populate_table_row(self, row, book):
+        if book['good']:
+            icon = get_icon('ok.png')
+            val = 0
+        else:
+            icon = get_icon('minus.png')
+            val = 1
+        if 'icon' in book:
+            icon = get_icon(book['icon'])
+
+        status_cell = IconWidgetItem(None,icon,val)
+        status_cell.setData(Qt.UserRole, QVariant(val))
+        self.setItem(row, 0, status_cell)
+        
+        title_cell = ReadOnlyTableWidgetItem(book['title'])
+        title_cell.setData(Qt.UserRole, QVariant(row))
+        self.setItem(row, 1, title_cell)
+        
+        self.setItem(row, 2, AuthorTableWidgetItem(book['author'], book['author_sort']))
+        
+        url_cell = ReadOnlyTableWidgetItem(book['url'])
+        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
+        self.setItem(row, 3, url_cell)
+        
+        comment_cell = ReadOnlyTableWidgetItem(book['comment'])
+        #comment_cell.setData(Qt.UserRole, QVariant(book))
+        self.setItem(row, 4, comment_cell)
+
+    def get_books(self):
+        books = []
+        #print("=========================\nbooks:%s"%self.books)
+        for row in range(self.rowCount()):
+            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
+            book = self.books[rnum]
+            books.append(book)
+        return books
+
+    def remove_selected_rows(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        message = '<p>Are you sure you want to remove this book from the list?'
+        if len(rows) > 1:
+            message = '<p>Are you sure you want to remove the selected %d books from the list?'%len(rows)
+        if not confirm(message,'fanfictiondownloader_delete_item', self):
+            return
+        first_sel_row = self.currentRow()
+        for selrow in reversed(rows):
+            self.removeRow(selrow.row())
+        if first_sel_row < self.rowCount():
+            self.select_and_scroll_to_row(first_sel_row)
+        elif self.rowCount() > 0:
+            self.select_and_scroll_to_row(first_sel_row - 1)
+
+    def select_and_scroll_to_row(self, row):
+        self.selectRow(row)
+        self.scrollToItem(self.currentItem())
+
+    def move_rows_up(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        first_sel_row = rows[0].row()
+        if first_sel_row <= 0:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in selrows:
+            self.swap_row_widgets(selrow - 1, selrow + 1)
+        scroll_to_row = first_sel_row - 1
+        if scroll_to_row > 0:
+            scroll_to_row = scroll_to_row - 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def move_rows_down(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        last_sel_row = rows[-1].row()
+        if last_sel_row == self.rowCount() - 1:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in reversed(selrows):
+            self.swap_row_widgets(selrow + 2, selrow)
+        scroll_to_row = last_sel_row + 1
+        if scroll_to_row < self.rowCount() - 1:
+            scroll_to_row = scroll_to_row + 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def swap_row_widgets(self, src_row, dest_row):
+        self.blockSignals(True)
+        self.insertRow(dest_row)
+        for col in range(0, self.columnCount()):
+            self.setItem(dest_row, col, self.takeItem(src_row, col))
+        self.removeRow(src_row)
+        self.blockSignals(False)

calibre-plugin/ffdl_plugin.py

@@ -0,0 +1,788 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import time, os, copy
+from ConfigParser import SafeConfigParser
+from StringIO import StringIO
+from functools import partial
+from datetime import datetime
+
+from PyQt4.Qt import (QApplication, QMenu, QToolButton)
+
+from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
+from calibre.ebooks.metadata import MetaInformation, authors_to_string
+from calibre.ebooks.metadata.meta import get_metadata
+from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
+
+# The class that all interface action plugins must inherit from
+from calibre.gui2.actions import InterfaceAction
+
+from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin_icon_resources, get_icon,
+                                         create_menu_action_unique, get_library_uuid)
+
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge
+
+from calibre_plugins.fanfictiondownloader_plugin.config import (prefs)
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
+    AddNewDialog, UpdateExistingDialog, DisplayStoryListDialog,
+    MetadataProgressDialog, UserPassDialog, AboutDialog,
+    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
+    NotGoingToDownload )
+
+# because calibre immediately transforms html into zip and don't want
+# to have an 'if html'.  db.has_format is cool with the case mismatch,
+# but if I'm doing it anyway...
+formmapping = {
+    'epub':'EPUB',
+    'mobi':'MOBI',
+    'html':'ZIP',
+    'txt':'TXT'
+    }
+
+PLUGIN_ICONS = ['images/icon.png']
+
+class FanFictionDownLoaderPlugin(InterfaceAction):
+
+    name = 'FanFictionDownLoader'
+
+    # Declare the main action associated with this plugin
+    # The keyboard shortcut can be None if you dont want to use a keyboard
+    # shortcut. Remember that currently calibre has no central management for
+    # keyboard shortcuts, so try to use an unusual/unused shortcut.
+    # (text, icon_path, tooltip, keyboard shortcut)
+    # icon_path isn't in the zip--icon loaded below.
+    action_spec = (name, None,
+                   'Download FanFiction stories from various web sites', None)
+
+    action_type = 'global'
+    # make button menu drop down only
+    #popup_type = QToolButton.InstantPopup
+
+    def genesis(self):
+
+        # This method is called once per plugin, do initial setup here
+
+        # Read the plugin icons and store for potential sharing with the config widget
+        icon_resources = self.load_resources(PLUGIN_ICONS)
+        set_plugin_icon_resources(self.name, icon_resources)
+        
+        # Show the config dialog
+        # The config dialog can also be shown from within
+        # Preferences->Plugins, which is why the do_user_config
+        # method is defined on the base plugin class
+        do_user_config = self.interface_action_base_plugin.do_user_config
+        base = self.interface_action_base_plugin
+        self.version = base.name+" v%d.%d.%d"%base.version
+
+        # Set the icon for this interface action
+        # The get_icons function is a builtin function defined for all your
+        # plugin code. It loads icons from the plugin zip file. It returns
+        # QIcon objects, if you want the actual data, use the analogous
+        # get_resources builtin function.
+
+        # Note that if you are loading more than one icon, for performance, you
+        # should pass a list of names to get_icons. In this case, get_icons
+        # will return a dictionary mapping names to QIcons. Names that
+        # are not found in the zip file will result in null QIcons.
+        icon = get_icon('images/icon.png')
+
+        # The qaction is automatically created from the action_spec defined
+        # above
+        self.qaction.setIcon(icon)
+
+        # Call function when plugin triggered.
+        self.qaction.triggered.connect(self.plugin_button)
+
+        # Assign our menu to this action
+        self.menu = QMenu(self.gui)
+        self.qaction.setMenu(self.menu)
+
+        self.menu.aboutToShow.connect(self.about_to_show_menu)
+
+        self.actions_unique_map = {}
+
+        self.add_action = self.create_menu_item_ex(self.menu, '&Add New from URL(s)', image='plus.png',
+                                                   unique_name='Add New FanFiction Book(s) from URL(s)',
+                                                   shortcut_name='Add New FanFiction Book(s) from URL(s)',
+                                                   triggered=self.add_dialog )
+
+        self.update_action = self.create_menu_item_ex(self.menu, '&Update Existing FanFiction Book(s)', image='plusplus.png',
+                                                      unique_name='Update Existing FanFiction Book(s)',
+                                                      shortcut_name='Update Existing FanFiction Book(s)',
+                                                      triggered=self.update_existing) #partial(self._update_existing,'qwerty'))
+
+        self.menu.addSeparator()
+        self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False,
+                                                       image= 'config.png',
+                                                       unique_name='Configure FanFictionDownLoader',
+                                                       shortcut_name='Configure FanFictionDownLoader',
+                                                       triggered=partial(do_user_config,parent=self.gui))
+
+        self.config_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False,
+                                                       image= 'images/icon.png',
+                                                       unique_name='About FanFictionDownLoader',
+                                                       shortcut_name='About FanFictionDownLoader',
+                                                       triggered=self.about)
+
+
+    def about_to_show_menu(self):
+        self.update_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
+
+    def about(self):
+        # Get the about text from a file inside the plugin zip file
+        # The get_resources function is a builtin function defined for all your
+        # plugin code. It loads files from the plugin zip file. It returns
+        # the bytes from the specified file.
+        #
+        # Note that if you are loading more than one file, for performance, you
+        # should pass a list of names to get_resources. In this case,
+        # get_resources will return a dictionary mapping names to bytes. Names that
+        # are not found in the zip file will not be in the returned dictionary.
+        text = get_resources('about.txt')
+#        QMessageBox.about(self, 'About the FanFictionDownLoader Plugin',
+ #               text.decode('utf-8'))
+        AboutDialog(self.gui,self.qaction.icon(),text).exec_()
+        
+    def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
+                           shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
+                           unique_name=None):
+        ac = create_menu_action_unique(self, parent_menu, menu_text, image, tooltip,
+                                       shortcut, triggered, is_checked, shortcut_name, unique_name)
+        self.actions_unique_map[ac.calibre_shortcut_unique_name] = ac.calibre_shortcut_unique_name
+        return ac
+
+    def plugin_button(self):
+        if len(self.gui.library_view.get_selected_ids()) > 0 and prefs['updatedefault']:
+            self.update_existing()
+        else:
+            self.add_dialog()
+    
+    def add_dialog(self):
+
+        #print("add_dialog()")
+
+        url_list = self.get_urls_clip()
+        url_list_text = "\n".join(url_list)
+        
+        # self.gui is the main calibre GUI. It acts as the gateway to access
+        # all the elements of the calibre user interface, it should also be the
+        # parent of the dialog
+        # AddNewDialog just collects URLs, format and presents buttons.
+        d = AddNewDialog(self.gui,
+                         prefs,
+                         self.qaction.icon(),
+                         url_list_text,
+                         )
+        d.exec_()
+        if d.result() != d.Accepted:
+            return
+        
+        url_list = get_url_list(d.get_urlstext())
+        add_books = self._convert_urls_to_books(url_list)
+        #print("add_books:%s"%add_books)
+        #print("options:%s"%d.get_ffdl_options())
+
+        options = d.get_ffdl_options()
+        options['version'] = self.version
+        print(self.version)
+        
+        self.start_downloads( options, add_books )
+        
+    def update_existing(self):
+        #print("update_existing()")
+        previous = self.gui.library_view.currentIndex()
+        db = self.gui.current_db
+        book_ids = self.gui.library_view.get_selected_ids()
+        books = self._convert_calibre_ids_to_books(db, book_ids)
+        #print("update books:%s"%books)
+
+        d = UpdateExistingDialog(self.gui,
+                                 'Update Existing List',
+                                 prefs,
+                                 self.qaction.icon(),
+                                 books,
+                                 )
+        d.exec_()
+        if d.result() != d.Accepted:
+            return
+
+        update_books = d.get_books()
+
+        #print("update_books:%s"%update_books)
+        #print("options:%s"%d.get_ffdl_options())
+        # only if there's some good ones.
+        if 0 < len(filter(lambda x : x['good'], update_books)):
+            options = d.get_ffdl_options()
+            options['version'] = self.version
+            print(self.version)
+            self.start_downloads( options, update_books )
+        
+    def get_urls_clip(self):
+        url_list = []
+        if prefs['urlsfromclip']:
+            for url in unicode(QApplication.instance().clipboard().text()).split():
+                if( self._is_good_downloader_url(url) ):
+                    url_list.append(url)
+        return url_list
+
+    def apply_settings(self):
+        # No need to do anything with perfs here, but we could.
+        prefs
+
+    def start_downloads(self, options, books):
+
+        #print("start_downloads:%s"%books)
+
+        # create and pass temp dir.
+        tdir = PersistentTemporaryDirectory(prefix='fanfictiondownloader_')
+        options['tdir']=tdir
+
+        self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
+        
+        MetadataProgressDialog(self.gui,
+                               books,
+                               options,
+                               partial(self.get_metadata_for_book, options = options),
+                               partial(self.start_download_list, options = options))
+        # MetadataProgressDialog calls get_metadata_for_book for each 'good' story,
+        # get_metadata_for_book updates book for each,
+        # MetadataProgressDialog calls start_download_list at the end which goes
+        # into the BG, or shows list if no 'good' books.
+
+    def get_metadata_for_book(self,book,
+                              options={'fileform':'epub',
+                                       'collision':ADDNEW,
+                                       'updatemeta':True}):
+        '''
+        Update passed in book dict with metadata from website and
+        necessary data.  To be called from MetadataProgressDialog
+        'loop'.  Also pops dialogs for is adult, user/pass.
+        '''
+        
+        # The current database shown in the GUI
+        # db is an instance of the class LibraryDatabase2 from database.py
+        # This class has many, many methods that allow you to do a lot of
+        # things.
+        db = self.gui.current_db
+        
+        fileform  = options['fileform']
+        collision = options['collision']
+        updatemeta= options['updatemeta']
+
+        if not book['good']:
+            # book has already been flagged bad for whatever reason.
+            return
+        
+        url = book['url']
+        print("url:%s"%url)
+        
+        ## was self.ffdlconfig, but we need to be able to change it
+        ## when doing epub update.
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        ffdlconfig.readfp(StringIO(prefs['personal.ini']))
+        adapter = adapters.getAdapter(ffdlconfig,url)
+
+        options['personal.ini'] = prefs['personal.ini']
+
+        ## three tries, that's enough if both user/pass & is_adult needed,
+        ## or a couple tries of one or the other
+        for x in range(0,2):
+            try:
+                adapter.getStoryMetadataOnly()
+            except exceptions.FailedToLogin:
+                print("Login Failed, Need Username/Password.")
+                userpass = UserPassDialog(self.gui,url)
+                userpass.exec_() # exec_ will make it act modal
+                if userpass.status:
+                    adapter.username = userpass.user.text()
+                    adapter.password = userpass.passwd.text()
+                
+            except exceptions.AdultCheckRequired:
+                if question_dialog(self.gui, 'Are You Adult?', '<p>'+
+                                   "%s requires that you be an adult.  Please confirm you are an adult in your locale:"%url,
+                                   show_copy_button=False):
+                    adapter.is_adult=True
+
+        # let other exceptions percolate up.
+        story = adapter.getStoryMetadataOnly()
+        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+
+        book['title'] = story.getMetadata("title", removeallentities=True)
+        book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True)
+        book['publisher'] = story.getMetadata("site")
+        book['tags'] = writer.getTags()
+        book['pubdate'] = story.getMetadataRaw('datePublished')
+        book['timestamp'] = story.getMetadataRaw('dateCreated')
+        book['comments'] = story.getMetadata("description") #, removeallentities=True) comments handles entities better.
+        
+        # adapter.opener is the element with a threadlock.  But del
+        # adapter.opener doesn't work--subproc fails when it tries
+        # to pull in the adapter object that hasn't been imported yet.
+        # book['adapter'] = adapter
+        
+        book['is_adult'] = adapter.is_adult
+        book['username'] = adapter.username
+        book['password'] = adapter.password
+
+        book['icon'] = 'plus.png'
+        
+        if collision in (CALIBREONLY):
+            book['icon'] = 'metadata.png'
+        
+        # XXX should really do a 'you can't do that' dialog when they
+        # hit 'OK' for this case.
+        if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub':
+            raise NotGoingToDownload("Cannot update non-epub format.")
+        
+        book_id = None
+        
+        if book['calibre_id'] != None:
+            # updating an existing book.  Update mode applies.
+            print("update existing id:%s"%book['calibre_id'])
+            book_id = book['calibre_id']
+            # No handling needed: OVERWRITEALWAYS,CALIBREONLY
+            
+        # only care about collisions when not ADDNEW
+        elif collision != ADDNEW:
+            # 'new' book from URL.  collision handling applies.
+            print("from URL")
+
+            # find dups
+            mi = MetaInformation(story.getMetadata("title", removeallentities=True),
+                                 (story.getMetadata("author", removeallentities=True),)) # author is a list.
+            identicalbooks = db.find_identical_books(mi)
+            ## removed for being overkill.
+            # for ib in identicalbooks:
+            #     # only *really* identical if URL matches, too.
+            #     # XXX make an option?
+            #     if self._get_story_url(db,ib) == url:
+            #         identicalbooks.append(ib)
+            #print("identicalbooks:%s"%identicalbooks)
+
+            if collision == SKIP and identicalbooks:
+                raise NotGoingToDownload("Skipping duplicate story.","list_remove.png")
+
+            if len(identicalbooks) > 1:
+                raise NotGoingToDownload("More than one identical book--can't tell which to update/overwrite.","minusminus.png")
+
+            if collision == CALIBREONLY and not identicalbooks:
+                raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png")
+
+            if len(identicalbooks)>0:
+                book_id = identicalbooks.pop()
+                book['calibre_id'] = book_id
+                book['icon'] = 'edit-redo.png'
+
+        if book_id != None and collision != ADDNEW:
+            if options['collision'] in (CALIBREONLY):
+                book['comment'] = 'Metadata collected.'
+                # don't need temp file created below.
+                return
+            
+            ## newer/chaptercount checks are the same for both:
+            # Update epub, but only if more chapters.
+            if collision == UPDATE:
+                # 'book' can exist without epub.  If there's no existing epub,
+                # let it go and it will download it.
+                if db.has_format(book_id,fileform,index_is_id=True):
+                    toupdateio = StringIO()
+                    (epuburl,chaptercount) = doMerge(toupdateio,
+                                                     [StringIO(db.format(book_id,'EPUB',
+                                                                              index_is_id=True))],
+                                                     titlenavpoints=False,
+                                                     striptitletoc=True,
+                                                     forceunique=False)
+    
+                    urlchaptercount = int(story.getMetadata('numChapters'))
+                    if chaptercount == urlchaptercount: # and not onlyoverwriteifnewer:
+                        raise NotGoingToDownload("Already contains %d chapters."%chaptercount,'edit-undo.png')
+                    elif chaptercount > urlchaptercount:
+                        raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d." % (chaptercount,urlchaptercount),'dialog_error.png')
+    
+            if collision == OVERWRITE and \
+                    db.has_format(book_id,formmapping[fileform],index_is_id=True):
+                # check make sure incoming is newer.
+                lastupdated=story.getMetadataRaw('dateUpdated').date()
+                fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8]).date()
+                if fileupdated > lastupdated:
+                    raise NotGoingToDownload("Not Overwriting, web site is not newer.",'edit-undo.png')
+    
+            # For update, provide a tmp file copy of the existing epub so
+            # it can't change underneath us.
+            if collision in (UPDATE,UPDATEALWAYS) and \
+                    db.has_format(book['calibre_id'],'EPUB',index_is_id=True):
+                tmp = PersistentTemporaryFile(prefix='old-%s-'%book['calibre_id'],
+                                              suffix='.epub',
+                                              dir=options['tdir'])
+                db.copy_format_to(book_id,fileform,tmp,index_is_id=True)
+                print("existing epub tmp:"+tmp.name)
+                book['epub_for_update'] = tmp.name
+
+        if book['good']: # there shouldn't be any !'good' books at this point.
+            # if still 'good', make a temp file to write the output to.
+            tmp = PersistentTemporaryFile(prefix='new-%s-'%book['calibre_id'],
+                                               suffix='.'+options['fileform'],
+                                               dir=options['tdir'])
+            print("title:"+book['title'])
+            print("outfile:"+tmp.name)
+            book['outfile'] = tmp.name            
+                
+        return
+        
+    def start_download_list(self,book_list,
+                            options={'fileform':'epub',
+                                     'collision':ADDNEW,
+                                     'updatemeta':True}):
+        '''
+        Called by MetadataProgressDialog to start story downloads BG processing.
+        adapter_list is a list of tuples of (url,adapter)
+        '''
+        #print("start_download_list:book_list:%s"%book_list)
+
+        ## No need to BG process when CALIBREONLY!  Fake it.
+        if options['collision'] in (CALIBREONLY):
+            class NotJob(object):
+                def __init__(self,result):
+                    self.failed=False
+                    self.result=result
+            notjob = NotJob(book_list)
+            self.download_list_completed(notjob,options=options)
+            return
+
+        # ## XXX show list before starting download.
+        # d = DisplayStoryListDialog(self.gui,
+        #                            'Download List',
+        #                            prefs,
+        #                            self.qaction.icon(),
+        #                            book_list,
+        #                            label_text='Status of stories to be downloaded'
+        #                            )
+        # d.exec_()
+        # if d.result() != d.Accepted:
+        #     return
+            
+        for book in book_list:
+            if book['good']:
+                break
+        else:
+            ## No good stories to try to download, go straight to
+            ## list.
+            d = DisplayStoryListDialog(self.gui,
+                                       'Nothing to Download',
+                                       prefs,
+                                       self.qaction.icon(),
+                                       book_list,
+                                       label_text='None of the URLs/stories given can be/need to be downloaded.'
+                                       )
+            d.exec_()
+            return
+            
+        func = 'arbitrary_n'
+        cpus = self.gui.job_manager.server.pool_size
+        args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', 'do_download_worker',
+                (book_list, options, cpus)]
+        desc = 'Download FanFiction Book'
+        job = self.gui.job_manager.run_job(
+                self.Dispatcher(partial(self.download_list_completed,options=options)),
+                func, args=args,
+                description=desc)
+        
+        self.gui.status_bar.show_message('Starting %d FanFictionDownLoads'%len(book_list),3000)
+
+    def download_list_completed(self, job, options={}):
+        if job.failed:
+            self.gui.job_exception(job, dialog_title='Failed to Download Stories')
+            return
+        #print("download_list_completed job:%s"%job.result)
+        # for b in job.result:
+        #     print("job.result: %s"%b['title'])
+
+        db = self.gui.current_db
+
+        d = DisplayStoryListDialog(self.gui,
+                                   'Downloads finished, confirm to update Calibre',
+                                   prefs,
+                                   self.qaction.icon(),
+                                   job.result,
+                                   label_text='Stories will not be added or updated in Calibre without confirmation.'
+                                   )
+        d.exec_()
+        if d.result() == d.Accepted:
+            
+            ## in case the user removed any from the list.
+            book_list = d.get_books()
+            # for b in book_list:
+            #     print("d list: %s"%b['title'])
+
+            # update_list = filter(lambda x : x['good'] and x['calibre_id'] != None,
+            #                      book_list)
+
+            # add_list = filter(lambda x : x['good'] and x['calibre_id'] == None,
+            #                   book_list)
+
+            good_list = filter(lambda x : x['good'], book_list)
+
+            total_good = len(good_list) #update_list)+len(add_list)
+
+            self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good), 3000)
+            
+            #print("==================================================")
+            # addfiles,addfileforms,addmis=[],[],[]
+
+            list_000_ids = []
+            
+            for book in good_list:
+                print("add/update %s %s"%(book['title'],book['url']))
+                mi = self._make_mi_from_book(book)
+                
+                if options['collision'] != CALIBREONLY:
+                    self._add_or_update_book(book,options,prefs,mi)
+                    # book000 = copy.copy(book)
+                    # book000['title'] = "000 %s"%book000['title']
+                    # book000['calibre_id'] = self._find_existing_book_id(db,book000)
+                    # list_000_ids.append(self._add_or_update_book(book000,options,prefs))
+
+                if options['collision'] == CALIBREONLY or \
+                        (options['updatemeta'] and book['good']) :
+                    if prefs['keeptags']:
+                        old_tags = db.get_tags(book['calibre_id'])
+                        # remove old Completed/In-Progress only if there's a new one.
+                        if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
+                            old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags)
+                        # remove old Last Update tags if there are new ones.
+                        if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
+                            old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
+                        # mi.tags needs to be list, but set kills dups.
+                        mi.tags = list(set(list(old_tags)+mi.tags)) 
+                        
+                    db.set_metadata(book['calibre_id'],mi)
+                    
+            add_list = filter(lambda x : x['good'] and x['added'], book_list)
+            update_list = filter(lambda x : x['good'] and not x['added'], book_list)
+            update_ids = [ x['calibre_id'] for x in update_list ]
+                    
+            if len(add_list)+len(list_000_ids):
+                ## even shows up added to searchs.  Nice.
+                self.gui.library_view.model().books_added(len(add_list)+len(list_000_ids))
+
+            if update_ids:
+                self.gui.library_view.model().refresh_ids(update_ids+list_000_ids)
+
+            self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
+            
+            if len(update_list) + len(add_list) != total_good:
+                d = DisplayStoryListDialog(self.gui,
+                                           'Updates completed, final status',
+                                           prefs,
+                                           self.qaction.icon(),
+                                           book_list,
+                                           label_text='Stories have be added or updated in Calibre, some had additional problems.'
+                                           )
+                d.exec_()
+    
+        print("all done, remove temp dir.")
+        remove_dir(options['tdir'])
+
+    def _add_or_update_book(self,book,options,prefs,mi=None):
+        db = self.gui.current_db
+        
+        if mi == None:
+            mi = self._make_mi_from_book(book)
+
+        book_id = book['calibre_id']
+        if book_id == None:
+            book_id = db.create_book_entry(mi,
+                                           add_duplicates=True)
+            book['calibre_id'] = book_id
+            book['added'] = True
+        else:
+            book['added'] = False
+
+        if not db.add_format_with_hooks(book_id,
+                                        options['fileform'],
+                                        book['outfile'], index_is_id=True):
+            book['comment'] = "Adding format to book failed for some reason..."
+            book['good']=False
+            book['icon']='dialog_error.png'
+
+        if prefs['deleteotherforms']:
+            fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
+            for fmt in fmts:
+                if fmt != formmapping[options['fileform']]:
+                    print("remove f:"+fmt)
+                    db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+
+        # rl_plugin = self.gui.iactions['Reading List']    
+        # rl_plugin.add_books_to_list("Send",
+        #                             [book_id],
+        #                             refresh_screen=False,
+        #                             display_warnings=False)
+                
+        return book_id
+
+    def _find_existing_book_id(self,db,book,matchurl=True):
+        mi = MetaInformation(book["title"],(book["author"],)) # author is a list.
+        identicalbooks = db.find_identical_books(mi)
+        if matchurl: # only *really* identical if URL matches, too.
+            for ib in identicalbooks:
+                if self._get_story_url(db,ib) == book['url']:
+                    return ib
+        if identicalbooks:
+            return identicalbooks.pop()
+    
+        
+
+    def _make_mi_from_book(self,book):
+        mi = MetaInformation(book['title'],(book['author'],)) # author is a list.
+        mi.set_identifiers({'url':book['url']})
+        mi.publisher = book['publisher']
+        mi.tags = book['tags']
+        #mi.languages = ['en']
+        mi.pubdate = book['pubdate']
+        mi.timestamp = book['timestamp']
+        mi.comments = book['comments']
+        return mi
+
+
+    def _convert_urls_to_books(self, urls):
+        books = []
+        for url in urls:
+            book = {}
+            book['good'] = True
+            book['calibre_id'] = None
+            book['title'] = 'Unknown'
+            book['author'] = 'Unknown'
+            book['author_sort'] = 'Unknown'
+                
+            book['comment'] = ''
+            book['url'] = ''
+            book['added'] = False
+            
+            self._set_book_url_and_comment(book,url)
+                    
+            books.append(book)
+        return books
+    
+    def _convert_calibre_ids_to_books(self, db, ids):
+        books = []
+        for book_id in ids:
+            mi = db.get_metadata(book_id, index_is_id=True)
+            book = {}
+            book['good'] = True
+            book['calibre_id'] = mi.id
+            book['title'] = mi.title
+            book['author'] = authors_to_string(mi.authors)
+            book['author_sort'] = mi.author_sort
+            # book['series'] = mi.series
+            # if mi.series:
+            #     book['series_index'] = mi.series_index
+            # else:
+            #     book['series_index'] = 0
+                
+            book['comment'] = ''
+            book['url'] = ""
+            book['added'] = False
+            
+            url = self._get_story_url(db,book_id)
+            self._set_book_url_and_comment(book,url)
+                    
+            books.append(book)
+        return books
+
+    def _set_book_url_and_comment(self,book,url):
+        if not url:
+            book['comment'] = "No story URL found."
+            book['good'] = False
+            book['icon'] = 'search_delete_saved.png'
+        else:
+            # get normalized url or None.
+            book['url'] = self._is_good_downloader_url(url)
+            if book['url'] == None:
+                book['url'] = url
+                book['comment'] = "URL is not a valid story URL."
+                book['good'] = False
+                book['icon']='dialog_error.png'
+    
+    def _get_story_url(self, db, book_id):
+        identifiers = db.get_identifiers(book_id,index_is_id=True)
+        if 'url' in identifiers:
+            # identifiers have :->| in url.
+            #print("url from book:"+identifiers['url'].replace('|',':'))
+            return identifiers['url'].replace('|',':')
+        else:
+            ## only epub has URL in it--at least where I can easily find it.
+            if db.has_format(book_id,'EPUB',index_is_id=True):
+                existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
+                mi = get_metadata(existingepub,'EPUB')
+                #print("mi:%s"%mi)
+                identifiers = mi.get_identifiers()
+                if 'url' in identifiers:
+                    #print("url from epub:"+identifiers['url'].replace('|',':'))
+                    return identifiers['url'].replace('|',':')
+        return None
+
+    def _is_good_downloader_url(self,url):
+        # this is the accepted way to 'check for existance'?  really?
+        try:
+            self.dummyconfig
+        except AttributeError:
+            self.dummyconfig = SafeConfigParser()
+        # pulling up an adapter is pretty low over-head.  If
+        # it fails, it's a bad url.
+        try:
+            adapter = adapters.getAdapter(self.dummyconfig,url)
+            url = adapter.url
+            del adapter
+            return url
+        except:
+            return None;
+
+
+
+def get_job_details(job):
+    '''
+    Convert the job result into a set of parameters including a detail message
+    summarising the success of the extraction operation.
+    This is used by both the threaded and worker approaches to extraction
+    '''
+    extracted_ids, same_isbn_ids, failed_ids, no_format_ids = job.result
+    if not hasattr(job, 'html_details'):
+        job.html_details = job.details
+    det_msg = []
+    for i, title in failed_ids:
+        if i in no_format_ids:
+            msg = title + ' (No formats)'
+        else:
+            msg = title + ' (ISBN not found)'
+        det_msg.append(msg)
+    if same_isbn_ids:
+        if det_msg:
+            det_msg.append('----------------------------------')
+        for i, title in same_isbn_ids:
+            msg = title + ' (Same ISBN)'
+            det_msg.append(msg)
+    if len(extracted_ids) > 0:
+        if det_msg:
+            det_msg.append('----------------------------------')
+        for i, title, last_modified, isbn in extracted_ids:
+            msg = '%s (Extracted %s)'%(title, isbn)
+            det_msg.append(msg)
+
+    det_msg = '\n'.join(det_msg)
+    return extracted_ids, same_isbn_ids, failed_ids, det_msg
+
+def get_url_list(urls):
+    def f(x):
+        if x.strip(): return True
+        else: return False
+    # set removes dups.
+    return set(filter(f,urls.strip().splitlines()))
+

calibre-plugin/jobs.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Jim Miller'
+__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import time, os, traceback
+
+from ConfigParser import SafeConfigParser
+from StringIO import StringIO
+#from itertools import izip
+#from threading import Event
+
+#from calibre.gui2.convert.single import sort_formats_by_preference
+from calibre.utils.config import prefs
+from calibre.utils.ipc.server import Server
+from calibre.utils.ipc.job import ParallelJob
+from calibre.utils.logging import Log
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
+    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge
+
+# ------------------------------------------------------------------------------
+#
+#              Functions to perform downloads using worker jobs
+#
+# ------------------------------------------------------------------------------
+
+def do_download_worker(book_list, options,
+                       cpus, notification=lambda x,y:x):
+    '''
+    Master job, to launch child jobs to extract ISBN for a set of books
+    This is run as a worker job in the background to keep the UI more
+    responsive and get around the memory leak issues as it will launch
+    a child job for each book as a worker process
+    '''
+    server = Server(pool_size=cpus)
+
+    print(options['version'])
+    total = 0
+    # Queue all the jobs
+    for book in book_list:
+        if book['good']:
+            total += 1
+            args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
+                    'do_download_for_worker',
+                    (book,options)]
+            job = ParallelJob('arbitrary',
+                              "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
+                              done=None,
+                              args=args)
+            job._book = book
+            # job._book_id = book_id
+            # job._title = title
+            # job._modified_date = modified_date
+            # job._existing_isbn = existing_isbn
+            server.add_job(job)
+    
+    # This server is an arbitrary_n job, so there is a notifier available.
+    # Set the % complete to a small number to avoid the 'unavailable' indicator
+    notification(0.01, 'Downloading FanFiction Stories')
+
+    # dequeue the job results as they arrive, saving the results
+    count = 0
+    while True:
+        job = server.changed_jobs_queue.get()
+        # A job can 'change' when it is not finished, for example if it
+        # produces a notification. Ignore these.
+        job.update()
+        if not job.is_finished:
+            continue
+        # A job really finished. Get the information.
+        output_book = job.result
+        #print("output_book:%s"%output_book)
+        book_list.remove(job._book)
+        book_list.append(job.result)
+        book_id = job._book['calibre_id']
+        #title = job._title
+        count = count + 1
+        notification(float(count)/total, 'Downloaded Story')
+        # Add this job's output to the current log
+        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        print(job.details)
+
+        if count >= total:
+            # All done!
+            break
+
+    server.close()
+    
+    # return the book list as the job result
+    return book_list
+
+def do_download_for_worker(book,options):
+    '''
+    Child job, to extract isbn from formats for this specific book,
+    when run as a worker job
+    '''
+    try:
+        book['comment'] = 'Download started...'
+
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        ffdlconfig.readfp(StringIO(options['personal.ini']))
+        
+        adapter = adapters.getAdapter(ffdlconfig,book['url'])
+        adapter.is_adult = book['is_adult'] 
+        adapter.username = book['username'] 
+        adapter.password = book['password']
+        
+        story = adapter.getStoryMetadataOnly()
+        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+
+        outfile = book['outfile']
+
+        ## No need to download at all.  Shouldn't ever get down here.
+        if options['collision'] in (CALIBREONLY):
+            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            book['comment'] = 'Metadata collected.'
+            
+        ## checks were done earlier, it's new or not dup or newer--just write it.
+        elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
+                ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
+            print("write to %s"%outfile)
+            writer.writeStory(outfilename=outfile, forceOverwrite=True)
+            book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
+            
+        ## checks were done earlier, just update it.
+        elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
+            
+            urlchaptercount = int(story.getMetadata('numChapters'))
+            ## First, get existing epub with titlepage and tocpage stripped.
+            updateio = StringIO()
+            (epuburl,chaptercount) = doMerge(updateio,
+                                             [book['epub_for_update']],
+                                             titlenavpoints=False,
+                                             striptitletoc=True,
+                                             forceunique=False)
+            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+            print("write to %s"%outfile)
+
+            ## Get updated title page/metadata by itself in an epub.
+            ## Even if the title page isn't included, this carries the metadata.
+            titleio = StringIO()
+            writer.writeStory(outstream=titleio,metaonly=True)
+
+            newchaptersio = None
+            if urlchaptercount > chaptercount :
+                ## Go get the new chapters
+                newchaptersio = StringIO()
+                adapter.setChaptersRange(chaptercount+1,urlchaptercount)
+
+                adapter.config.set("overrides",'include_tocpage','false')
+                adapter.config.set("overrides",'include_titlepage','false')
+                writer.writeStory(outstream=newchaptersio)
+
+            ## Merge the three epubs together.
+            doMerge(outfile,
+                    [titleio,updateio,newchaptersio],
+                    fromfirst=True,
+                    titlenavpoints=False,
+                    striptitletoc=False,
+                    forceunique=False)
+            
+            book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
+                (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
+        
+    except NotGoingToDownload as d:
+        book['good']=False
+        book['comment']=unicode(d)
+        book['icon'] = d.icon
+
+    except Exception as e:
+        book['good']=False
+        book['comment']=unicode(e)
+        book['icon']='dialog_error.png'
+        print("Exception: %s:%s"%(book,unicode(e)))
+        traceback.print_exc()
+
+    #time.sleep(10)
+    return book

cron.yaml

@@ -3,6 +3,8 @@ cron:
   url: /r3m0v3r
   schedule: every 2 hours
 
-- description: orphan cleanup job
-  url: /r3m0v3rOrphans
-  schedule: every 4 hours
+# There's a bug in the Python 2.7 runtime that prevents this from
+# working properly.  In theory, there should never be orphans anyway.
+#- description: orphan cleanup job
+#  url: /r3m0v3rOrphans
+#  schedule: every 4 hours

defaults.ini

@@ -88,8 +88,10 @@ include_tocpage: true
 #output_filename: books/${title}-${siteabbrev}_${storyId}${formatext}
 #output_filename: books/${formatname}/${siteabbrev}/${authorId}/${title}-${siteabbrev}_${storyId}${formatext}
 output_filename: ${title}-${siteabbrev}_${storyId}${formatext}
+
 ## Make directories as needed.
 make_directories: true
+
 ## Always overwrite output files.  Otherwise, the downloader checks
 ## the timestamp on the existing file and only overwrites if the story
 ## has been updated more recently.  Command line version only
@@ -97,6 +99,7 @@ make_directories: true
 
 ## put output (with output_filename) in a zip file zip_filename.
 zip_output: false
+
 ## Can include directories.  .zip will be added if not in name somewhere
 zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 
@@ -105,6 +108,10 @@ zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 ## zip_filename.  
 allow_unsafe_filename: false
 
+## entries to make epub subjects and calibre tags
+## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
+include_subject_tags: extratags, genre, category, characters, lastupdate, status
+
 ## extra tags (comma separated) to include, primarily for epub.
 extratags: FanFiction
 
@@ -113,6 +120,15 @@ extratags: FanFiction
 ## Primarily for commandline.
 #slow_down_sleep_time:0.5
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Must be hex code, # will be added.
+background_color: ffffff
+
+## For use only with stand-alone CLI version--run a command on the
+## generated file after it's produced.  All of the titlepage_entries
+## values are available, plus output_filename.
+#post_process_cmd: addbook -f "${output_filename}" -t "${title}"
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -128,10 +144,6 @@ windows_eol: true
 ## epub is already a zip file.
 zip_output: false
 
-## entries to make epub subject tags
-## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
-include_subject_tags: extratags, genre, category, characters, lastupdate, status
-
 ## epub carries the TOC in metadata.
 ## mobi generated from epub will have a TOC at the end.
 include_tocpage: false
@@ -245,6 +257,12 @@ output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

downloader.py

@@ -24,6 +24,9 @@ from os.path import normpath, expanduser, isfile, join
 from StringIO import StringIO
 from optparse import OptionParser      
 import getpass
+import string
+
+from subprocess import call
 
 from epubmerge import doMerge
 
@@ -38,7 +41,9 @@ import ConfigParser
 def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     writer = writers.getWriter(writeformat,config,adapter)
     writer.writeStory(outstream=outstream,metaonly=metaonly)
+    output_filename=writer.getOutputFileName()
     del writer
+    return output_filename
 
 def main():
 
@@ -65,7 +70,7 @@ def main():
                      help="Update an existing epub with new chapter, give epub filename instead of storyurl.  Not compatible with inserted TOC.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
-                     help="Force update of an existing epub, download and overwrite all chapters.",)
+                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
    
    (options, args) = parser.parse_args()
 
@@ -97,6 +102,10 @@ def main():
        config.add_section("overrides")
    except ConfigParser.DuplicateSectionError:
        pass
+
+   if options.force:
+       config.set("overrides","always_overwrite","true")
+       
    if options.options:
        for opt in options.options:
            (var,val) = opt.split('=')
@@ -112,27 +121,28 @@ def main():
                                         striptitletoc=True,
                                         forceunique=False)
            print "Updating %s, URL: %s" % (args[0],url)
-           filename = args[0]
+           output_filename = args[0]
            config.set("overrides","output_filename",args[0])
        else:
            url = args[0]
 
        adapter = adapters.getAdapter(config,url)
-           
-       try:
-           adapter.getStoryMetadataOnly()
-       except exceptions.FailedToLogin:
-           print "Login Failed, Need Username/Password."
-           sys.stdout.write("Username: ")
-           adapter.username = sys.stdin.readline().strip()
-           adapter.password = getpass.getpass(prompt='Password: ')
-           #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
-           adapter.getStoryMetadataOnly()
-       except exceptions.AdultCheckRequired:
-           print "Please confirm you are an adult in your locale: (y/n)?"
-           if sys.stdin.readline().strip().lower().startswith('y'):
-               adapter.is_adult=True
-           adapter.getStoryMetadataOnly()
+
+       ## three tries, that's enough if both user/pass & is_adult needed,
+       ## or a couple tries of one or the other
+       for x in range(0,2):
+           try:
+               adapter.getStoryMetadataOnly()
+           except exceptions.FailedToLogin:
+               print "Login Failed, Need Username/Password."
+               sys.stdout.write("Username: ")
+               adapter.username = sys.stdin.readline().strip()
+               adapter.password = getpass.getpass(prompt='Password: ')
+               #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
+           except exceptions.AdultCheckRequired:
+               print "Please confirm you are an adult in your locale: (y/n)?"
+               if sys.stdin.readline().strip().lower().startswith('y'):
+                   adapter.is_adult=True
 
        if options.update and not options.force:
            urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
@@ -184,16 +194,14 @@ def main():
            
            adapter.setChaptersRange(options.begin,options.end)
            
-           if options.format == "all":
-               ## For testing.  Doing all three formats actually causes
-               ## some interesting config issues with format-specific
-               ## sections.  But it should rarely be an issue.
-               writeStory(config,adapter,"epub",options.metaonly)
-               writeStory(config,adapter,"html",options.metaonly)
-               writeStory(config,adapter,"txt",options.metaonly)
-           else:
-               writeStory(config,adapter,options.format,options.metaonly)
+           output_filename=writeStory(config,adapter,options.format,options.metaonly)
        
+       if not options.metaonly and adapter.getConfig("post_process_cmd"):
+           metadata = adapter.story.metadata
+           metadata['output_filename']=output_filename
+           call(string.Template(adapter.getConfig("post_process_cmd"))
+                .substitute(metadata), shell=True)
+           
        del adapter
    
    except exceptions.InvalidStoryURL, isu:

epubmerge.py

@@ -216,7 +216,7 @@ def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None,
                         try:
                             outputepub.writestr(href,
                                                 epub.read(relpath+item.getAttribute("href")))
-                            if re.match(r'.*/(file|chapter)\d+\.xhtml',href):
+                            if re.match(r'.*/(file|chapter)\d+\.x?html',href):
                                 filecount+=1
                             items.append((id,href,item.getAttribute("media-type")))
                             filelist.append(href)
@@ -224,10 +224,21 @@ def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None,
                             pass # Skip missing files.
                 
         for itemref in metadom.getElementsByTagName("itemref"):
+
             if not striptitletoc or not re.match(r'(title|toc)_page', itemref.getAttribute("idref")):
                 itemrefs.append(bookid+itemref.getAttribute("idref"))
 
         booknum=booknum+1;
+        if not forceunique:
+            # If not forceunique, it's an epub update.
+            # If there's a "calibre_bookmarks.txt", it's from reading
+            # in Calibre and should be preserved.
+            try:
+                fn = "META-INF/calibre_bookmarks.txt"
+                outputepub.writestr(fn,epub.read(fn))
+            except:
+                pass
+        
 
     ## create content.opf file. 
     uniqueid="epubmerge-uid-%d" % time() # real sophisticated uid scheme.
@@ -355,7 +366,7 @@ def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None,
     ## during TOC generation to save loops.
     outputepub.writestr("content.opf",contentdom.toxml('utf-8'))
     outputepub.writestr("toc.ncx",tocncxdom.toxml('utf-8'))
-    
+
     # declares all the files created by Windows.  otherwise, when
     # it runs in appengine, windows unzips the files as 000 perms.
     for zf in outputepub.filelist:

fanficdownloader/adapters/__init__.py

@@ -15,21 +15,52 @@
 # limitations under the License.
 #
 
-import os, re, sys, glob
+import os, re, sys, glob, types
 from os.path import dirname, basename, normpath
 import logging
 import urlparse as up
 
-import fanficdownloader.exceptions as exceptions
+from .. import exceptions as exceptions
+
+## must import each adapter here.
+
+import adapter_test1
+import adapter_fanfictionnet
+import adapter_castlefansorg
+import adapter_fanfictionnet
+import adapter_fictionalleyorg
+import adapter_fictionpresscom
+import adapter_ficwadcom
+import adapter_fimfictionnet
+import adapter_harrypotterfanfictioncom
+import adapter_mediaminerorg
+import adapter_potionsandsnitchesnet
+import adapter_tenhawkpresentscom
+import adapter_adastrafanficcom
+import adapter_thewriterscoffeeshopcom
+import adapter_tthfanficorg
+import adapter_twilightednet
+import adapter_twiwritenet
+import adapter_whoficcom
+import adapter_siyecouk
 
 ## This bit of complexity allows adapters to be added by just adding
-## the source file.  It eliminates the long if/else clauses we used to
-## need to pick out the adapter.
+## importing.  It eliminates the long if/else clauses we used to need
+## to pick out the adapter.
     
 ## List of registered site adapters.
-    
 __class_list = []
 
+def imports():
+    for name, val in globals().items():
+        if isinstance(val, types.ModuleType):
+            yield val.__name__
+
+for x in imports():
+    if "fanficdownloader.adapters.adapter_" in x:
+        #print x
+        __class_list.append(sys.modules[x].getClass())
+            
 def getAdapter(config,url):
     ## fix up leading protocol.
     fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
@@ -63,16 +94,3 @@ def getClassFor(domain):
     for cls in __class_list:
         if cls.matchesSite(domain):
             return cls
-
-## Automatically import each adapter_*.py file.
-## Each implement getClass() to their class
-
-filelist = glob.glob(dirname(__file__)+'/adapter_*.py')
-sys.path.insert(0,normpath(dirname(__file__)))
-
-for file in filelist:
-    #print "file: "+basename(file)[:-3]
-    module = __import__(basename(file)[:-3])
-    __class_list.append(module.getClass())
-
-del sys.path[0]

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -21,9 +21,9 @@ import re
 import urllib
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -20,9 +20,9 @@ import logging
 import re
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 
@@ -51,11 +51,11 @@ from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 # updated to reflect the class below it.  That, plus getSiteDomain()
 # take care of 'Registering'.
 def getClass():
-    return FanficCastleTVNetAdapter # XXX
+    return CastleFansOrgAdapter # XXX
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
+class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -74,10 +74,10 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
         logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/fanfic/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','csltv') # XXX
+        self.story.setMetadata('siteabbrev','cslf') # XXX
 
         # If all stories from the site fall into the same category,
         # the site itself isn't likely to label them as such, so we
@@ -91,13 +91,13 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'fanfic.castletv.net' # XXX
+        return 'castlefans.org' # XXX
 
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -120,7 +120,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
         params['cookiecheck'] = '1'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfic/user.php?action=login'
         logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
@@ -192,7 +192,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfic/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -21,8 +21,8 @@ import re
 import urllib2
 import time
 
-import fanficdownloader.BeautifulSoup as bs
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -21,9 +21,9 @@ import re
 import urllib
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -22,9 +22,9 @@ import urllib2
 import time
 import httplib, urllib
 
-import fanficdownloader.BeautifulSoup as bs
-import fanficdownloader.exceptions as exceptions
-from fanficdownloader.htmlcleanup import stripHTML
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
+from ..htmlcleanup import stripHTML
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -21,9 +21,9 @@ import re
 import urllib2
 import cookielib as cl
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 
@@ -87,8 +87,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             raise exceptions.AdultCheckRequired(self.url)
         
         soup = bs.BeautifulSoup(data).find("div", {"class":"content_box post_content_box"})
-        
-        title, author = [link.text for link in soup.find("h2").findAll("a")]
+
+        titleheader = soup.find("h2")
+        title = titleheader.find("a", href=re.compile(r'^/story/')).text
+        author = titleheader.find("a", href=re.compile(r'^/user/')).text
         self.story.setMetadata("title", title)
         self.story.setMetadata("author", author)
         self.story.setMetadata("authorId", author) # The author's name will be unique
@@ -126,7 +128,9 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         status = status_bar.text.split("|")[0].strip().replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
         self.story.setMetadata('status', status)
         self.story.setMetadata('rating', status_bar.span.text)
-        self.story.setMetadata('numWords', status_bar.div.b.text)
+        # This way is less elegant, perhaps, but more robust in face of format changes.
+        numWords = status_bar.find("div",{"class":"word_count"}).b.text
+        self.story.setMetadata('numWords', numWords)
         
         description_soup = soup.find("div", {"class":"description"})
         # Sometimes the description has an expanding element
@@ -158,4 +162,4 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return utf8FromSoup(soup)
-        
+        

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -21,9 +21,9 @@ import re
 import urllib
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -21,9 +21,9 @@ import re
 import urllib
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -21,9 +21,9 @@ import re
 import urllib
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/adapter_siyecouk.py

@@ -0,0 +1,279 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return SiyeCoUkAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8",]# 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        # self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        # self.password = ""
+        # self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','siye') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.siye.co.uk' # XXX
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.siye.co.uk','siye.co.uk']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+r"(www\.)?"+re.escape("siye.co.uk/siye/viewstory.php?sid=")+r"\d+$"
+
+    # ## Login seems to be reasonably standard across eFiction sites.
+    # def needToLoginCheck(self, data):
+    #     if 'Registered Users Only' in data \
+    #             or 'There is no such account on our website' in data \
+    #             or "That password doesn't match the one in our database" in data:
+    #         return True
+    #     else:
+    #         return False
+        
+    # def performLogin(self, url):
+    #     params = {}
+
+    #     if self.password:
+    #         params['penname'] = self.username
+    #         params['password'] = self.password
+    #     else:
+    #         params['penname'] = self.getConfig("username")
+    #         params['password'] = self.getConfig("password")
+    #     params['cookiecheck'] = '1'
+    #     params['submit'] = 'Submit'
+    
+    #     loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+    #     logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+    #                                                           params['penname']))
+    
+    #     d = self._fetchUrl(loginUrl, params)
+    
+    #     if "Member Account" not in d : #Member Account
+    #         logging.info("Failed to login to URL %s as %s" % (loginUrl,
+    #                                                           params['penname']))
+    #         raise exceptions.FailedToLogin(url,params['penname'])
+    #         return False
+    #     else:
+    #         return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # if self.is_adult or self.getConfig("is_adult"):
+        #     # Weirdly, different sites use different warning numbers.
+        #     # If the title search below fails, there's a good chance
+        #     # you need a different number.  print data at that point
+        #     # and see what the 'click here to continue' url says.
+        #     addurl = "&ageconsent=ok&warning=4" # XXX
+        # else:
+        #     addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        # Except it doesn't this time. :-/
+        url = self.url #+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # if self.needToLoginCheck(data):
+        #     # need to log in for this one.
+        #     self.performLogin(url)
+        #     data = self._fetchUrl(url)
+
+        # # The actual text that is used to announce you need to be an
+        # # adult varies from site to site.  Again, print data before
+        # # the title search to troubleshoot.
+        # if "Age Consent Required" in data: # XXX 
+        #     raise exceptions.AdultCheckRequired(self.url)
+            
+        # if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+        #     raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/siye/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # need(or easier) to pull other metadata from the author's list page.
+        authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+
+        ## Title
+        titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',titlea.string)
+        
+        # Find the chapters (from soup, not authsoup):
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/siye/'+chapter['href']))
+
+        if self.chapterUrls:
+            self.story.setMetadata('numChapters',len(self.chapterUrls))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+            self.story.setMetadata('numChapters',1)
+
+        # The stuff we can get from the chapter list/one-shot page are
+        # in the first table with 95% width.
+        metatable = soup.find('table',{'width':'95%'})
+
+        # Categories
+        cat_as = metatable.findAll('a', href=re.compile(r'categories.php'))
+        for cat_a in cat_as:
+            self.story.addToList('category',stripHTML(cat_a))
+
+        moremetaparts = stripHTML(metatable).split('\n')
+        for part in moremetaparts:
+            part = part.strip()
+            if part.startswith("Characters:"):
+                part = part[part.find(':')+1:]
+                for item in part.split(','):
+                    if item.strip() != "None":
+                        self.story.addToList('characters',item)
+
+            if part.startswith("Genres:"):
+                part = part[part.find(':')+1:]
+                for item in part.split(','):
+                    if item.strip() != "None":
+                        self.story.addToList('genre',item)
+
+            if part.startswith("Warnings:"):
+                part = part[part.find(':')+1:]
+                for item in part.split(','):
+                    if item.strip() != "None":
+                        self.story.addToList('warnings',item)
+
+            if part.startswith("Rating:"):
+                part = part[part.find(':')+1:]
+                self.story.setMetadata('rating',part)
+
+            if part.startswith("Summary:"):
+                part = part[part.find(':')+1:]
+                self.story.setMetadata('description',part)
+
+        
+
+        # want to get the next tr of the table.
+        #print("%s"%titlea.parent.parent.findNextSibling('tr'))
+            
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        moremeta = stripHTML(titlea.parent.parent.findNextSibling('tr'))
+        for part in moremeta.replace(' - ','\n').split('\n'):
+            #print("part:%s"%part)
+            try:
+                (name,value) = part.split(': ')
+            except:
+                # not going to worry about fancier processing for the bits
+                # that don't match.
+                continue
+            name=name.strip()
+            value=value.strip()
+            if name == 'Published':
+                self.story.setMetadata('datePublished', makeDate(value, self.dateformat))
+            if name == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat))
+            if name == 'Completed':
+                if value == 'Yes':
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+            if name == 'Words':
+                self.story.setMetadata('numWords', value)
+
+        
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        # soup = bs.BeautifulSoup(self._fetchUrl(url))
+        # BeautifulSoup objects to <p> inside <span>, which
+        # technically isn't allowed.
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        # not the most unique thing in the world, but it appears to be
+        # the best we can do here.
+        story = soup.find('span', {'style' : 'font-size: 100%;'})
+        
+        if None == story:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return utf8FromSoup(story)

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -21,9 +21,9 @@ import re
 import urllib
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/adapter_test1.py

@@ -19,8 +19,8 @@ import datetime
 import time
 import logging
 
-import fanficdownloader.BeautifulSoup as bs
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from .. import exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 
@@ -54,6 +54,12 @@ class TestSiteAdapter(BaseSiteAdapter):
         if self.story.getMetadata('storyId') == '666':
             raise exceptions.StoryDoesNotExist(self.url)
 
+        if self.story.getMetadata('storyId').startswith('670'):
+            time.sleep(1.0)
+            
+        if self.story.getMetadata('storyId').startswith('671'):
+            time.sleep(1.0)
+            
         if self.getConfig("username"):
             self.username = self.getConfig("username")
         
@@ -61,25 +67,26 @@ class TestSiteAdapter(BaseSiteAdapter):
             raise exceptions.FailedToLogin(self.url,self.username)
 
         if self.story.getMetadata('storyId') == '664':
-            self.story.setMetadata(u'title',"Test Story Title "+self.crazystring)
+            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring)
+            self.story.setMetadata('author','Test Author aa bare amp(&) quote(') amp(&)')
         else:
-            self.story.setMetadata(u'title',"Test Story Title")
+            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
+            self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('storyUrl',self.url)
         self.story.setMetadata('description',u'Description '+self.crazystring+u''' Done
 
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
-        self.story.setMetadata('datePublished',makeDate("1972-01-31","%Y-%m-%d"))
+        self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
         self.story.setMetadata('dateCreated',datetime.datetime.now())
         if self.story.getMetadata('storyId') == '669':
             self.story.setMetadata('dateUpdated',datetime.datetime.now())
         else:
-            self.story.setMetadata('dateUpdated',makeDate("1975-01-31","%Y-%m-%d"))
+            self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
         self.story.setMetadata('numWords','123456')
         self.story.setMetadata('status','In-Completed')
         self.story.setMetadata('rating','Tweenie')
         
-        self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('authorId','98765')
         self.story.setMetadata('authorUrl','http://author/url')
 
@@ -100,7 +107,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
-                            # ('Chapter 6',self.url+"&chapter=6"),
+                            ('Chapter 6',self.url+"&chapter=6"),
                             # ('Chapter 7',self.url+"&chapter=6"),
                             # ('Chapter 8',self.url+"&chapter=6"),
                             # ('Chapter 9',self.url+"&chapter=6"),
@@ -128,8 +135,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         if self.story.getMetadata('storyId') == '667':
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
 
-        if self.story.getMetadata('storyId') == '670':
-            time.sleep(2.0)
+        if self.story.getMetadata('storyId').startswith('670') or \
+                self.story.getMetadata('storyId').startswith('672'):
+            time.sleep(1.0)
 
         if "chapter=1" in url :
             text=u'''
@@ -143,6 +151,8 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
+<p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
+<p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
 <p>And other storyId will succeed with the same output.</p>
 </div>
 '''

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -21,9 +21,9 @@ import re
 import urllib
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+import time
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+
+class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev','tth')
+        self.dateformat = "%d %b %y"
+        self.is_adult=False
+        self.username = None
+        self.password = None
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL("http://"+self.getSiteDomain()\
+                         +"/Story-"+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+
+    @staticmethod
+    def getSiteDomain():
+        return 'www.tthfanfic.org'
+
+    def getSiteExampleURLs(self):
+        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm ttp://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
+
+    # http://www.tthfanfic.org/T-526321777848988007890480555880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm
+    # http://www.tthfanfic.org/Story-5583
+    # http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm
+    def getSiteURLPattern(self):
+        return r"http://www.tthfanfic.org/(T-\d+/)?Story-(?P<id>\d+)(-\d+)?(/.*)?$"
+
+    # tth won't send you future updates if you aren't 'caught up'
+    # on the story.  Login isn't required for F21, but logging in will
+    # mark stories you've downloaded as 'read' on tth.
+    def performLogin(self):
+        params = {}
+
+        if self.password:
+            params['urealname'] = self.username
+            params['password'] = self.password
+        else:
+            params['urealname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['loginsubmit'] = 'Login'
+
+        if not params['password']:
+            return
+        
+        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['urealname']))
+
+        ## need to pull empty login page first to get ctkn and
+        ## password name, which are BUSs
+# <form method='post' action='/login.php' accept-charset="utf-8">
+# <input type='hidden' name='ctkn' value='4bdf761f5bea06bf4477072afcbd0f8d721d1a4f989c09945a9e87afb7a66de1'/>
+# <input type='text' id='urealname' name='urealname' value=''/>
+# <input type='password' id='password' name='6bb3fcd148d148629223690bf19733b8'/>
+# <input type='submit' value='Login' name='loginsubmit'/>
+        soup = bs.BeautifulSoup(self._fetchUrl(loginUrl))
+        params['ctkn']=soup.find('input', {'name':'ctkn'})['value']
+        params[soup.find('input', {'id':'password'})['name']] = params['password']
+        
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Stories Published" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    def extractChapterUrlsAndMetadata(self):
+        # fetch the chapter.  From that we will get almost all the
+        # metadata and chapter list
+
+        url=self.url
+        logging.debug("URL: "+url)
+
+        # tth won't send you future updates if you aren't 'caught up'
+        # on the story.  Login isn't required for F21, but logging in will
+        # mark stories you've downloaded as 'read' on tth.
+        self.performLogin()
+        
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        try:
+            data = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+            
+        if "<h2>Story Not Found</h2>" in data:
+            raise exceptions.StoryDoesNotExist(url)
+        
+        if "NOTE: This story is rated FR21 which is above your chosen filter level." in data:
+            if self.is_adult or self.getConfig("is_adult"):
+                form = soup.find('form', {'id':'sitemaxratingform'})
+                params={'ctkn':form.find('input', {'name':'ctkn'})['value'],
+                        'sitemaxrating':'5'}
+                logging.info("Attempting to get rating cookie for %s" % url)
+                data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params)
+                # refetch story page.
+                data = self._fetchUrl(url)
+                soup = bs.BeautifulSoup(data)
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+
+        # http://www.tthfanfic.org/AuthorStories-3449/Greywizard.htm
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"^/AuthorStories-\d+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',stripHTML(a))
+
+        try:
+            # going to pull part of the meta data from author list page.
+            logging.debug("author URL: "+self.story.getMetadata('authorUrl'))
+            authordata = self._fetchUrl(self.story.getMetadata('authorUrl'))
+            authorsoup = bs.BeautifulSoup(authordata)
+            # author can have several pages, scan until we find it.
+            while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
+                nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
+                logging.debug("author nextpage URL: "+nextpage)
+                authordata = self._fetchUrl(nextpage)
+                authorsoup = bs.BeautifulSoup(authordata)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+
+        storydiv = authorsoup.find('div', {'id':'st'+self.story.getMetadata('storyId'), 'class':re.compile(r"storylistitem")})
+        self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'})))
+        self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'})))
+
+        verticaltable = soup.find('table', {'class':'verticaltable'})
+
+        BtVS = True
+        for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
+            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
+                self.story.addToList('category',cat.string)
+            else:
+                if 'Non-BtVS' in cat.string:
+                    BtVS = False
+        if BtVS:
+            self.story.addToList('category','Buffy: The Vampire Slayer')
+
+        verticaltabletds = verticaltable.findAll('td')
+        self.story.setMetadata('rating', verticaltabletds[2].string)
+        self.story.setMetadata('numWords', verticaltabletds[4].string)
+        
+        # Complete--if completed.
+        if 'Yes' in verticaltabletds[10].string:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+            
+        self.story.setMetadata('datePublished',makeDate(stripHTML(verticaltabletds[8].string), self.dateformat))
+        self.story.setMetadata('dateUpdated',makeDate(stripHTML(verticaltabletds[9].string), self.dateformat))
+
+        for icon in storydiv.find('span',{'class':'storyicons'}).findAll('img'):
+            if( icon['title'] not in ['Non-Crossover'] ) :
+                self.story.addToList('genre',icon['title'])
+            
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'chapnav' } )
+    	 
+        if select is None:
+    	   # no selector found, so it's a one-chapter story.
+    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option')
+            for o in allOptions:
+                url = "http://"+self.host+o['value']
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(o),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        return
+
+
+    def getChapterText(self, url):
+        logging.debug('Getting chapter text from: %s' % url)
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+
+        div = soup.find('div', {'id' : 'storyinnerbody'})
+        
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        # strip out included chapter title, if present, to avoid doubling up.
+        try:
+            div.find('h3').extract()
+        except:
+            pass
+        return utf8FromSoup(div)
+
+def getClass():
+    return TwistingTheHellmouthSiteAdapter
+

fanficdownloader/adapters/adapter_twilightednet.py

@@ -21,9 +21,9 @@ import re
 import urllib
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -21,9 +21,9 @@ import re
 import urllib
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 
@@ -84,7 +84,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
@@ -119,6 +119,8 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
+        data = data[data.index("<body"):] # desperate--strip before <body
+        # in calibre plugin only, soup wasn't parsing the html properly.
         soup = bs.BeautifulSoup(data)
 
         ## Title
@@ -211,7 +213,10 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        data = data[data.index("<body"):] # desperate--strip before <body
+        # in calibre plugin only, soup wasn't parsing the html properly.
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})

fanficdownloader/adapters/adapter_whoficcom.py

@@ -20,8 +20,8 @@ import logging
 import re
 import urllib2
 
-import fanficdownloader.BeautifulSoup as bs
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 

fanficdownloader/adapters/base_adapter.py

@@ -39,13 +39,13 @@ except:
     pass
     #logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine")
 
-from fanficdownloader.story import Story
-from fanficdownloader.configurable import Configurable
-from fanficdownloader.htmlcleanup import removeEntities, removeAllEntities, stripHTML
-from fanficdownloader.exceptions import InvalidStoryURL
+from ..story import Story
+from ..configurable import Configurable
+from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML
+from ..exceptions import InvalidStoryURL
 
 try:
-    import fanficdownloader.chardet as chardet
+    from .. import chardet as chardet
 except ImportError:
     chardet = None
 
@@ -63,10 +63,15 @@ class BaseSiteAdapter(Configurable):
         return re.match(self.getSiteURLPattern(), self.url)
 
     def __init__(self, config, url):
+        self.config = config
         Configurable.__init__(self, config)
         self.addConfigSection(self.getSiteDomain())
         self.addConfigSection("overrides")
         
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
         self.opener = u2.build_opener(u2.HTTPCookieProcessor())
         self.storyDone = False
         self.metadataDone = False

fanficdownloader/story.py

@@ -17,7 +17,7 @@
 
 import os
 
-from htmlcleanup import conditionalRemoveEntities
+from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 
 class Story:
     
@@ -25,20 +25,22 @@ class Story:
         try:
             self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
         except:
-            self.metadata = {'version':'4.0'}
+            self.metadata = {'version':'4.2'}
         self.chapters = [] # chapters will be tuples of (title,html)
         self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
 
     def setMetadata(self, key, value):
+        ## still keeps < < and &
         self.metadata[key]=conditionalRemoveEntities(value)
 
     def getMetadataRaw(self,key):
         if self.metadata.has_key(key):
             return self.metadata[key]
         
-    def getMetadata(self, key):
+    def getMetadata(self, key, removeallentities=False):
+        value = None
         if self.getLists().has_key(key):
-            return ', '.join(self.getList(key))
+            value = ', '.join(self.getList(key))
         if self.metadata.has_key(key):
             value = self.metadata[key]
             if value:
@@ -48,16 +50,33 @@ class Story:
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
                  value = value.strftime("%Y-%m-%d")
+                 
+        if removeallentities and value != None:
+            return removeAllEntities(value)
+        else:
             return value
 
+    def getAllMetadata(self):
+        '''
+        All single value *and* list value metadata as strings.
+        '''
+        allmetadata = {}
+        for k in self.metadata.keys():
+            allmetadata[k] = self.getMetadata(k)
+        for l in self.listables.keys():
+            allmetadata[l] = self.getMetadata(l)
+
+        return allmetadata
+        
     def addToList(self,listname,value):
         if value==None:
             return
+        value = conditionalRemoveEntities(value)
         if not self.listables.has_key(listname):
             self.listables[listname]=[]
         # prevent duplicates.
         if not value in self.listables[listname]:
-            self.listables[listname].append(conditionalRemoveEntities(value))
+            self.listables[listname].append(value)
 
     def getList(self,listname):
         if not self.listables.has_key(listname):

fanficdownloader/writers/__init__.py

@@ -18,7 +18,7 @@
 ## This could (should?) use a dynamic loader like adapters, but for
 ## now, it's static, since there's so few of them.
 
-from fanficdownloader.exceptions import FailedToDownload
+from ..exceptions import FailedToDownload
 
 from writer_html import HTMLWriter
 from writer_txt  import TextWriter

fanficdownloader/writers/base_writer.py

@@ -24,8 +24,8 @@ import zipfile
 from zipfile import ZipFile, ZIP_DEFLATED
 import logging
 
-from fanficdownloader.configurable import Configurable
-from fanficdownloader.htmlcleanup import removeEntities, removeAllEntities, stripHTML
+from ..configurable import Configurable
+from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML
 
 class BaseStoryWriter(Configurable):
 
@@ -102,6 +102,9 @@ class BaseStoryWriter(Configurable):
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())
 
+    def getMetadata(self,key):
+        return stripHTML(self.story.getMetadata(key))
+
     def getOutputFileName(self):
         if self.getConfig('zip_output'):
             return self.getZipFileName()
@@ -115,7 +118,7 @@ class BaseStoryWriter(Configurable):
         return self.formatFileName(self.getConfig('zip_filename'))
 
     def formatFileName(self,template):
-        values = self.story.metadata
+        values = origvalues = self.story.getAllMetadata()
         # fall back default:
         if not template:
             template="${title}-${siteabbrev}_${storyId}${formatext}"
@@ -123,7 +126,7 @@ class BaseStoryWriter(Configurable):
         if not self.getConfig('allow_unsafe_filename'):
             values={}
             pattern = re.compile(r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+")
-            for k in self.story.metadata.keys():
+            for k in origvalues.keys():
                 values[k]=re.sub(pattern,'_', removeAllEntities(self.story.getMetadata(k)))
 
         return string.Template(template).substitute(values).encode('utf8')
@@ -180,12 +183,13 @@ class BaseStoryWriter(Configurable):
             self._write(out,END.substitute(self.story.metadata))
 
     # if no outstream is given, write to file.
-    def writeStory(self,outstream=None,metaonly=False):
+    def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False):
         for tag in self.getConfigList("extratags"):
             self.story.addToList("extratags",tag)
 
         self.metaonly = metaonly
-        outfilename=self.getOutputFileName()
+        if outfilename == None:
+            outfilename=self.getOutputFileName()
 
         if not outstream:
             close=True
@@ -199,7 +203,7 @@ class BaseStoryWriter(Configurable):
                         os.mkdir(path) ## os.makedirs() doesn't work in 2.5.2?
 
             ## Check for output file date vs updated date here
-            if not self.getConfig('always_overwrite'):
+            if not (self.getConfig('always_overwrite') or forceOverwrite):
                 if os.path.exists(outfilename):
                     ## date() truncs off time, which files have, but sites don't report.
                     lastupdated=self.story.getMetadataRaw('dateUpdated').date()
@@ -242,6 +246,32 @@ class BaseStoryWriter(Configurable):
         if close:
             outstream.close()
 
+    def getTags(self):
+        # set to avoid duplicates subject tags.
+        subjectset = set()
+        
+        if self.story.getMetadataRaw('dateUpdated'):
+            # Last Update tags for Bill.
+            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update Year/Month: %Y/%m"))
+            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update: %Y/%m/%d"))
+        
+        for entry in self.validEntries:
+            if entry in self.getConfigList("include_subject_tags") and \
+                    entry not in self.story.getLists() and \
+                    self.story.getMetadata(entry):
+                subjectset.add(self.getMetadata(entry))
+                
+        # listables all go into dc:subject tags, but only if they are configured.
+        for (name,lst) in self.story.getLists().iteritems():
+            if name in self.getConfigList("include_subject_tags"):
+                for tag in lst:
+                    subjectset.add(tag)
+                    
+        for tag in self.getConfigList("extratags"):
+            subjectset.add(tag)
+
+        return list(subjectset)
+            
     def writeStoryImpl(self, out):
         "Must be overriden by sub classes."
         pass

fanficdownloader/writers/writer_epub.py

@@ -26,7 +26,7 @@ from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 from xml.dom.minidom import parse, parseString, getDOMImplementation
 
 from base_writer import *
-from fanficdownloader.htmlcleanup import stripHTML
+from ..htmlcleanup import stripHTML
 
 class EpubWriter(BaseStoryWriter):
 
@@ -41,7 +41,10 @@ class EpubWriter(BaseStoryWriter):
     def __init__(self, config, story):
         BaseStoryWriter.__init__(self, config, story)
 
-        self.EPUB_CSS='''body { margin-left: 2%; margin-right: 2%; margin-top: 2%; margin-bottom: 2%; text-align: justify; }
+        self.EPUB_CSS = string.Template('''
+body { margin: 2%;
+       text-align: justify;
+       background-color: #${background_color}; }
 pre { font-size: x-small; }
 sml { font-size: small; }
 h1 { text-align: center; }
@@ -63,7 +66,7 @@ h6 { text-align: center; }
 .smcap    {font-variant: small-caps;}
 .u        {text-decoration: underline;}
 .bold     {font-weight: bold;}
-'''
+''')
         
         self.EPUB_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -153,9 +156,6 @@ h6 { text-align: center; }
 </html>
 ''')
 
-    def getMetadata(self,key):
-        return stripHTML(self.story.getMetadata(key))
-
     def writeStoryImpl(self, out):
 
         ## Python 2.5 ZipFile is rather more primative than later
@@ -249,27 +249,12 @@ h6 { text-align: center; }
             metadata.appendChild(newTag(contentdom,"meta",
                                         attrs={"name":"calibre:timestamp",
                                                "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")}))
-            # Last Update tags for Bill.
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update Year/Month: %Y/%m"))
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update: %Y/%m/%d"))
         
         if self.getMetadata('description'):
             metadata.appendChild(newTag(contentdom,"dc:description",text=
                                         self.getMetadata('description')))
 
-        # set to avoid duplicates subject tags.
-        subjectset = set()
-        for entry in self.validEntries:
-            if entry in self.getConfigList("include_subject_tags") and \
-                    entry not in self.story.getLists() and \
-                    self.story.getMetadata(entry):
-                subjectset.add(self.getMetadata(entry))
-        # listables all go into dc:subject tags, but only if they are configured.
-        for (name,lst) in self.story.getLists().iteritems():
-            if name in self.getConfigList("include_subject_tags"):
-                for tag in lst:
-                    subjectset.add(tag)
-        for subject in subjectset:
+        for subject in self.getTags():
             metadata.appendChild(newTag(contentdom,"dc:subject",text=subject))
 
                     
@@ -376,7 +361,7 @@ h6 { text-align: center; }
         del tocncxdom
 
         # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS)
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute({"background_color":self.getConfig("background_color")})) 
 
         # write title page.
         if self.getConfig("titlepage_use_table"):

fanficdownloader/writers/writer_html.py

@@ -39,6 +39,7 @@ class HTMLWriter(BaseStoryWriter):
 <head>
 <title>${title} by ${author}</title>
 <style type="text/css">
+body { background-color: #${background_color}; }
 .CI {
     text-align:center;
     margin-top:0px;
@@ -94,6 +95,9 @@ class HTMLWriter(BaseStoryWriter):
 
     def writeStoryImpl(self, out):
 
+        # minor cheat, tucking bg into metadata.
+        if self.getConfig("background_color"):
+            self.story.metadata["background_color"] = self.getConfig("background_color")
         self._write(out,self.HTML_FILE_START.substitute(self.story.metadata))
 
         self.writeTitlePage(out,

fanficdownloader/writers/writer_mobi.py

@@ -20,8 +20,8 @@ import string
 import StringIO
 
 from base_writer import *
-from fanficdownloader.htmlcleanup import stripHTML
-from fanficdownloader.mobi import Converter
+from ..htmlcleanup import stripHTML
+from ..mobi import Converter
 
 class MobiWriter(BaseStoryWriter):
 
@@ -124,9 +124,6 @@ class MobiWriter(BaseStoryWriter):
 </html>
 ''')
 
-    def getMetadata(self,key):
-        return stripHTML(self.story.getMetadata(key))
-
     def writeStoryImpl(self, out):
 
         files = []

fanficdownloader/writers/writer_txt.py

@@ -21,7 +21,7 @@ from textwrap import wrap
 
 from base_writer import *
 
-from fanficdownloader.html2text import html2text, BODY_WIDTH
+from ..html2text import html2text, BODY_WIDTH
 
 ## In BaseStoryWriter, we define _write to encode <unicode> objects
 ## back into <string> for true output.  But txt needs to write the

index.html

@@ -54,33 +54,22 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>Please Switch to <a href="http://fanfictiondownloader.appspot.com/">New Version</a></h3>
+	  <h3>This is the Official Multithreading Version</h3>
 	  <p>
-	    We have a new, more efficient, version of the system up now.  Please start using the 
-	    <a href="http://fanfictiondownloader.appspot.com/">New
-	      Version here</a>.  If for some reason, the new version
-	      doesn't work for you, please let us know on
-	      the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
-	      Downloader Google Group</a>.
-	  </p>
-	  <h3>New Google Quotas</h3>
-	  <p>
-	    Google has changed their quota limits for free
-	    applications using their AppEngine system, like this one.
-	    We expect that there will be times when the
-	    system exceeds it's permitted processing quota.
+	    This version of the application uses Python 2.7 and
+	    multithreading to try and reduce our usage.  Google
+	    considers Python 2.7 Experimental still, so there may be issues.
 	  </p>
 	  <p>
-	    You also have the option of running the downloader on your
-	    own computer if you have Python available.
-	    <a href="http://code.google.com/p/fanficdownloader/downloads/list">Download here.</a>
+	    <b>Changed Site</b><br />
+	    fanfic.castletv.net changed to castlefans.org.
 	  </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
 	      Downloader Google Group</a>.  The
-	    <a href="http://4-0-6.fanfictionloader.appspot.com">Previous
+	    <a href="http://4-0-7.fanfictiondownloader.appspot.com">Previous
 	      Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
@@ -201,10 +190,10 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://fanfiction.tenhawkpresents.com/viewstory.php?sid=294">http://fanfiction.tenhawkpresents.com/viewstory.php?sid=294</a>.
 	  </dd>
-	  <dt>fanfic.castletv.net</dt>
+	  <dt>castlefans.org</dt>
 	  <dd>
 	    Use the URL of the story's chapter list, such as
-	    <br /><a href="http://fanfic.castletv.net/viewstory.php?sid=123">http://fanfic.castletv.net/viewstory.php?sid=123</a>.
+	    <br /><a href="http://castlefans.org/fanfic/viewstory.php?sid=123">http://castlefans.org/fanfic/viewstory.php?sid=123</a>.
 	  </dd>
 	  <dt>fimfiction.net</dt>
 	  <dd>
@@ -213,6 +202,18 @@
 	    <br /> or the URL of any chapter, such as
 	    <br /><a href="http://www.fimfiction.com/story/123/1/">http://www.fimfiction.com/story/123/1/</a>.
 	  </dd>
+	  <dt>tthfanfic.org</dt>
+	  <dd>
+	    Use the URL of any story, with or without chapter, title and notice, such as
+	    <br /><a href="http://www.tthfanfic.org/Story-5583">http://www.tthfanfic.org/Story-5583</a>
+	    <br /><a href="http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm">http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm</a>.
+	    <br /><a href="http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm">http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm</a>.
+	  </dd>
+	  <dt>www.siye.co.uk</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.siye.co.uk/siye/viewstory.php?sid=123">http://www.siye.co.uk/siye/viewstory.php?sid=123</a>.
+	  </dd>
 	</dl>	
 
 	

main.py

@@ -41,23 +41,24 @@ import ConfigParser
 ## Console page first, you will get a django version mismatch error when you
 ## to go hit one of the application pages.  Just change a file again, and
 ## make sure to hit an app page before the SDK page to clear it.
-os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
-from google.appengine.dist import use_library
-use_library('django', '1.2')
+#os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+#from google.appengine.dist import use_library
+#use_library('django', '1.2')
 
 from google.appengine.ext import db
 from google.appengine.api import taskqueue
 from google.appengine.api import users
-from google.appengine.ext import webapp
+#from google.appengine.ext import webapp
+import webapp2
 from google.appengine.ext.webapp import template
-from google.appengine.ext.webapp import util
+#from google.appengine.ext.webapp2 import util
 from google.appengine.runtime import DeadlineExceededError
 
 from ffstorage import *
 
 from fanficdownloader import adapters, writers, exceptions
 
-class UserConfigServer(webapp.RequestHandler):
+class UserConfigServer(webapp2.RequestHandler):
     def getUserConfig(self,user):
         config = ConfigParser.SafeConfigParser()
 
@@ -73,7 +74,7 @@ class UserConfigServer(webapp.RequestHandler):
 
         return config
 
-class MainHandler(webapp.RequestHandler):
+class MainHandler(webapp2.RequestHandler):
     def get(self):
         user = users.get_current_user()
         if user:
@@ -160,7 +161,7 @@ class EditConfigServer(UserConfigServer):
             self.response.out.write(template.render(path, template_values))
 
 
-class FileServer(webapp.RequestHandler):
+class FileServer(webapp2.RequestHandler):
 
     def get(self):
         fileId = self.request.get('id')
@@ -221,7 +222,7 @@ class FileServer(webapp.RequestHandler):
             path = os.path.join(os.path.dirname(__file__), 'status.html')
             self.response.out.write(template.render(path, template_values))
 
-class FileStatusServer(webapp.RequestHandler):
+class FileStatusServer(webapp2.RequestHandler):
     def get(self):
         user = users.get_current_user()
         if not user:
@@ -257,7 +258,7 @@ class FileStatusServer(webapp.RequestHandler):
         path = os.path.join(os.path.dirname(__file__), 'status.html')
         self.response.out.write(template.render(path, template_values))
 
-class ClearRecentServer(webapp.RequestHandler):
+class ClearRecentServer(webapp2.RequestHandler):
     def get(self):
         user = users.get_current_user()
         if not user:
@@ -282,7 +283,7 @@ class ClearRecentServer(webapp.RequestHandler):
         logging.info('Deleted %d instances download.' % num)
         self.redirect("/?error=recentcleared")
 
-class RecentFilesServer(webapp.RequestHandler):
+class RecentFilesServer(webapp2.RequestHandler):
     def get(self):
         user = users.get_current_user()
         if not user:
@@ -556,20 +557,14 @@ def urlEscape(data):
     p = re.compile(r'([^\w])')
     return p.sub(toPercentDecimal, data.encode("utf-8"))
 
-def main():
-  application = webapp.WSGIApplication([('/', MainHandler),
-                                        ('/fdowntask', FanfictionDownloaderTask),
-                                        ('/fdown', FanfictionDownloader),
-                                        (r'/file.*', FileServer),
-                                        ('/status', FileStatusServer),
-                                        ('/recent', RecentFilesServer),
-                                        ('/editconfig', EditConfigServer),
-                                        ('/clearrecent', ClearRecentServer),
-                                        ],
-                                       debug=False)
-  util.run_wsgi_app(application)
-
-
-if __name__ == '__main__':
-    logging.getLogger().setLevel(logging.DEBUG)
-    main()
+logging.getLogger().setLevel(logging.DEBUG)
+app = webapp2.WSGIApplication([('/', MainHandler),
+                               ('/fdowntask', FanfictionDownloaderTask),
+                               ('/fdown', FanfictionDownloader),
+                               (r'/file.*', FileServer),
+                               ('/status', FileStatusServer),
+                               ('/recent', RecentFilesServer),
+                               ('/editconfig', EditConfigServer),
+                               ('/clearrecent', ClearRecentServer),
+                               ],
+                              debug=False)

makeplugin.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# epubmerge.py 1.0
+
+# Copyright 2011, Jim Miller
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from glob import glob
+
+from makezip import createZipFile
+
+if __name__=="__main__":
+    filename="FanFictionDownLoader.zip"
+    exclude=['*.pyc','*~','*.xcf']
+    # from top dir. 'w' for overwrite
+    createZipFile(filename,"w",
+                  ['plugin-defaults.ini','example.ini','epubmerge.py','fanficdownloader'],
+                  exclude=exclude)
+    #from calibre-plugin dir. 'a' for append
+    os.chdir('calibre-plugin')
+    files=['about.txt','images',]
+    files.extend(glob('*.py'))
+    files.extend(glob('plugin-import-name-*.txt'))
+    createZipFile("../"+filename,"a",
+                  files,exclude=exclude)

makezip.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# epubmerge.py 1.0
+
+# Copyright 2011, Jim Miller
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, zipfile, sys
+from glob import glob
+
+def addFolderToZip(myZipFile,folder,exclude=[]):
+    folder = folder.encode('ascii') #convert path to ascii for ZipFile Method
+    excludelist=[]
+    for ex in exclude:
+        excludelist.extend(glob(folder+"/"+ex))
+    for file in glob(folder+"/*"):
+        if file in excludelist:
+            continue
+        if os.path.isfile(file):
+            #print file
+            myZipFile.write(file, file, zipfile.ZIP_DEFLATED)
+        elif os.path.isdir(file):
+            addFolderToZip(myZipFile,file,exclude=exclude)
+
+def createZipFile(filename,mode,files,exclude=[]):
+    myZipFile = zipfile.ZipFile( filename, mode ) # Open the zip file for writing
+    excludelist=[]
+    for ex in exclude:
+        excludelist.extend(glob(ex))
+    for file in files:
+        if file in excludelist:
+            continue
+        file = file.encode('ascii') #convert path to ascii for ZipFile Method
+        if os.path.isfile(file):
+            (filepath, filename) = os.path.split(file)
+            #print file
+            myZipFile.write( file, filename, zipfile.ZIP_DEFLATED )
+        if os.path.isdir(file):
+            addFolderToZip(myZipFile,file,exclude=exclude)
+    myZipFile.close()
+    return (1,filename)
+

plugin-defaults.ini

@@ -0,0 +1,242 @@
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[defaults]
+
+## [defaults] section applies to all formats and sites but may be
+## overridden at several levels
+
+## All available titlepage_entries and the label used for them:
+## <entryname>_label:<label>
+## Labels may be customized.
+title_label:Title
+storyUrl_label:Story URL
+description_label:Summary
+author_label:Author
+authorUrl_label:Author URL
+## epub, txt, html
+formatname_label:File Format
+## .epub, .txt, .html
+formatext_label:File Extension
+## Category and Genre have overlap, depending on the site.
+## Sometimes Harry Potter is a category and Fantasy a genre. (fanfiction.net)
+## Sometimes Fantasy is category *and* a genre (fictionpress.com)
+## Sometimes there are multiple categories and/or genres.
+category_label:Category
+genre_label:Genre
+characters_label:Characters
+## Completed/In-Progress
+status_label:Status
+## Dates story first published, last updated, and downloaded(last with time).
+datePublished_label:Published
+dateUpdated_label:Updated
+dateCreated_label:Packaged
+## Rating depends on the site.  Some use K,T,M,etc, and some PG,R,NC-17
+rating_label:Rating
+## Also depends on the site.  
+warnings_label:Warnings
+numChapters_label:Chapters
+numWords_label:Words
+## www.fanfiction.net, fictionalley.com, etc.
+site_label:Publisher
+## ffnet, fpcom, etc.
+siteabbrev_label:Site Abbrev
+## The site's unique story/author identifier.  Usually a number.
+storyId_label:Story ID
+authorId_label:Author ID
+## Primarily to put specific values in dc:subject tags for epub.  Will
+## show up in Calibre as tags.  Also carried into mobi when converted.
+extratags_label:Extra Tags
+## The version of fanficdownloader
+##
+version_label:FFD Version
+
+## items to include in the title page
+## Empty entries will *not* appear, even if in the list.
+## All current formats already include title and author.
+titlepage_entries: category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## include title page as first page.
+include_titlepage: true
+
+## include a TOC page before the story text
+include_tocpage: true
+
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+#website_encodings: auto, utf8, Windows-1252
+
+## entries to make epub subjects and calibre tags
+## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
+include_subject_tags: extratags, genre, category, characters, status
+
+## extra tags (comma separated) to include, primarily for epub.
+extratags: FanFiction
+
+## number of seconds to sleep between calls to the story site.  May by
+## useful if pulling large numbers of stories or if the site is slow.
+## Primarily for commandline.
+#slow_down_sleep_time:0.5
+
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Must be hex code, # will be added.
+background_color: ffffff
+
+## Each output format has a section that overrides [defaults]
+[html]
+
+[txt]
+## Add URLs since there aren't links.
+titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+
+## use \r\n for line endings, the windows convention.  text output only.
+windows_eol: true
+
+[epub]
+
+## epub is already a zip file.
+zip_output: false
+
+## epub carries the TOC in metadata.
+## mobi generated from epub will have a TOC at the end.
+include_tocpage: false
+
+## epub->mobi conversions typically don't like tables.
+titlepage_use_table: false
+
+## When using tables, make these span both columns.
+wide_titlepage_entries: description, storyUrl, author URL
+
+[mobi]
+## mobi TOC cannot be turned off right now.
+#include_tocpage: true
+
+
+## Each site has a section that overrides [defaults] *and* the format
+## sections test1.com specifically is not a real story site.  Instead,
+## it is a fake site for testing configuration and output.  It uses
+## URLs like: http://test1.com?sid=12345
+[test1.com]
+extratags: FanFiction,Testing
+
+## If necessary, you can define [<site>:<format>] sections to
+## customize the formats differently for the same site.  Overrides
+## defaults, format and site.
+[test1.com:txt]
+extratags: FanFiction,Testing,Text
+
+[test1.com:html]
+extratags: FanFiction,Testing,HTML
+
+[www.fanfiction.net]
+
+[www.fictionpress.com]
+## Clear FanFiction from defaults, fictionpress.com is original fiction.
+extratags:
+
+[www.ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.twilighted.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.twiwrite.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.whofic.com]
+
+[www.mediaminer.org]
+
+[www.thewriterscoffeeshop.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.fictionalley.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fictionally.org storyIds are not unique.  Combine with authorId.
+output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
+
+[www.harrypotterfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[overrides]
+## It may sometimes be useful to override all of the specific format,
+## site and site:format sections in your private configuration.  For
+## example, this extratags param here would override all of the
+## extratags params in all other sections.  Only commandline options
+## beat overrides.
+#extratags:fanficdownloader

recent.html

@@ -43,15 +43,6 @@
 	<div id='greeting'>
 	  <p>Hi, {{ nickname }}! These are the fanfics you've recently requested.</p>
 	  <p><a href="/clearrecent">Clear your Recent Downloads List</a></p>
-	  <h3>Please Switch to <a href="http://fanfictiondownloader.appspot.com/">New Version</a></h3>
-	  <p>
-	    We have a new, more efficient, version of the system up now.  Please start using the 
-	    <a href="http://fanfictiondownloader.appspot.com/">New
-	      Version here</a>.  If for some reason, the new version
-	      doesn't work for you, please let us know on
-	      the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
-	      Downloader Google Group</a>.
-	  </p>
 	</div>
       </div>
       

status.html

@@ -66,15 +66,6 @@
 	</div>
 	{% endif %}
 	  <p>See your personal list of <a href="/recent">previously downloaded fanfics</a>.</p>
-	  <h3>Please Switch to <a href="http://fanfictiondownloader.appspot.com/">New Version</a></h3>
-	  <p>
-	    We have a new, more efficient, version of the system up now.  Please start using the 
-	    <a href="http://fanfictiondownloader.appspot.com/">New
-	      Version here</a>.  If for some reason, the new version
-	      doesn't work for you, please let us know on
-	      the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
-	      Downloader Google Group</a>.
-	  </p>
       </div>
       <div style='text-align: center'>
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"

utils/remover.py

@@ -25,15 +25,16 @@ Copyright 2011 Fanficdownloader team
 import datetime
 import logging
 
-from google.appengine.ext.webapp import util
-from google.appengine.ext import webapp
+#from google.appengine.ext.webapp import util
+import webapp2
+#from google.appengine.ext import webapp
 from google.appengine.api import users
 from google.appengine.api import taskqueue
 from google.appengine.api import memcache
 
 from ffstorage import *
 
-class Remover(webapp.RequestHandler):
+class Remover(webapp2.RequestHandler):
     def get(self):
         logging.debug("Starting r3m0v3r")
         user = users.get_current_user()
@@ -56,9 +57,10 @@ class Remover(webapp.RequestHandler):
             logging.debug('Delete '+d.url)
 
         logging.info('Deleted instances: %d' % num)
+        self.response.headers['Content-Type'] = 'text/html'
         self.response.out.write('Deleted instances: %d<br>' % num)
 
-class RemoveOrphanDataChunks(webapp.RequestHandler):
+class RemoveOrphanDataChunks(webapp2.RequestHandler):
 
     def get(self):
         logging.debug("Starting RemoveOrphanDataChunks")
@@ -98,15 +100,10 @@ class RemoveOrphanDataChunks(webapp.RequestHandler):
             memcache.set('orphan_search_cursor',chunks.cursor())
         
         logging.info('Deleted %d orphan chunks from %d total.' % (deleted,num))
+        self.response.headers['Content-Type'] = 'text/html'
         self.response.out.write('Deleted %d orphan chunks from %d total.' % (deleted,num))
 
-def main():
-    application = webapp.WSGIApplication([('/r3m0v3r', Remover),
-                                          ('/r3m0v3rOrphans', RemoveOrphanDataChunks)],
-                                         debug=False)
-    util.run_wsgi_app(application)
-
-
-if __name__ == '__main__':
-    logging.getLogger().setLevel(logging.DEBUG)
-    main()
+logging.getLogger().setLevel(logging.DEBUG)
+app = webapp2.WSGIApplication([('/r3m0v3r', Remover),
+                               ('/r3m0v3rOrphans', RemoveOrphanDataChunks)],
+                              debug=False)

utils/tally.py

@@ -18,15 +18,16 @@
 import datetime
 import logging
 
-from google.appengine.ext.webapp import util
-from google.appengine.ext import webapp
+#from google.appengine.ext.webapp import util
+import webapp2
+#from google.appengine.ext import webapp
 from google.appengine.api import users
 from google.appengine.api import taskqueue
 from google.appengine.api import memcache
 
 from ffstorage import *
 
-class Tally(webapp.RequestHandler):
+class Tally(webapp2.RequestHandler):
     def get(self):
         logging.debug("Starting Tally")
         user = users.get_current_user()
@@ -57,13 +58,7 @@ class Tally(webapp.RequestHandler):
         logging.info('Tallied %d fics.' % num)
         self.response.out.write('<br/>Tallied %d fics.<br/>' % num)
 
-def main():
-    application = webapp.WSGIApplication([('/tally', Tally),
-                                          ],
-                                         debug=False)
-    util.run_wsgi_app(application)
-
-
-if __name__ == '__main__':
-    logging.getLogger().setLevel(logging.DEBUG)
-    main()
+logging.getLogger().setLevel(logging.DEBUG)
+app = webapp2.WSGIApplication([('/tally', Tally),
+                               ],
+                              debug=False)