Plugin 1.1.0--BG process (not thread) for reliability, true updates, menu.

app.yaml

@@ -1,5 +1,5 @@
 # ffd-retief-hrd fanfictiondownloader
-application: fanfictiondownloader
+application: ffd-retief-hrd
 version: 4-2-0
 runtime: python27
 api_version: 1

calibre-plugin/__init__.py

@@ -25,11 +25,9 @@ class InterfacePluginDemo(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 0, 4)
+    version             = (1, 1, 0)
     minimum_calibre_version = (0, 8, 30)
 
-#    action_menu_clone_qaction = True
-
     #: This field defines the GUI plugin class that contains all the code
     #: that actually does something. Its format is module_path:class_name
     #: The specified class must be defined in the specified module.

calibre-plugin/about.txt

@@ -1,11 +1,21 @@
 <p>FanFictionDownLoader Plugin</p>
 <hr />
 
-<p><a href="http://code.google.com/p/fanficdownloader/">http://code.google.com/p/fanficdownloader/</a></p>
+<p>Created by Jim Miller, borrowing heavily from Grant Drake's
+'<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
+'<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
+'<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
+plugins.</p>
 
-<p>Created by Jim Miller, borrowing heavily from
-Kovid Goyal's 'The InterfacePlugin Demo' and
-Grant Drake's 'Count Pages' plugins.</p>
+<p>
+Calibre officially distributes plugins from the mobileread.com forum site.
+The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>
+</p>
 
-<p>Requires calibre >= 0.8.30</p>
+<p> However, I monitor the
+<a href="http://groups.google.com/group/fanfic-downloader">general users
+group</a> for the downloader more closely.  That also covers the web application and CLI.
+</p>
 
+The source project for this plugin is <a href="http://code.google.com/p/fanficdownloader/source/checkout">
+also available</a>.

calibre-plugin/common_utils.py

@@ -0,0 +1,447 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import os
+from PyQt4 import QtGui
+from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime)
+from calibre.constants import iswindows
+from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME
+from calibre.gui2.actions import menu_action_unique_name
+from calibre.gui2.keyboard import ShortcutConfig
+from calibre.utils.config import config_dir
+from calibre.utils.date import now, format_date, qt_to_dt, UNDEFINED_DATE
+
+# Global definition of our plugin name. Used for common functions that require this.
+plugin_name = None
+# Global definition of our plugin resources. Used to share between the xxxAction and xxxBase
+# classes if you need any zip images to be displayed on the configuration dialog.
+plugin_icon_resources = {}
+
+
+def set_plugin_icon_resources(name, resources):
+    '''
+    Set our global store of plugin name and icon resources for sharing between
+    the InterfaceAction class which reads them and the ConfigWidget
+    if needed for use on the customization dialog for this plugin.
+    '''
+    global plugin_icon_resources, plugin_name
+    plugin_name = name
+    plugin_icon_resources = resources
+
+
+def get_icon(icon_name):
+    '''
+    Retrieve a QIcon for the named image from the zip file if it exists,
+    or if not then from Calibre's image cache.
+    '''
+    if icon_name:
+        pixmap = get_pixmap(icon_name)
+        if pixmap is None:
+            # Look in Calibre's cache for the icon
+            return QIcon(I(icon_name))
+        else:
+            return QIcon(pixmap)
+    return QIcon()
+
+
+def get_pixmap(icon_name):
+    '''
+    Retrieve a QPixmap for the named image
+    Any icons belonging to the plugin must be prefixed with 'images/'
+    '''
+    global plugin_icon_resources, plugin_name
+
+    if not icon_name.startswith('images/'):
+        # We know this is definitely not an icon belonging to this plugin
+        pixmap = QPixmap()
+        pixmap.load(I(icon_name))
+        return pixmap
+
+    # Check to see whether the icon exists as a Calibre resource
+    # This will enable skinning if the user stores icons within a folder like:
+    # ...\AppData\Roaming\calibre\resources\images\Plugin Name\
+    if plugin_name:
+        local_images_dir = get_local_images_dir(plugin_name)
+        local_image_path = os.path.join(local_images_dir, icon_name.replace('images/', ''))
+        if os.path.exists(local_image_path):
+            pixmap = QPixmap()
+            pixmap.load(local_image_path)
+            return pixmap
+
+    # As we did not find an icon elsewhere, look within our zip resources
+    if icon_name in plugin_icon_resources:
+        pixmap = QPixmap()
+        pixmap.loadFromData(plugin_icon_resources[icon_name])
+        return pixmap
+    return None
+
+
+def get_local_images_dir(subfolder=None):
+    '''
+    Returns a path to the user's local resources/images folder
+    If a subfolder name parameter is specified, appends this to the path
+    '''
+    images_dir = os.path.join(config_dir, 'resources/images')
+    if subfolder:
+        images_dir = os.path.join(images_dir, subfolder)
+    if iswindows:
+        images_dir = os.path.normpath(images_dir)
+    return images_dir
+
+
+def create_menu_item(ia, parent_menu, menu_text, image=None, tooltip=None,
+                     shortcut=(), triggered=None, is_checked=None):
+    '''
+    Create a menu action with the specified criteria and action
+    Note that if no shortcut is specified, will not appear in Preferences->Keyboard
+    This method should only be used for actions which either have no shortcuts,
+    or register their menus only once. Use create_menu_action_unique for all else.
+    '''
+    if shortcut is not None:
+        if len(shortcut) == 0:
+            shortcut = ()
+        else:
+            shortcut = _(shortcut)
+    ac = ia.create_action(spec=(menu_text, None, tooltip, shortcut),
+        attr=menu_text)
+    if image:
+        ac.setIcon(get_icon(image))
+    if triggered is not None:
+        ac.triggered.connect(triggered)
+    if is_checked is not None:
+        ac.setCheckable(True)
+        if is_checked:
+            ac.setChecked(True)
+
+    parent_menu.addAction(ac)
+    return ac
+
+
+def create_menu_action_unique(ia, parent_menu, menu_text, image=None, tooltip=None,
+                       shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
+                       unique_name=None):
+    '''
+    Create a menu action with the specified criteria and action, using the new
+    InterfaceAction.create_menu_action() function which ensures that regardless of
+    whether a shortcut is specified it will appear in Preferences->Keyboard
+    '''
+    orig_shortcut = shortcut
+    kb = ia.gui.keyboard
+    if unique_name is None:
+        unique_name = menu_text
+    if not shortcut == False:
+        full_unique_name = menu_action_unique_name(ia, unique_name)
+        if full_unique_name in kb.shortcuts:
+            shortcut = False
+        else:
+            if shortcut is not None and not shortcut == False:
+                if len(shortcut) == 0:
+                    shortcut = None
+                else:
+                    shortcut = _(shortcut)
+
+    if shortcut_name is None:
+        shortcut_name = menu_text.replace('&','')
+
+    ac = ia.create_menu_action(parent_menu, unique_name, menu_text, icon=None, shortcut=shortcut,
+        description=tooltip, triggered=triggered, shortcut_name=shortcut_name)
+    if shortcut == False and not orig_shortcut == False:
+        if ac.calibre_shortcut_unique_name in ia.gui.keyboard.shortcuts:
+            kb.replace_action(ac.calibre_shortcut_unique_name, ac)
+    if image:
+        ac.setIcon(get_icon(image))
+    if is_checked is not None:
+        ac.setCheckable(True)
+        if is_checked:
+            ac.setChecked(True)
+    return ac
+
+
+def swap_author_names(author):
+    if author.find(',') == -1:
+        return author
+    name_parts = author.strip().partition(',')
+    return name_parts[2].strip() + ' ' + name_parts[0]
+
+
+def get_library_uuid(db):
+    try:
+        library_uuid = db.library_id
+    except:
+        library_uuid = ''
+    return library_uuid
+
+
+class ImageLabel(QLabel):
+
+    def __init__(self, parent, icon_name, size=16):
+        QLabel.__init__(self, parent)
+        pixmap = get_pixmap(icon_name)
+        self.setPixmap(pixmap)
+        self.setMaximumSize(size, size)
+        self.setScaledContents(True)
+
+
+class ImageTitleLayout(QHBoxLayout):
+    '''
+    A reusable layout widget displaying an image followed by a title
+    '''
+    def __init__(self, parent, icon_name, title):
+        QHBoxLayout.__init__(self)
+        title_image_label = QLabel(parent)
+        pixmap = get_pixmap(icon_name)
+        if pixmap is None:
+            pixmap = get_pixmap('library.png')
+            # error_dialog(parent, _('Restart required'),
+            #              _('You must restart Calibre before using this plugin!'), show=True)
+        else:
+            title_image_label.setPixmap(pixmap)
+        title_image_label.setMaximumSize(32, 32)
+        title_image_label.setScaledContents(True)
+        self.addWidget(title_image_label)
+
+        title_font = QFont()
+        title_font.setPointSize(16)
+        shelf_label = QLabel(title, parent)
+        shelf_label.setFont(title_font)
+        self.addWidget(shelf_label)
+        self.insertStretch(-1)
+
+
+class SizePersistedDialog(QDialog):
+    '''
+    This dialog is a base class for any dialogs that want their size/position
+    restored when they are next opened.
+    '''
+    def __init__(self, parent, unique_pref_name):
+        QDialog.__init__(self, parent)
+        self.unique_pref_name = unique_pref_name
+        self.geom = gprefs.get(unique_pref_name, None)
+        self.finished.connect(self.dialog_closing)
+
+    def resize_dialog(self):
+        if self.geom is None:
+            self.resize(self.sizeHint())
+        else:
+            self.restoreGeometry(self.geom)
+
+    def dialog_closing(self, result):
+        geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = geom
+
+
+class ReadOnlyTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, text):
+        if text is None:
+            text = ''
+        QTableWidgetItem.__init__(self, text, QtGui.QTableWidgetItem.UserType)
+        self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+
+
+class RatingTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, rating, is_read_only=False):
+        QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType)
+        self.setData(Qt.DisplayRole, rating)
+        if is_read_only:
+            self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+
+
+class DateTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, date_read, is_read_only=False, default_to_today=False):
+        if date_read == UNDEFINED_DATE and default_to_today:
+            date_read = now()
+        if is_read_only:
+            QTableWidgetItem.__init__(self, format_date(date_read, None), QtGui.QTableWidgetItem.UserType)
+            self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+        else:
+            QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType)
+            self.setData(Qt.DisplayRole, QDateTime(date_read))
+
+
+class NoWheelComboBox(QComboBox):
+
+    def wheelEvent (self, event):
+        # Disable the mouse wheel on top of the combo box changing selection as plays havoc in a grid
+        event.ignore()
+
+
+class CheckableTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, checked=False, is_tristate=False):
+        QTableWidgetItem.__init__(self, '')
+        self.setFlags(Qt.ItemFlags(Qt.ItemIsSelectable | Qt.ItemIsUserCheckable | Qt.ItemIsEnabled ))
+        if is_tristate:
+            self.setFlags(self.flags() | Qt.ItemIsTristate)
+        if checked:
+            self.setCheckState(Qt.Checked)
+        else:
+            if is_tristate and checked is None:
+                self.setCheckState(Qt.PartiallyChecked)
+            else:
+                self.setCheckState(Qt.Unchecked)
+
+    def get_boolean_value(self):
+        '''
+        Return a boolean value indicating whether checkbox is checked
+        If this is a tristate checkbox, a partially checked value is returned as None
+        '''
+        if self.checkState() == Qt.PartiallyChecked:
+            return None
+        else:
+            return self.checkState() == Qt.Checked
+
+
+class TextIconWidgetItem(QTableWidgetItem):
+
+    def __init__(self, text, icon):
+        QTableWidgetItem.__init__(self, text)
+        if icon:
+            self.setIcon(icon)
+
+
+class ReadOnlyTextIconWidgetItem(ReadOnlyTableWidgetItem):
+
+    def __init__(self, text, icon):
+        ReadOnlyTableWidgetItem.__init__(self, text)
+        if icon:
+            self.setIcon(icon)
+
+
+class ReadOnlyLineEdit(QLineEdit):
+
+    def __init__(self, text, parent):
+        if text is None:
+            text = ''
+        QLineEdit.__init__(self, text, parent)
+        self.setEnabled(False)
+
+
+class KeyValueComboBox(QComboBox):
+
+    def __init__(self, parent, values, selected_key):
+        QComboBox.__init__(self, parent)
+        self.values = values
+        self.populate_combo(selected_key)
+
+    def populate_combo(self, selected_key):
+        self.clear()
+        selected_idx = idx = -1
+        for key, value in self.values.iteritems():
+            idx = idx + 1
+            self.addItem(value)
+            if key == selected_key:
+                selected_idx = idx
+        self.setCurrentIndex(selected_idx)
+
+    def selected_key(self):
+        for key, value in self.values.iteritems():
+            if value == unicode(self.currentText()).strip():
+                return key
+
+
+class CustomColumnComboBox(QComboBox):
+
+    def __init__(self, parent, custom_columns, selected_column, initial_items=['']):
+        QComboBox.__init__(self, parent)
+        self.populate_combo(custom_columns, selected_column, initial_items)
+
+    def populate_combo(self, custom_columns, selected_column, initial_items=['']):
+        self.clear()
+        self.column_names = initial_items
+        if len(initial_items) > 0:
+            self.addItems(initial_items)
+        selected_idx = 0
+        for idx, value in enumerate(initial_items):
+            if value == selected_column:
+                selected_idx = idx
+        for key in sorted(custom_columns.keys()):
+            self.column_names.append(key)
+            self.addItem('%s (%s)'%(key, custom_columns[key]['name']))
+            if key == selected_column:
+                selected_idx = len(self.column_names) - 1
+        self.setCurrentIndex(selected_idx)
+
+    def get_selected_column(self):
+        return self.column_names[self.currentIndex()]
+
+
+class KeyboardConfigDialog(SizePersistedDialog):
+    '''
+    This dialog is used to allow editing of keyboard shortcuts.
+    '''
+    def __init__(self, gui, group_name):
+        SizePersistedDialog.__init__(self, gui, 'Keyboard shortcut dialog')
+        self.gui = gui
+        self.setWindowTitle('Keyboard shortcuts')
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+
+        self.keyboard_widget = ShortcutConfig(self)
+        layout.addWidget(self.keyboard_widget)
+        self.group_name = group_name
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.commit)
+        button_box.rejected.connect(self.reject)
+        layout.addWidget(button_box)
+
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.initialize()
+
+    def initialize(self):
+        self.keyboard_widget.initialize(self.gui.keyboard)
+        self.keyboard_widget.highlight_group(self.group_name)
+
+    def commit(self):
+        self.keyboard_widget.commit()
+        self.accept()
+
+
+class DateDelegate(QStyledItemDelegate):
+    '''
+    Delegate for dates. Because this delegate stores the
+    format as an instance variable, a new instance must be created for each
+    column. This differs from all the other delegates.
+    '''
+    def __init__(self, parent):
+        QStyledItemDelegate.__init__(self, parent)
+        self.format = 'dd MMM yyyy'
+
+    def displayText(self, val, locale):
+        d = val.toDateTime()
+        if d <= UNDEFINED_QDATETIME:
+            return ''
+        return format_date(qt_to_dt(d, as_utc=False), self.format)
+
+    def createEditor(self, parent, option, index):
+        qde = QStyledItemDelegate.createEditor(self, parent, option, index)
+        qde.setDisplayFormat(self.format)
+        qde.setMinimumDateTime(UNDEFINED_QDATETIME)
+        qde.setSpecialValueText(_('Undefined'))
+        qde.setCalendarPopup(True)
+        return qde
+
+    def setEditorData(self, editor, index):
+        val = index.model().data(index, Qt.DisplayRole).toDateTime()
+        if val is None or val == UNDEFINED_QDATETIME:
+            val = now()
+        editor.setDateTime(val)
+
+    def setModelData(self, editor, model, index):
+        val = editor.dateTime()
+        if val <= UNDEFINED_QDATETIME:
+            model.setData(index, UNDEFINED_QDATETIME, Qt.EditRole)
+        else:
+            model.setData(index, QDateTime(val), Qt.EditRole)

calibre-plugin/config.py

@@ -10,9 +10,12 @@ __docformat__ = 'restructuredtext en'
 from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit,
                       QTextEdit, QComboBox, QCheckBox, QPushButton)
 
+from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
 
-from calibre_plugins.fanfictiondownloader_plugin.dialogs import (OVERWRITE, ADDNEW, SKIP,CALIBREONLY,UPDATE)
+from calibre_plugins.fanfictiondownloader_plugin.dialogs \
+    import (SKIP, ADDNEW, UPDATE, UPDATEALWAYS, OVERWRITE, OVERWRITEALWAYS,
+             CALIBREONLY,collision_order)
 
 # This is where all preferences for this plugin will be stored
 # Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
@@ -21,17 +24,12 @@ from calibre_plugins.fanfictiondownloader_plugin.dialogs import (OVERWRITE, ADDN
 # so as to ensure you dont accidentally clobber a calibre config file
 prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
 
-# urlsfrompriority values
-CLIP = 'Clipboard'
-SELECTED = 'Selected Stories'
-
 # Set defaults
 prefs.defaults['personal.ini'] = get_resources('example.ini')
 prefs.defaults['updatemeta'] = True
-prefs.defaults['onlyoverwriteifnewer'] = False
+#prefs.defaults['onlyoverwriteifnewer'] = False
 prefs.defaults['urlsfromclip'] = True
-prefs.defaults['urlsfromselected'] = True
-prefs.defaults['urlsfrompriority'] = SELECTED
+prefs.defaults['updatedefault'] = True
 prefs.defaults['fileform'] = 'epub'
 prefs.defaults['collision'] = OVERWRITE
 prefs.defaults['deleteotherforms'] = False
@@ -62,13 +60,10 @@ class ConfigWidget(QWidget):
         label.setToolTip("What to do if there's already an existing story with the same title and author.")
         horz.addWidget(label)
         self.collision = QComboBox(self)
-        self.collision.addItem(OVERWRITE)
-        self.collision.addItem(UPDATE)
-        self.collision.addItem(ADDNEW)
-        self.collision.addItem(SKIP)
-        self.collision.addItem(CALIBREONLY)
+        for o in collision_order:
+            self.collision.addItem(o)
         self.collision.setCurrentIndex(self.collision.findText(prefs['collision']))
-        self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
+        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
         label.setBuddy(self.collision)
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
@@ -78,33 +73,21 @@ class ConfigWidget(QWidget):
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
-        self.onlyoverwriteifnewer = QCheckBox('Default Only Overwrite Story if Newer',self)
-        self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer or from the same day.")
-        self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer'])
-        self.l.addWidget(self.onlyoverwriteifnewer)
+        # self.onlyoverwriteifnewer = QCheckBox('Default Only Overwrite Story if Newer',self)
+        # self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer or from the same day.")
+        # self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer'])
+        # self.l.addWidget(self.onlyoverwriteifnewer)
         
         self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
-        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when starting plugin?')
+        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New?')
         self.urlsfromclip.setChecked(prefs['urlsfromclip'])
         self.l.addWidget(self.urlsfromclip)
 
-        self.urlsfromselected = QCheckBox('Take URLs from Selected Stories?',self)
-        self.urlsfromselected.setToolTip('Prefill URLs from valid URLs in selected stories when starting plugin?')
-        self.urlsfromselected.setChecked(prefs['urlsfromselected'])
-        self.l.addWidget(self.urlsfromselected)
-
-        horz = QHBoxLayout()
-        label = QLabel('Take URLs from which first:')
-        label.setToolTip("If both clipbaord and selected enabled and populated, which is used?")
-        horz.addWidget(label)
-        self.urlsfrompriority = QComboBox(self)
-        self.urlsfrompriority.addItem(SELECTED)
-        self.urlsfrompriority.addItem(CLIP)
-        self.urlsfrompriority.setCurrentIndex(self.urlsfrompriority.findText(prefs['urlsfrompriority']))
-        self.urlsfrompriority.setToolTip('If both clipbaord and selected enabled and populated, which is used?')
-        label.setBuddy(self.urlsfrompriority)
-        horz.addWidget(self.urlsfrompriority)
-        self.l.addLayout(horz)
+        self.updatedefault = QCheckBox('Default to Update when books selected?',self)
+        self.updatedefault.setToolTip('The top FanFictionDownLoader plugin button will start Update if\n'+
+                                      'books are selected.  If unchecked, it will always bring up \'Add New\'.')
+        self.updatedefault.setChecked(prefs['updatedefault'])
+        self.l.addWidget(self.updatedefault)
 
         self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
         self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
@@ -124,14 +107,19 @@ class ConfigWidget(QWidget):
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
         
+        reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self)
+        reset_confirmation_button.setToolTip(_(
+                    'Reset all show me again dialogs for the FanFictionDownLoader plugin'))
+        reset_confirmation_button.clicked.connect(self.reset_dialogs)
+        self.l.addWidget(reset_confirmation_button)
+        
     def save_settings(self):
         prefs['fileform'] = unicode(self.fileform.currentText())
         prefs['collision'] = unicode(self.collision.currentText())
         prefs['updatemeta'] = self.updatemeta.isChecked()
-        prefs['urlsfrompriority'] = unicode(self.urlsfrompriority.currentText())
         prefs['urlsfromclip'] = self.urlsfromclip.isChecked()
-        prefs['urlsfromselected'] = self.urlsfromselected.isChecked()
-        prefs['onlyoverwriteifnewer'] = self.onlyoverwriteifnewer.isChecked()
+        prefs['updatedefault'] = self.updatedefault.isChecked()
+#        prefs['onlyoverwriteifnewer'] = self.onlyoverwriteifnewer.isChecked()
         prefs['deleteotherforms'] = self.deleteotherforms.isChecked()
         
         ini = unicode(self.ini.toPlainText())
@@ -145,7 +133,16 @@ class ConfigWidget(QWidget):
     def show_defaults(self):
         text = get_resources('defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
-            
+
+    def reset_dialogs(self):
+        for key in dynamic.keys():
+            if key.startswith('fanfictiondownloader_') and key.endswith('_again') \
+                                                  and dynamic[key] is False:
+                dynamic[key] = True
+        info_dialog(self, _('Done'),
+                _('Confirmation dialogs have all been reset'), show=True)
+
+        
 class ShowDefaultsIniDialog(QDialog):
 
     def __init__(self, icon, text, parent=None):

calibre-plugin/dialogs.py

@@ -9,28 +9,48 @@ __docformat__ = 'restructuredtext en'
 
 import traceback
 
-from PyQt4.Qt import (QDialog, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, 
-                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, 
-                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt )
+from PyQt4 import QtGui
+from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
+                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon,
+                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
+                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt,QAbstractItemView )
 
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
+from calibre.gui2.dialogs.confirm_delete import confirm
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+from calibre_plugins.fanfictiondownloader_plugin.common_utils \
+    import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
+            ImageTitleLayout, get_icon)
 
-OVERWRITE='Overwrite'
-UPDATE='Update EPUB'
-ADDNEW='Add New'
 SKIP='Skip'
+ADDNEW='Add New Book'
+UPDATE='Update EPUB if New Chapters'
+UPDATEALWAYS='Update EPUB Always'
+OVERWRITE='Overwrite if Newer'
+OVERWRITEALWAYS='Overwrite Always'
 CALIBREONLY='Update Calibre Metadata Only'
+collision_order=[SKIP,
+                 ADDNEW,
+                 UPDATE,
+                 UPDATEALWAYS,
+                 OVERWRITE,
+                 OVERWRITEALWAYS,
+                 CALIBREONLY,]
+        
+class NotGoingToDownload(Exception):
+    def __init__(self,error,icon='dialog_error.png'):
+        self.error=error
+        self.icon=icon
 
-class DownloadDialog(QDialog):
+    def __str__(self):
+        return self.error
+    
+class AddNewDialog(SizePersistedDialog):
 
-    def __init__(self, gui, prefs, icon, url_list_text, do_user_config, start_downloads):
-        QDialog.__init__(self, gui)
+    def __init__(self, gui, prefs, icon, url_list_text):
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
         self.gui = gui
-        self.do_user_config = do_user_config
-        self.start_downloads = start_downloads
 
         self.setMinimumWidth(300)
         self.l = QVBoxLayout()
@@ -41,19 +61,19 @@ class DownloadDialog(QDialog):
 
         self.l.addWidget(QLabel('Story URL(s), one per line:'))
         self.url = QTextEdit(self)
-        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from selected books or clipboard, but only valid URLs.')
+        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.')
         self.url.setLineWrapMode(QTextEdit.NoWrap)
         self.url.setText(url_list_text)
         self.l.addWidget(self.url)
         
-        self.ffdl_button = QPushButton(
-            'Download Stories', self)
-        self.ffdl_button.setToolTip('Start download(s).')
-        self.ffdl_button.clicked.connect(self.ffdl)
-        # if there's already URL(s), focus 'go' button
-        if url_list_text:
-            self.ffdl_button.setFocus()
-        self.l.addWidget(self.ffdl_button)
+        # self.ffdl_button = QPushButton(
+        #     'Download Stories', self)
+        # self.ffdl_button.setToolTip('Start download(s).')
+        # self.ffdl_button.clicked.connect(self.ffdl)
+        # # if there's already URL(s), focus 'go' button
+        # if url_list_text:
+        #     self.ffdl_button.setFocus()
+        # self.l.addWidget(self.ffdl_button)
 
         horz = QHBoxLayout()
         label = QLabel('Output &Format:')
@@ -74,17 +94,16 @@ class DownloadDialog(QDialog):
         label.setToolTip("What to do if there's already an existing story with the same title and author.")
         horz.addWidget(label)
         self.collision = QComboBox(self)
-        self.collision.addItem(OVERWRITE)
-        self.collision.addItem(UPDATE)
-        self.collision.addItem(ADDNEW)
-        self.collision.addItem(SKIP)
-        self.collision.addItem(CALIBREONLY)
+        # add collision options
+        for o in collision_order:
+            self.collision.addItem(o)
+
         self.collision.setCurrentIndex(self.collision.findText(prefs['collision']))
-        self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+
-                                  UPDATE+' will download new chapters only and add to existing EPUB.\n'+
-                                  ADDNEW+' will create a new story with the same title and author.\n'+
-                                  SKIP+' will not download existing stories.\n'+
-                                  CALIBREONLY+' will not download stories, but will update Calibre metadata.')
+        # self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+
+        #                           UPDATE+' will download new chapters only and add to existing EPUB.\n'+
+        #                           ADDNEW+' will create a new story with the same title and author.\n'+
+        #                           SKIP+' will not download existing stories.\n'+
+        #                           CALIBREONLY+' will not download stories, but will update Calibre metadata.')
         label.setBuddy(self.collision)
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
@@ -94,50 +113,42 @@ class DownloadDialog(QDialog):
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
-        self.onlyoverwriteifnewer = QCheckBox('Only Overwrite Story if Newer',self)
-        self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer.\n"+
-                                             "From the same day counts as 'newer' because the sites don't give update time.")
-        self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer'])
-        self.l.addWidget(self.onlyoverwriteifnewer)
+        # self.onlyoverwriteifnewer = QCheckBox('Only Overwrite Story if Newer',self)
+        # self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer.\n"+
+        #                                      "From the same day counts as 'newer' because the sites don't give update time.")
+        # self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer'])
+        # self.l.addWidget(self.onlyoverwriteifnewer)
         
-        horz = QHBoxLayout()
-        self.about_button = QPushButton('About', self)
-        self.about_button.clicked.connect(self.about)
-        horz.addWidget(self.about_button)
-        self.conf_button = QPushButton(
-                'Configure this plugin', self)
-        self.conf_button.clicked.connect(self.config)
-        horz.addWidget(self.conf_button)
-        self.l.addLayout(horz)
+        # horz = QHBoxLayout()
+        # self.about_button = QPushButton('About', self)
+        # self.about_button.clicked.connect(self.about)
+        # horz.addWidget(self.about_button)
+        # self.conf_button = QPushButton(
+        #         'Configure this plugin', self)
+        # self.conf_button.clicked.connect(self.config)
+        # horz.addWidget(self.conf_button)
+        # self.l.addLayout(horz)
 
-        self.resize(self.sizeHint())
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
+        
+        if url_list_text:
+            button_box.button(QDialogButtonBox.Ok).setFocus()
+        
+        self.resize_dialog()
+        #self.resize(self.sizeHint())
 
-    def about(self):
-        # Get the about text from a file inside the plugin zip file
-        # The get_resources function is a builtin function defined for all your
-        # plugin code. It loads files from the plugin zip file. It returns
-        # the bytes from the specified file.
-        #
-        # Note that if you are loading more than one file, for performance, you
-        # should pass a list of names to get_resources. In this case,
-        # get_resources will return a dictionary mapping names to bytes. Names that
-        # are not found in the zip file will not be in the returned dictionary.
-        text = get_resources('about.txt')
-#        QMessageBox.about(self, 'About the FanFictionDownLoader Plugin',
- #               text.decode('utf-8'))
-        AboutDialog(self.windowIcon(),text,self).exec_()
-
-    def ffdl(self):
-        self.start_downloads(unicode(self.url.toPlainText()),
-                             unicode(self.fileform.currentText()),
-                             unicode(self.collision.currentText()),
-                             self.updatemeta.isChecked(),
-                             self.onlyoverwriteifnewer.isChecked())
-        self.hide()
-
-    def config(self):
-        self.do_user_config(parent=self)
+    def get_ffdl_options(self):
+        return {
+            'fileform': unicode(self.fileform.currentText()),
+            'collision': unicode(self.collision.currentText()),
+            'updatemeta': unicode(self.updatemeta.isChecked()),
+            }
 
+    def get_urlstext(self):
+        return unicode(self.url.toPlainText())
         
 class UserPassDialog(QDialog):
     '''
@@ -184,19 +195,22 @@ class MetadataProgressDialog(QProgressDialog):
     '''
     ProgressDialog displayed while fetching metadata for each story.
     '''
-    def __init__(self, gui, loop_list, fileform, getadapter_function, download_list_function, db):
+    def __init__(self, gui,
+                 book_list,
+                 options,
+                 metadata_function,
+                 startdownload_function):
         QProgressDialog.__init__(self,
                                  "Fetching metadata for stories...",
-                                 QString(), 0, len(loop_list), gui)
+                                 QString(), 0, len(book_list), gui)
         self.setWindowTitle("Downloading metadata for stories")
         self.setMinimumWidth(500)
         self.gui = gui
-        self.db = db
-        self.loop_list = loop_list
-        self.fileform = fileform
-        self.getadapter_function = getadapter_function
-        self.download_list_function = download_list_function
-        self.i, self.loop_bad, self.loop_good = 0, [], []
+        self.book_list = book_list
+        self.options = options
+        self.metadata_function = metadata_function
+        self.startdownload_function = startdownload_function
+        self.i = 0
         
         ## self.do_loop does QTimer.singleShot on self.do_loop also.
         ## A weird way to do a loop, but that was the example I had.
@@ -204,7 +218,7 @@ class MetadataProgressDialog(QProgressDialog):
         self.exec_()
 
     def updateStatus(self):
-        self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.loop_list)))
+        self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.book_list)))
         self.setValue(self.i+1)
         print(self.labelText())
 
@@ -214,49 +228,41 @@ class MetadataProgressDialog(QProgressDialog):
         if self.i == 0:
             self.setValue(0)
 
-        if self.i >= len(self.loop_list) or self.wasCanceled():
-            return self.do_when_finished()
-
-        else:
-            current = self.loop_list[self.i]
-            try:
-                ## collision spec passed into getadapter by partial from ffdl_plugin
-                ## no retval only if it exists, but collision is SKIP
-                retval = self.getadapter_function(current,self.fileform)
-                self.loop_good.append((current,retval))
-            except Exception as e:
-                print("%s:%s"%(current,e))
-                self.loop_bad.append((current,e))
-                traceback.print_exc()
+        book = self.book_list[self.i]
+        try:
+            ## collision spec passed into getadapter by partial from ffdl_plugin
+            ## no retval only if it exists, but collision is SKIP
+            self.metadata_function(book)
             
-            self.updateStatus()
-            self.i += 1
+        except NotGoingToDownload as d:
+            book['good']=False
+            book['comment']=unicode(d)
+            book['icon'] = d.icon
+
+        except Exception as e:
+            book['good']=False
+            book['comment']=unicode(e)
+            print("%s:%s"%(book,unicode(e)))
+            # XXX trace for not-expected exceptions
+            #traceback.print_exc()
+            
+        self.updateStatus()
+        self.i += 1
+            
+        if self.i >= len(self.book_list) or self.wasCanceled():
+            return self.do_when_finished()
+        else:
             QTimer.singleShot(0, self.do_loop)
 
     def do_when_finished(self):
         self.hide()
-        
-        # Queues a job to process these ePub/Mobi books in the background.
-        self.download_list_function(self.loop_good,self.fileform)
-        
-        if self.loop_bad != []:
-            res = []
-            for j in self.loop_bad:
-                res.append('%s : %s'%j)
-            msg = '%s' % '\n'.join(res)
-            warning_dialog(self.gui, _('Not going to download some stories'),
-                _('Not going to download %d of %d stories.') %
-                (len(self.loop_bad), len(self.loop_list)),
-                msg).exec_()
-        # else:
-        #     info_dialog(self.gui, "Starting Downloads",
-        #                 "Got metadata and started download for %d stories."%len(self.loop_good),
-        #                 show_copy_button=False).exec_()
-        self.gui = None
+        self.gui = None        
+        # Queues a job to process these books in the background.
+        self.startdownload_function(self.book_list)
 
 class AboutDialog(QDialog):
 
-    def __init__(self, icon, text, parent=None):
+    def __init__(self, parent, icon, text):
         QDialog.__init__(self, parent)
         self.resize(400, 250)
         self.l = QGridLayout()
@@ -267,7 +273,7 @@ class AboutDialog(QDialog):
         self.label = QLabel(text)
         self.label.setOpenExternalLinks(True)
         self.label.setWordWrap(True)
-        self.setWindowTitle(_('Update available!'))
+        self.setWindowTitle(_('About FanFictionDownLoader'))
         self.setWindowIcon(icon)
         self.l.addWidget(self.logo, 0, 0)
         self.l.addWidget(self.label, 0, 1)
@@ -277,10 +283,321 @@ class AboutDialog(QDialog):
         self.l.addWidget(self.bb, 2, 0, 1, -1)
         self.bb.accepted.connect(self.accept)
 
-        
-class NotGoingToDownload(Exception):
-    def __init__(self,error):
-        self.error=error
+class IconWidgetItem(ReadOnlyTextIconWidgetItem):
+    def __init__(self, text, icon, sort_key):
+        ReadOnlyTextIconWidgetItem.__init__(self, text, icon)
+        self.sort_key = sort_key
 
-    def __str__(self):
-        return self.error
+    #Qt uses a simple < check for sorting items, override this to use the sortKey
+    def __lt__(self, other):
+        return self.sort_key < other.sort_key
+
+class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):
+    def __init__(self, text, sort_key):
+        ReadOnlyTableWidgetItem.__init__(self, text)
+        self.sort_key = sort_key
+
+    #Qt uses a simple < check for sorting items, override this to use the sortKey
+    def __lt__(self, other):
+        return self.sort_key < other.sort_key
+
+class UpdateExistingDialog(SizePersistedDialog):
+    def __init__(self, gui, header, prefs, icon, books,
+                 save_size_name='FanFictionDownLoader plugin:update list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        self.gui = gui
+        
+        self.setWindowTitle(header)
+        self.setWindowIcon(icon)
+        
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, 'images/icon.png',
+                                        header)
+        layout.addLayout(title_layout)
+        books_layout = QHBoxLayout()
+        layout.addLayout(books_layout)
+
+        self.books_table = StoryListTableWidget(self)
+        books_layout.addWidget(self.books_table)
+
+        button_layout = QVBoxLayout()
+        books_layout.addLayout(button_layout)
+        # self.move_up_button = QtGui.QToolButton(self)
+        # self.move_up_button.setToolTip('Move selected books up the list')
+        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
+        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
+        # button_layout.addWidget(self.move_up_button)
+        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem)
+        self.remove_button = QtGui.QToolButton(self)
+        self.remove_button.setToolTip('Remove selected books from the list')
+        self.remove_button.setIcon(get_icon('list_remove.png'))
+        self.remove_button.clicked.connect(self.remove_from_list)
+        button_layout.addWidget(self.remove_button)
+        spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem1)
+        # self.move_down_button = QtGui.QToolButton(self)
+        # self.move_down_button.setToolTip('Move selected books down the list')
+        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
+        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
+        # button_layout.addWidget(self.move_down_button)
+
+        options_layout = QHBoxLayout()
+
+        label = QLabel('Output &Format:')
+        options_layout.addWidget(label)
+        self.fileform = QComboBox(self)
+        self.fileform.addItem('epub')
+        self.fileform.addItem('mobi')
+        self.fileform.addItem('html')
+        self.fileform.addItem('txt')
+        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        label.setBuddy(self.fileform)
+        options_layout.addWidget(self.fileform)
+        
+        label = QLabel('Update Mode:')
+        label.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
+        options_layout.addWidget(label)
+        self.collision = QComboBox(self)
+        for o in collision_order:
+            if o not in [ADDNEW,SKIP]:
+                self.collision.addItem(o)
+        self.collision.setCurrentIndex(self.collision.findText(prefs['collision']))
+        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
+        label.setBuddy(self.collision)
+        options_layout.addWidget(self.collision)
+
+        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
+        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?  May set default from plugin configuration.')
+        self.updatemeta.setChecked(prefs['updatemeta'])
+        options_layout.addWidget(self.updatemeta)
+                
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        options_layout.addWidget(button_box)
+        
+        layout.addLayout(options_layout)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.books_table.populate_table(books)
+        
+    def remove_from_list(self):
+        self.books_table.remove_selected_rows()
+
+    def get_books(self):
+        return self.books_table.get_books()
+
+    def get_ffdl_options(self):
+        return {
+            'fileform': unicode(self.fileform.currentText()),
+            'collision': unicode(self.collision.currentText()),
+            'updatemeta': unicode(self.updatemeta.isChecked()),
+            }
+
+class DisplayStoryListDialog(SizePersistedDialog):
+    def __init__(self, gui, header, prefs, icon, books,
+                 label_text='',
+                 save_size_name='FanFictionDownLoader plugin:display list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        # UpdateExistingDialog.__init__(self, gui, header, prefs, icon, books,
+        #                               save_size_name='FanFictionDownLoader plugin:display list dialog')
+
+        self.gui = gui
+        
+        self.setWindowTitle(header)
+        self.setWindowIcon(icon)
+        
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, 'images/icon.png',
+                                        header)
+        layout.addLayout(title_layout)
+
+        self.books_table = StoryListTableWidget(self)
+        layout.addWidget(self.books_table)
+
+        options_layout = QHBoxLayout()
+        self.label = QLabel(label_text)
+        #self.label.setOpenExternalLinks(True)
+        #self.label.setWordWrap(True)
+        options_layout.addWidget(self.label)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+
+        options_layout.addWidget(button_box)
+        
+        layout.addLayout(options_layout)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.books_table.populate_table(books)
+        
+    def get_books(self):
+        return self.books_table.get_books()
+
+class StoryListTableWidget(QTableWidget):
+
+    def __init__(self, parent):
+        QTableWidget.__init__(self, parent)
+        self.setSelectionBehavior(QAbstractItemView.SelectRows)
+
+    def populate_table(self, books):
+        self.clear()
+        self.setAlternatingRowColors(True)
+        self.setRowCount(len(books))
+        header_labels = ['','Title', 'Author', 'URL', 'Comment']
+        self.setColumnCount(len(header_labels))
+        self.setHorizontalHeaderLabels(header_labels)
+        self.horizontalHeader().setStretchLastSection(True)
+        #self.verticalHeader().setDefaultSectionSize(24)
+        self.verticalHeader().hide()
+
+        self.books={}
+        for row, book in enumerate(books):
+            self.populate_table_row(row, book)
+            self.books[row] = book
+
+        # turning True breaks up/down.  Do we need either sorting or up/down?
+        self.setSortingEnabled(True)
+        self.resizeColumnsToContents()
+        self.setMinimumColumnWidth(1, 100)
+        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumColumnWidth(3, 100)
+        self.setMinimumSize(300, 0)
+        # if len(books) > 0:
+        #     self.selectRow(0)
+        self.sortItems(1)
+        self.sortItems(0)
+
+    def setMinimumColumnWidth(self, col, minimum):
+        if self.columnWidth(col) < minimum:
+            self.setColumnWidth(col, minimum)
+
+    def populate_table_row(self, row, book):
+        if book['good']:
+            icon = get_icon('ok.png')
+            val = 0
+        else:
+            icon = get_icon('minus.png')
+            val = 1
+        if 'icon' in book:
+            icon = get_icon(book['icon'])
+
+        status_cell = IconWidgetItem(None,icon,val)
+        status_cell.setData(Qt.UserRole, QVariant(val))
+        self.setItem(row, 0, status_cell)
+        
+        title_cell = ReadOnlyTableWidgetItem(book['title'])
+        title_cell.setData(Qt.UserRole, QVariant(row))
+        self.setItem(row, 1, title_cell)
+        
+        self.setItem(row, 2, AuthorTableWidgetItem(book['author'], book['author_sort']))
+        
+        url_cell = ReadOnlyTableWidgetItem(book['url'])
+        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
+        self.setItem(row, 3, url_cell)
+        
+        comment_cell = ReadOnlyTableWidgetItem(book['comment'])
+        #comment_cell.setData(Qt.UserRole, QVariant(book))
+        self.setItem(row, 4, comment_cell)
+
+    # def get_calibre_ids(self):
+    #     ids = []
+    #     for row in range(self.rowCount()):
+    #         ids.append(self.item(row, 1).data(Qt.UserRole).toPyObject())
+    #     return ids
+
+    # def get_urls(self):
+    #     urls = []
+    #     for row in range(self.rowCount()):
+    #         urls.append(self.item(row, 2).data(Qt.UserRole).toPyObject())
+    #     return urls
+
+    def get_books(self):
+        books = []
+        #print("=========================\nbooks:%s"%self.books)
+        for row in range(self.rowCount()):
+            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
+            #print("get_books rnum:%s"%rnum)
+            book = self.books[rnum]
+            #if book['good']:
+            books.append(book)
+        return books
+
+    def remove_selected_rows(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        message = '<p>Are you sure you want to remove this book from the list?'
+        if len(rows) > 1:
+            message = '<p>Are you sure you want to remove the selected %d books from the list?'%len(rows)
+        if not confirm(message,'fanfictiondownloader_delete_item', self):
+            return
+        first_sel_row = self.currentRow()
+        for selrow in reversed(rows):
+            self.removeRow(selrow.row())
+        if first_sel_row < self.rowCount():
+            self.select_and_scroll_to_row(first_sel_row)
+        elif self.rowCount() > 0:
+            self.select_and_scroll_to_row(first_sel_row - 1)
+
+    def select_and_scroll_to_row(self, row):
+        self.selectRow(row)
+        self.scrollToItem(self.currentItem())
+
+    def move_rows_up(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        first_sel_row = rows[0].row()
+        if first_sel_row <= 0:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in selrows:
+            self.swap_row_widgets(selrow - 1, selrow + 1)
+        scroll_to_row = first_sel_row - 1
+        if scroll_to_row > 0:
+            scroll_to_row = scroll_to_row - 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def move_rows_down(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        last_sel_row = rows[-1].row()
+        if last_sel_row == self.rowCount() - 1:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in reversed(selrows):
+            self.swap_row_widgets(selrow + 2, selrow)
+        scroll_to_row = last_sel_row + 1
+        if scroll_to_row < self.rowCount() - 1:
+            scroll_to_row = scroll_to_row + 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def swap_row_widgets(self, src_row, dest_row):
+        self.blockSignals(True)
+        self.insertRow(dest_row)
+        for col in range(0, self.columnCount()):
+            self.setItem(dest_row, col, self.takeItem(src_row, col))
+        self.removeRow(src_row)
+        self.blockSignals(False)

calibre-plugin/jobs.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Jim Miller'
+__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import time, os, traceback
+
+from ConfigParser import SafeConfigParser
+from StringIO import StringIO
+#from itertools import izip
+#from threading import Event
+
+#from calibre.gui2.convert.single import sort_formats_by_preference
+from calibre.utils.config import prefs
+from calibre.utils.ipc.server import Server
+from calibre.utils.ipc.job import ParallelJob
+from calibre.utils.logging import Log
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
+    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge
+
+# ------------------------------------------------------------------------------
+#
+#              Functions to perform downloads using worker jobs
+#
+# ------------------------------------------------------------------------------
+
+def do_download_worker(book_list, options,
+                       cpus, notification=lambda x,y:x):
+    '''
+    Master job, to launch child jobs to extract ISBN for a set of books
+    This is run as a worker job in the background to keep the UI more
+    responsive and get around the memory leak issues as it will launch
+    a child job for each book as a worker process
+    '''
+    server = Server(pool_size=cpus)
+
+    total = 0
+    # Queue all the jobs
+    for book in book_list:
+        if book['good']:
+            print("add job for %s"%book['url'])
+            total += 1
+            args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
+                    'do_download_for_worker',
+                    (book,options)]
+            job = ParallelJob('arbitrary',
+                              "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
+                              done=None,
+                              args=args)
+            job._book = book
+            # job._book_id = book_id
+            # job._title = title
+            # job._modified_date = modified_date
+            # job._existing_isbn = existing_isbn
+            server.add_job(job)
+    
+    # This server is an arbitrary_n job, so there is a notifier available.
+    # Set the % complete to a small number to avoid the 'unavailable' indicator
+    notification(0.01, 'Downloading FanFiction Stories')
+
+    # dequeue the job results as they arrive, saving the results
+    count = 0
+    while True:
+        job = server.changed_jobs_queue.get()
+        # A job can 'change' when it is not finished, for example if it
+        # produces a notification. Ignore these.
+        job.update()
+        if not job.is_finished:
+            continue
+        # A job really finished. Get the information.
+        output_book = job.result
+        #print("output_book:%s"%output_book)
+        book_list.remove(job._book)
+        book_list.append(job.result)
+        book_id = job._book['calibre_id']
+        #title = job._title
+        count = count + 1
+        notification(float(count)/total, 'Downloaded Story')
+        # Add this job's output to the current log
+        print('Logfile for book ID %s (%s)'%(book_id, book['title']))
+        print(job.details)
+
+        if count >= total:
+            # All done!
+            break
+
+    server.close()
+    
+    # return the book list as the job result
+    return book_list
+
+def do_download_for_worker(book,options):
+    '''
+    Child job, to extract isbn from formats for this specific book,
+    when run as a worker job
+    '''
+    try:
+        print("do_download_for_worker")
+        # print("is_adult:%s"%book['is_adult'])
+        # print("personal.ini len:%s"%len(options['personal.ini']))
+        # print("defaults.ini len:%s"%len(get_resources("defaults.ini")))
+        #time.sleep(2.0)
+        book['comment'] = 'Download started...'
+
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("defaults.ini")))
+        ffdlconfig.readfp(StringIO(options['personal.ini']))
+        
+        adapter = adapters.getAdapter(ffdlconfig,book['url'])
+        adapter.is_adult = book['is_adult'] 
+        adapter.username = book['username'] 
+        adapter.password = book['password']
+        
+        story = adapter.getStoryMetadataOnly()
+        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+
+        outfile = book['outfile']
+
+        ## No need to download at all.  Shouldn't ever get down here.
+        if options['collision'] in (CALIBREONLY):
+            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            book['comment'] = 'Metadata collected.'
+            
+        ## checks were done earlier, it's new or not dup or newer--just write it.
+        elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
+                ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
+            print("write to %s"%outfile)
+            writer.writeStory(outfilename=outfile)
+            book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
+            
+        ## checks were done earlier, just update it.
+        elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
+            
+            urlchaptercount = int(story.getMetadata('numChapters'))
+            ## First, get existing epub with titlepage and tocpage stripped.
+            updateio = StringIO()
+            (epuburl,chaptercount) = doMerge(updateio,
+                                             [book['epub_for_update']],
+                                             titlenavpoints=False,
+                                             striptitletoc=True,
+                                             forceunique=False)
+            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+
+            ## Get updated title page/metadata by itself in an epub.
+            ## Even if the title page isn't included, this carries the metadata.
+            titleio = StringIO()
+            writer.writeStory(outstream=titleio,metaonly=True)
+
+            newchaptersio = None
+            if urlchaptercount > chaptercount :
+                ## Go get the new chapters
+                newchaptersio = StringIO()
+                adapter.setChaptersRange(chaptercount+1,urlchaptercount)
+
+                adapter.config.set("overrides",'include_tocpage','false')
+                adapter.config.set("overrides",'include_titlepage','false')
+                writer.writeStory(outstream=newchaptersio)
+
+            ## Merge the three epubs together.
+            doMerge(outfile,
+                    [titleio,updateio,newchaptersio],
+                    fromfirst=True,
+                    titlenavpoints=False,
+                    striptitletoc=False,
+                    forceunique=False)
+            
+            book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
+                (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
+        
+    except NotGoingToDownload as d:
+        book['good']=False
+        book['comment']=unicode(d)
+        book['icon'] = d.icon
+
+    except Exception as e:
+        book['good']=False
+        book['comment']=unicode(e)
+        print("%s:%s"%(book,unicode(e)))
+        # XXX trace for not-expected exceptions
+        traceback.print_exc()
+
+    #time.sleep(10)
+    return book

fanficdownloader/adapters/adapter_test1.py

@@ -67,9 +67,11 @@ class TestSiteAdapter(BaseSiteAdapter):
             raise exceptions.FailedToLogin(self.url,self.username)
 
         if self.story.getMetadata('storyId') == '664':
-            self.story.setMetadata(u'title',"Test Story Title "+self.crazystring)
+            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring)
+            self.story.setMetadata('author','Test Author aa bare amp(&) quote(') amp(&)')
         else:
             self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
+            self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('storyUrl',self.url)
         self.story.setMetadata('description',u'Description '+self.crazystring+u''' Done
 
@@ -85,7 +87,6 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.setMetadata('status','In-Completed')
         self.story.setMetadata('rating','Tweenie')
         
-        self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('authorId','98765')
         self.story.setMetadata('authorUrl','http://author/url')
 

fanficdownloader/story.py

@@ -17,7 +17,7 @@
 
 import os
 
-from htmlcleanup import conditionalRemoveEntities
+from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 
 class Story:
     
@@ -30,15 +30,17 @@ class Story:
         self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
 
     def setMetadata(self, key, value):
+        ## still keeps < < and &
         self.metadata[key]=conditionalRemoveEntities(value)
 
     def getMetadataRaw(self,key):
         if self.metadata.has_key(key):
             return self.metadata[key]
         
-    def getMetadata(self, key):
+    def getMetadata(self, key, removeallentities=False):
+        value = None
         if self.getLists().has_key(key):
-            return ', '.join(self.getList(key))
+            value = ', '.join(self.getList(key))
         if self.metadata.has_key(key):
             value = self.metadata[key]
             if value:
@@ -48,6 +50,10 @@ class Story:
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
                  value = value.strftime("%Y-%m-%d")
+                 
+        if removeallentities and value != None:
+            return removeAllEntities(value)
+        else:
             return value
 
     def getAllMetadata(self):