diff --git a/app.yaml b/app.yaml index a21f6c6..d053fad 100644 --- a/app.yaml +++ b/app.yaml @@ -1,5 +1,5 @@ # ffd-retief-hrd fanfictiondownloader -application: fanfictiondownloader +application: ffd-retief-hrd version: 4-2-0 runtime: python27 api_version: 1 diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index a25fee9..d952652 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -25,11 +25,9 @@ class InterfacePluginDemo(InterfaceActionBase): description = 'UI plugin to download FanFiction stories from various sites.' supported_platforms = ['windows', 'osx', 'linux'] author = 'Jim Miller' - version = (1, 0, 4) + version = (1, 1, 0) minimum_calibre_version = (0, 8, 30) -# action_menu_clone_qaction = True - #: This field defines the GUI plugin class that contains all the code #: that actually does something. Its format is module_path:class_name #: The specified class must be defined in the specified module. diff --git a/calibre-plugin/about.txt b/calibre-plugin/about.txt index 1d11808..3ae79b0 100644 --- a/calibre-plugin/about.txt +++ b/calibre-plugin/about.txt @@ -1,11 +1,21 @@

FanFictionDownLoader Plugin

-

http://code.google.com/p/fanficdownloader/

+

Created by Jim Miller, borrowing heavily from Grant Drake's +'Reading List', +'Extract ISBN' and +'Count Pages' +plugins.

-

Created by Jim Miller, borrowing heavily from -Kovid Goyal's 'The InterfacePlugin Demo' and -Grant Drake's 'Count Pages' plugins.

+

+Calibre officially distributes plugins from the mobileread.com forum site. +The official distro channel for this plugin is there: FanFictionDownLoader +

-

Requires calibre >= 0.8.30

+

However, I monitor the +general users +group for the downloader more closely. That also covers the web application and CLI. +

+The source project for this plugin is +also available. diff --git a/calibre-plugin/common_utils.py b/calibre-plugin/common_utils.py new file mode 100644 index 0000000..19e8697 --- /dev/null +++ b/calibre-plugin/common_utils.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Grant Drake ' +__docformat__ = 'restructuredtext en' + +import os +from PyQt4 import QtGui +from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout, + QTableWidgetItem, QFont, QLineEdit, QComboBox, + QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime) +from calibre.constants import iswindows +from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME +from calibre.gui2.actions import menu_action_unique_name +from calibre.gui2.keyboard import ShortcutConfig +from calibre.utils.config import config_dir +from calibre.utils.date import now, format_date, qt_to_dt, UNDEFINED_DATE + +# Global definition of our plugin name. Used for common functions that require this. +plugin_name = None +# Global definition of our plugin resources. Used to share between the xxxAction and xxxBase +# classes if you need any zip images to be displayed on the configuration dialog. +plugin_icon_resources = {} + + +def set_plugin_icon_resources(name, resources): + ''' + Set our global store of plugin name and icon resources for sharing between + the InterfaceAction class which reads them and the ConfigWidget + if needed for use on the customization dialog for this plugin. + ''' + global plugin_icon_resources, plugin_name + plugin_name = name + plugin_icon_resources = resources + + +def get_icon(icon_name): + ''' + Retrieve a QIcon for the named image from the zip file if it exists, + or if not then from Calibre's image cache. + ''' + if icon_name: + pixmap = get_pixmap(icon_name) + if pixmap is None: + # Look in Calibre's cache for the icon + return QIcon(I(icon_name)) + else: + return QIcon(pixmap) + return QIcon() + + +def get_pixmap(icon_name): + ''' + Retrieve a QPixmap for the named image + Any icons belonging to the plugin must be prefixed with 'images/' + ''' + global plugin_icon_resources, plugin_name + + if not icon_name.startswith('images/'): + # We know this is definitely not an icon belonging to this plugin + pixmap = QPixmap() + pixmap.load(I(icon_name)) + return pixmap + + # Check to see whether the icon exists as a Calibre resource + # This will enable skinning if the user stores icons within a folder like: + # ...\AppData\Roaming\calibre\resources\images\Plugin Name\ + if plugin_name: + local_images_dir = get_local_images_dir(plugin_name) + local_image_path = os.path.join(local_images_dir, icon_name.replace('images/', '')) + if os.path.exists(local_image_path): + pixmap = QPixmap() + pixmap.load(local_image_path) + return pixmap + + # As we did not find an icon elsewhere, look within our zip resources + if icon_name in plugin_icon_resources: + pixmap = QPixmap() + pixmap.loadFromData(plugin_icon_resources[icon_name]) + return pixmap + return None + + +def get_local_images_dir(subfolder=None): + ''' + Returns a path to the user's local resources/images folder + If a subfolder name parameter is specified, appends this to the path + ''' + images_dir = os.path.join(config_dir, 'resources/images') + if subfolder: + images_dir = os.path.join(images_dir, subfolder) + if iswindows: + images_dir = os.path.normpath(images_dir) + return images_dir + + +def create_menu_item(ia, parent_menu, menu_text, image=None, tooltip=None, + shortcut=(), triggered=None, is_checked=None): + ''' + Create a menu action with the specified criteria and action + Note that if no shortcut is specified, will not appear in Preferences->Keyboard + This method should only be used for actions which either have no shortcuts, + or register their menus only once. Use create_menu_action_unique for all else. + ''' + if shortcut is not None: + if len(shortcut) == 0: + shortcut = () + else: + shortcut = _(shortcut) + ac = ia.create_action(spec=(menu_text, None, tooltip, shortcut), + attr=menu_text) + if image: + ac.setIcon(get_icon(image)) + if triggered is not None: + ac.triggered.connect(triggered) + if is_checked is not None: + ac.setCheckable(True) + if is_checked: + ac.setChecked(True) + + parent_menu.addAction(ac) + return ac + + +def create_menu_action_unique(ia, parent_menu, menu_text, image=None, tooltip=None, + shortcut=None, triggered=None, is_checked=None, shortcut_name=None, + unique_name=None): + ''' + Create a menu action with the specified criteria and action, using the new + InterfaceAction.create_menu_action() function which ensures that regardless of + whether a shortcut is specified it will appear in Preferences->Keyboard + ''' + orig_shortcut = shortcut + kb = ia.gui.keyboard + if unique_name is None: + unique_name = menu_text + if not shortcut == False: + full_unique_name = menu_action_unique_name(ia, unique_name) + if full_unique_name in kb.shortcuts: + shortcut = False + else: + if shortcut is not None and not shortcut == False: + if len(shortcut) == 0: + shortcut = None + else: + shortcut = _(shortcut) + + if shortcut_name is None: + shortcut_name = menu_text.replace('&','') + + ac = ia.create_menu_action(parent_menu, unique_name, menu_text, icon=None, shortcut=shortcut, + description=tooltip, triggered=triggered, shortcut_name=shortcut_name) + if shortcut == False and not orig_shortcut == False: + if ac.calibre_shortcut_unique_name in ia.gui.keyboard.shortcuts: + kb.replace_action(ac.calibre_shortcut_unique_name, ac) + if image: + ac.setIcon(get_icon(image)) + if is_checked is not None: + ac.setCheckable(True) + if is_checked: + ac.setChecked(True) + return ac + + +def swap_author_names(author): + if author.find(',') == -1: + return author + name_parts = author.strip().partition(',') + return name_parts[2].strip() + ' ' + name_parts[0] + + +def get_library_uuid(db): + try: + library_uuid = db.library_id + except: + library_uuid = '' + return library_uuid + + +class ImageLabel(QLabel): + + def __init__(self, parent, icon_name, size=16): + QLabel.__init__(self, parent) + pixmap = get_pixmap(icon_name) + self.setPixmap(pixmap) + self.setMaximumSize(size, size) + self.setScaledContents(True) + + +class ImageTitleLayout(QHBoxLayout): + ''' + A reusable layout widget displaying an image followed by a title + ''' + def __init__(self, parent, icon_name, title): + QHBoxLayout.__init__(self) + title_image_label = QLabel(parent) + pixmap = get_pixmap(icon_name) + if pixmap is None: + pixmap = get_pixmap('library.png') + # error_dialog(parent, _('Restart required'), + # _('You must restart Calibre before using this plugin!'), show=True) + else: + title_image_label.setPixmap(pixmap) + title_image_label.setMaximumSize(32, 32) + title_image_label.setScaledContents(True) + self.addWidget(title_image_label) + + title_font = QFont() + title_font.setPointSize(16) + shelf_label = QLabel(title, parent) + shelf_label.setFont(title_font) + self.addWidget(shelf_label) + self.insertStretch(-1) + + +class SizePersistedDialog(QDialog): + ''' + This dialog is a base class for any dialogs that want their size/position + restored when they are next opened. + ''' + def __init__(self, parent, unique_pref_name): + QDialog.__init__(self, parent) + self.unique_pref_name = unique_pref_name + self.geom = gprefs.get(unique_pref_name, None) + self.finished.connect(self.dialog_closing) + + def resize_dialog(self): + if self.geom is None: + self.resize(self.sizeHint()) + else: + self.restoreGeometry(self.geom) + + def dialog_closing(self, result): + geom = bytearray(self.saveGeometry()) + gprefs[self.unique_pref_name] = geom + + +class ReadOnlyTableWidgetItem(QTableWidgetItem): + + def __init__(self, text): + if text is None: + text = '' + QTableWidgetItem.__init__(self, text, QtGui.QTableWidgetItem.UserType) + self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled) + + +class RatingTableWidgetItem(QTableWidgetItem): + + def __init__(self, rating, is_read_only=False): + QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType) + self.setData(Qt.DisplayRole, rating) + if is_read_only: + self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled) + + +class DateTableWidgetItem(QTableWidgetItem): + + def __init__(self, date_read, is_read_only=False, default_to_today=False): + if date_read == UNDEFINED_DATE and default_to_today: + date_read = now() + if is_read_only: + QTableWidgetItem.__init__(self, format_date(date_read, None), QtGui.QTableWidgetItem.UserType) + self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled) + else: + QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType) + self.setData(Qt.DisplayRole, QDateTime(date_read)) + + +class NoWheelComboBox(QComboBox): + + def wheelEvent (self, event): + # Disable the mouse wheel on top of the combo box changing selection as plays havoc in a grid + event.ignore() + + +class CheckableTableWidgetItem(QTableWidgetItem): + + def __init__(self, checked=False, is_tristate=False): + QTableWidgetItem.__init__(self, '') + self.setFlags(Qt.ItemFlags(Qt.ItemIsSelectable | Qt.ItemIsUserCheckable | Qt.ItemIsEnabled )) + if is_tristate: + self.setFlags(self.flags() | Qt.ItemIsTristate) + if checked: + self.setCheckState(Qt.Checked) + else: + if is_tristate and checked is None: + self.setCheckState(Qt.PartiallyChecked) + else: + self.setCheckState(Qt.Unchecked) + + def get_boolean_value(self): + ''' + Return a boolean value indicating whether checkbox is checked + If this is a tristate checkbox, a partially checked value is returned as None + ''' + if self.checkState() == Qt.PartiallyChecked: + return None + else: + return self.checkState() == Qt.Checked + + +class TextIconWidgetItem(QTableWidgetItem): + + def __init__(self, text, icon): + QTableWidgetItem.__init__(self, text) + if icon: + self.setIcon(icon) + + +class ReadOnlyTextIconWidgetItem(ReadOnlyTableWidgetItem): + + def __init__(self, text, icon): + ReadOnlyTableWidgetItem.__init__(self, text) + if icon: + self.setIcon(icon) + + +class ReadOnlyLineEdit(QLineEdit): + + def __init__(self, text, parent): + if text is None: + text = '' + QLineEdit.__init__(self, text, parent) + self.setEnabled(False) + + +class KeyValueComboBox(QComboBox): + + def __init__(self, parent, values, selected_key): + QComboBox.__init__(self, parent) + self.values = values + self.populate_combo(selected_key) + + def populate_combo(self, selected_key): + self.clear() + selected_idx = idx = -1 + for key, value in self.values.iteritems(): + idx = idx + 1 + self.addItem(value) + if key == selected_key: + selected_idx = idx + self.setCurrentIndex(selected_idx) + + def selected_key(self): + for key, value in self.values.iteritems(): + if value == unicode(self.currentText()).strip(): + return key + + +class CustomColumnComboBox(QComboBox): + + def __init__(self, parent, custom_columns, selected_column, initial_items=['']): + QComboBox.__init__(self, parent) + self.populate_combo(custom_columns, selected_column, initial_items) + + def populate_combo(self, custom_columns, selected_column, initial_items=['']): + self.clear() + self.column_names = initial_items + if len(initial_items) > 0: + self.addItems(initial_items) + selected_idx = 0 + for idx, value in enumerate(initial_items): + if value == selected_column: + selected_idx = idx + for key in sorted(custom_columns.keys()): + self.column_names.append(key) + self.addItem('%s (%s)'%(key, custom_columns[key]['name'])) + if key == selected_column: + selected_idx = len(self.column_names) - 1 + self.setCurrentIndex(selected_idx) + + def get_selected_column(self): + return self.column_names[self.currentIndex()] + + +class KeyboardConfigDialog(SizePersistedDialog): + ''' + This dialog is used to allow editing of keyboard shortcuts. + ''' + def __init__(self, gui, group_name): + SizePersistedDialog.__init__(self, gui, 'Keyboard shortcut dialog') + self.gui = gui + self.setWindowTitle('Keyboard shortcuts') + layout = QVBoxLayout(self) + self.setLayout(layout) + + self.keyboard_widget = ShortcutConfig(self) + layout.addWidget(self.keyboard_widget) + self.group_name = group_name + + button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) + button_box.accepted.connect(self.commit) + button_box.rejected.connect(self.reject) + layout.addWidget(button_box) + + # Cause our dialog size to be restored from prefs or created on first usage + self.resize_dialog() + self.initialize() + + def initialize(self): + self.keyboard_widget.initialize(self.gui.keyboard) + self.keyboard_widget.highlight_group(self.group_name) + + def commit(self): + self.keyboard_widget.commit() + self.accept() + + +class DateDelegate(QStyledItemDelegate): + ''' + Delegate for dates. Because this delegate stores the + format as an instance variable, a new instance must be created for each + column. This differs from all the other delegates. + ''' + def __init__(self, parent): + QStyledItemDelegate.__init__(self, parent) + self.format = 'dd MMM yyyy' + + def displayText(self, val, locale): + d = val.toDateTime() + if d <= UNDEFINED_QDATETIME: + return '' + return format_date(qt_to_dt(d, as_utc=False), self.format) + + def createEditor(self, parent, option, index): + qde = QStyledItemDelegate.createEditor(self, parent, option, index) + qde.setDisplayFormat(self.format) + qde.setMinimumDateTime(UNDEFINED_QDATETIME) + qde.setSpecialValueText(_('Undefined')) + qde.setCalendarPopup(True) + return qde + + def setEditorData(self, editor, index): + val = index.model().data(index, Qt.DisplayRole).toDateTime() + if val is None or val == UNDEFINED_QDATETIME: + val = now() + editor.setDateTime(val) + + def setModelData(self, editor, model, index): + val = editor.dateTime() + if val <= UNDEFINED_QDATETIME: + model.setData(index, UNDEFINED_QDATETIME, Qt.EditRole) + else: + model.setData(index, QDateTime(val), Qt.EditRole) diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index 8fdb9bd..a91f731 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -10,9 +10,12 @@ __docformat__ = 'restructuredtext en' from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QTextEdit, QComboBox, QCheckBox, QPushButton) +from calibre.gui2 import dynamic, info_dialog from calibre.utils.config import JSONConfig -from calibre_plugins.fanfictiondownloader_plugin.dialogs import (OVERWRITE, ADDNEW, SKIP,CALIBREONLY,UPDATE) +from calibre_plugins.fanfictiondownloader_plugin.dialogs \ + import (SKIP, ADDNEW, UPDATE, UPDATEALWAYS, OVERWRITE, OVERWRITEALWAYS, + CALIBREONLY,collision_order) # This is where all preferences for this plugin will be stored # Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also @@ -21,17 +24,12 @@ from calibre_plugins.fanfictiondownloader_plugin.dialogs import (OVERWRITE, ADDN # so as to ensure you dont accidentally clobber a calibre config file prefs = JSONConfig('plugins/fanfictiondownloader_plugin') -# urlsfrompriority values -CLIP = 'Clipboard' -SELECTED = 'Selected Stories' - # Set defaults prefs.defaults['personal.ini'] = get_resources('example.ini') prefs.defaults['updatemeta'] = True -prefs.defaults['onlyoverwriteifnewer'] = False +#prefs.defaults['onlyoverwriteifnewer'] = False prefs.defaults['urlsfromclip'] = True -prefs.defaults['urlsfromselected'] = True -prefs.defaults['urlsfrompriority'] = SELECTED +prefs.defaults['updatedefault'] = True prefs.defaults['fileform'] = 'epub' prefs.defaults['collision'] = OVERWRITE prefs.defaults['deleteotherforms'] = False @@ -62,13 +60,10 @@ class ConfigWidget(QWidget): label.setToolTip("What to do if there's already an existing story with the same title and author.") horz.addWidget(label) self.collision = QComboBox(self) - self.collision.addItem(OVERWRITE) - self.collision.addItem(UPDATE) - self.collision.addItem(ADDNEW) - self.collision.addItem(SKIP) - self.collision.addItem(CALIBREONLY) + for o in collision_order: + self.collision.addItem(o) self.collision.setCurrentIndex(self.collision.findText(prefs['collision'])) - self.collision.setToolTip('Overwrite will replace the existing story. Add New will create a new story with the same title and author.') + # self.collision.setToolTip('Overwrite will replace the existing story. Add New will create a new story with the same title and author.') label.setBuddy(self.collision) horz.addWidget(self.collision) self.l.addLayout(horz) @@ -78,33 +73,21 @@ class ConfigWidget(QWidget): self.updatemeta.setChecked(prefs['updatemeta']) self.l.addWidget(self.updatemeta) - self.onlyoverwriteifnewer = QCheckBox('Default Only Overwrite Story if Newer',self) - self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer or from the same day.") - self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer']) - self.l.addWidget(self.onlyoverwriteifnewer) + # self.onlyoverwriteifnewer = QCheckBox('Default Only Overwrite Story if Newer',self) + # self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer or from the same day.") + # self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer']) + # self.l.addWidget(self.onlyoverwriteifnewer) self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self) - self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when starting plugin?') + self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New?') self.urlsfromclip.setChecked(prefs['urlsfromclip']) self.l.addWidget(self.urlsfromclip) - self.urlsfromselected = QCheckBox('Take URLs from Selected Stories?',self) - self.urlsfromselected.setToolTip('Prefill URLs from valid URLs in selected stories when starting plugin?') - self.urlsfromselected.setChecked(prefs['urlsfromselected']) - self.l.addWidget(self.urlsfromselected) - - horz = QHBoxLayout() - label = QLabel('Take URLs from which first:') - label.setToolTip("If both clipbaord and selected enabled and populated, which is used?") - horz.addWidget(label) - self.urlsfrompriority = QComboBox(self) - self.urlsfrompriority.addItem(SELECTED) - self.urlsfrompriority.addItem(CLIP) - self.urlsfrompriority.setCurrentIndex(self.urlsfrompriority.findText(prefs['urlsfrompriority'])) - self.urlsfrompriority.setToolTip('If both clipbaord and selected enabled and populated, which is used?') - label.setBuddy(self.urlsfrompriority) - horz.addWidget(self.urlsfrompriority) - self.l.addLayout(horz) + self.updatedefault = QCheckBox('Default to Update when books selected?',self) + self.updatedefault.setToolTip('The top FanFictionDownLoader plugin button will start Update if\n'+ + 'books are selected. If unchecked, it will always bring up \'Add New\'.') + self.updatedefault.setChecked(prefs['updatedefault']) + self.l.addWidget(self.updatedefault) self.deleteotherforms = QCheckBox('Delete other existing formats?',self) self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.') @@ -124,14 +107,19 @@ class ConfigWidget(QWidget): self.defaults.clicked.connect(self.show_defaults) self.l.addWidget(self.defaults) + reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self) + reset_confirmation_button.setToolTip(_( + 'Reset all show me again dialogs for the FanFictionDownLoader plugin')) + reset_confirmation_button.clicked.connect(self.reset_dialogs) + self.l.addWidget(reset_confirmation_button) + def save_settings(self): prefs['fileform'] = unicode(self.fileform.currentText()) prefs['collision'] = unicode(self.collision.currentText()) prefs['updatemeta'] = self.updatemeta.isChecked() - prefs['urlsfrompriority'] = unicode(self.urlsfrompriority.currentText()) prefs['urlsfromclip'] = self.urlsfromclip.isChecked() - prefs['urlsfromselected'] = self.urlsfromselected.isChecked() - prefs['onlyoverwriteifnewer'] = self.onlyoverwriteifnewer.isChecked() + prefs['updatedefault'] = self.updatedefault.isChecked() +# prefs['onlyoverwriteifnewer'] = self.onlyoverwriteifnewer.isChecked() prefs['deleteotherforms'] = self.deleteotherforms.isChecked() ini = unicode(self.ini.toPlainText()) @@ -145,7 +133,16 @@ class ConfigWidget(QWidget): def show_defaults(self): text = get_resources('defaults.ini') ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_() - + + def reset_dialogs(self): + for key in dynamic.keys(): + if key.startswith('fanfictiondownloader_') and key.endswith('_again') \ + and dynamic[key] is False: + dynamic[key] = True + info_dialog(self, _('Done'), + _('Confirmation dialogs have all been reset'), show=True) + + class ShowDefaultsIniDialog(QDialog): def __init__(self, icon, text, parent=None): diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py index 534ef4c..a14c53c 100644 --- a/calibre-plugin/dialogs.py +++ b/calibre-plugin/dialogs.py @@ -9,28 +9,48 @@ __docformat__ = 'restructuredtext en' import traceback -from PyQt4.Qt import (QDialog, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout, - QPushButton, QProgressDialog, QString, QLabel, QCheckBox, - QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, - QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt ) +from PyQt4 import QtGui +from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout, + QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, + QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant, + QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt,QAbstractItemView ) from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog +from calibre.gui2.dialogs.confirm_delete import confirm from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions +from calibre_plugins.fanfictiondownloader_plugin.common_utils \ + import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog, + ImageTitleLayout, get_icon) -OVERWRITE='Overwrite' -UPDATE='Update EPUB' -ADDNEW='Add New' SKIP='Skip' +ADDNEW='Add New Book' +UPDATE='Update EPUB if New Chapters' +UPDATEALWAYS='Update EPUB Always' +OVERWRITE='Overwrite if Newer' +OVERWRITEALWAYS='Overwrite Always' CALIBREONLY='Update Calibre Metadata Only' +collision_order=[SKIP, + ADDNEW, + UPDATE, + UPDATEALWAYS, + OVERWRITE, + OVERWRITEALWAYS, + CALIBREONLY,] + +class NotGoingToDownload(Exception): + def __init__(self,error,icon='dialog_error.png'): + self.error=error + self.icon=icon -class DownloadDialog(QDialog): + def __str__(self): + return self.error + +class AddNewDialog(SizePersistedDialog): - def __init__(self, gui, prefs, icon, url_list_text, do_user_config, start_downloads): - QDialog.__init__(self, gui) + def __init__(self, gui, prefs, icon, url_list_text): + SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog') self.gui = gui - self.do_user_config = do_user_config - self.start_downloads = start_downloads self.setMinimumWidth(300) self.l = QVBoxLayout() @@ -41,19 +61,19 @@ class DownloadDialog(QDialog): self.l.addWidget(QLabel('Story URL(s), one per line:')) self.url = QTextEdit(self) - self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from selected books or clipboard, but only valid URLs.') + self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.') self.url.setLineWrapMode(QTextEdit.NoWrap) self.url.setText(url_list_text) self.l.addWidget(self.url) - self.ffdl_button = QPushButton( - 'Download Stories', self) - self.ffdl_button.setToolTip('Start download(s).') - self.ffdl_button.clicked.connect(self.ffdl) - # if there's already URL(s), focus 'go' button - if url_list_text: - self.ffdl_button.setFocus() - self.l.addWidget(self.ffdl_button) + # self.ffdl_button = QPushButton( + # 'Download Stories', self) + # self.ffdl_button.setToolTip('Start download(s).') + # self.ffdl_button.clicked.connect(self.ffdl) + # # if there's already URL(s), focus 'go' button + # if url_list_text: + # self.ffdl_button.setFocus() + # self.l.addWidget(self.ffdl_button) horz = QHBoxLayout() label = QLabel('Output &Format:') @@ -74,17 +94,16 @@ class DownloadDialog(QDialog): label.setToolTip("What to do if there's already an existing story with the same title and author.") horz.addWidget(label) self.collision = QComboBox(self) - self.collision.addItem(OVERWRITE) - self.collision.addItem(UPDATE) - self.collision.addItem(ADDNEW) - self.collision.addItem(SKIP) - self.collision.addItem(CALIBREONLY) + # add collision options + for o in collision_order: + self.collision.addItem(o) + self.collision.setCurrentIndex(self.collision.findText(prefs['collision'])) - self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+ - UPDATE+' will download new chapters only and add to existing EPUB.\n'+ - ADDNEW+' will create a new story with the same title and author.\n'+ - SKIP+' will not download existing stories.\n'+ - CALIBREONLY+' will not download stories, but will update Calibre metadata.') + # self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+ + # UPDATE+' will download new chapters only and add to existing EPUB.\n'+ + # ADDNEW+' will create a new story with the same title and author.\n'+ + # SKIP+' will not download existing stories.\n'+ + # CALIBREONLY+' will not download stories, but will update Calibre metadata.') label.setBuddy(self.collision) horz.addWidget(self.collision) self.l.addLayout(horz) @@ -94,50 +113,42 @@ class DownloadDialog(QDialog): self.updatemeta.setChecked(prefs['updatemeta']) self.l.addWidget(self.updatemeta) - self.onlyoverwriteifnewer = QCheckBox('Only Overwrite Story if Newer',self) - self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer.\n"+ - "From the same day counts as 'newer' because the sites don't give update time.") - self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer']) - self.l.addWidget(self.onlyoverwriteifnewer) + # self.onlyoverwriteifnewer = QCheckBox('Only Overwrite Story if Newer',self) + # self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer.\n"+ + # "From the same day counts as 'newer' because the sites don't give update time.") + # self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer']) + # self.l.addWidget(self.onlyoverwriteifnewer) - horz = QHBoxLayout() - self.about_button = QPushButton('About', self) - self.about_button.clicked.connect(self.about) - horz.addWidget(self.about_button) - self.conf_button = QPushButton( - 'Configure this plugin', self) - self.conf_button.clicked.connect(self.config) - horz.addWidget(self.conf_button) - self.l.addLayout(horz) + # horz = QHBoxLayout() + # self.about_button = QPushButton('About', self) + # self.about_button.clicked.connect(self.about) + # horz.addWidget(self.about_button) + # self.conf_button = QPushButton( + # 'Configure this plugin', self) + # self.conf_button.clicked.connect(self.config) + # horz.addWidget(self.conf_button) + # self.l.addLayout(horz) - self.resize(self.sizeHint()) + button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) + button_box.accepted.connect(self.accept) + button_box.rejected.connect(self.reject) + self.l.addWidget(button_box) + + if url_list_text: + button_box.button(QDialogButtonBox.Ok).setFocus() + + self.resize_dialog() + #self.resize(self.sizeHint()) - def about(self): - # Get the about text from a file inside the plugin zip file - # The get_resources function is a builtin function defined for all your - # plugin code. It loads files from the plugin zip file. It returns - # the bytes from the specified file. - # - # Note that if you are loading more than one file, for performance, you - # should pass a list of names to get_resources. In this case, - # get_resources will return a dictionary mapping names to bytes. Names that - # are not found in the zip file will not be in the returned dictionary. - text = get_resources('about.txt') -# QMessageBox.about(self, 'About the FanFictionDownLoader Plugin', - # text.decode('utf-8')) - AboutDialog(self.windowIcon(),text,self).exec_() - - def ffdl(self): - self.start_downloads(unicode(self.url.toPlainText()), - unicode(self.fileform.currentText()), - unicode(self.collision.currentText()), - self.updatemeta.isChecked(), - self.onlyoverwriteifnewer.isChecked()) - self.hide() - - def config(self): - self.do_user_config(parent=self) + def get_ffdl_options(self): + return { + 'fileform': unicode(self.fileform.currentText()), + 'collision': unicode(self.collision.currentText()), + 'updatemeta': unicode(self.updatemeta.isChecked()), + } + def get_urlstext(self): + return unicode(self.url.toPlainText()) class UserPassDialog(QDialog): ''' @@ -184,19 +195,22 @@ class MetadataProgressDialog(QProgressDialog): ''' ProgressDialog displayed while fetching metadata for each story. ''' - def __init__(self, gui, loop_list, fileform, getadapter_function, download_list_function, db): + def __init__(self, gui, + book_list, + options, + metadata_function, + startdownload_function): QProgressDialog.__init__(self, "Fetching metadata for stories...", - QString(), 0, len(loop_list), gui) + QString(), 0, len(book_list), gui) self.setWindowTitle("Downloading metadata for stories") self.setMinimumWidth(500) self.gui = gui - self.db = db - self.loop_list = loop_list - self.fileform = fileform - self.getadapter_function = getadapter_function - self.download_list_function = download_list_function - self.i, self.loop_bad, self.loop_good = 0, [], [] + self.book_list = book_list + self.options = options + self.metadata_function = metadata_function + self.startdownload_function = startdownload_function + self.i = 0 ## self.do_loop does QTimer.singleShot on self.do_loop also. ## A weird way to do a loop, but that was the example I had. @@ -204,7 +218,7 @@ class MetadataProgressDialog(QProgressDialog): self.exec_() def updateStatus(self): - self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.loop_list))) + self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.book_list))) self.setValue(self.i+1) print(self.labelText()) @@ -214,49 +228,41 @@ class MetadataProgressDialog(QProgressDialog): if self.i == 0: self.setValue(0) - if self.i >= len(self.loop_list) or self.wasCanceled(): - return self.do_when_finished() - - else: - current = self.loop_list[self.i] - try: - ## collision spec passed into getadapter by partial from ffdl_plugin - ## no retval only if it exists, but collision is SKIP - retval = self.getadapter_function(current,self.fileform) - self.loop_good.append((current,retval)) - except Exception as e: - print("%s:%s"%(current,e)) - self.loop_bad.append((current,e)) - traceback.print_exc() + book = self.book_list[self.i] + try: + ## collision spec passed into getadapter by partial from ffdl_plugin + ## no retval only if it exists, but collision is SKIP + self.metadata_function(book) - self.updateStatus() - self.i += 1 + except NotGoingToDownload as d: + book['good']=False + book['comment']=unicode(d) + book['icon'] = d.icon + + except Exception as e: + book['good']=False + book['comment']=unicode(e) + print("%s:%s"%(book,unicode(e))) + # XXX trace for not-expected exceptions + #traceback.print_exc() + + self.updateStatus() + self.i += 1 + + if self.i >= len(self.book_list) or self.wasCanceled(): + return self.do_when_finished() + else: QTimer.singleShot(0, self.do_loop) def do_when_finished(self): self.hide() - - # Queues a job to process these ePub/Mobi books in the background. - self.download_list_function(self.loop_good,self.fileform) - - if self.loop_bad != []: - res = [] - for j in self.loop_bad: - res.append('%s : %s'%j) - msg = '%s' % '\n'.join(res) - warning_dialog(self.gui, _('Not going to download some stories'), - _('Not going to download %d of %d stories.') % - (len(self.loop_bad), len(self.loop_list)), - msg).exec_() - # else: - # info_dialog(self.gui, "Starting Downloads", - # "Got metadata and started download for %d stories."%len(self.loop_good), - # show_copy_button=False).exec_() - self.gui = None + self.gui = None + # Queues a job to process these books in the background. + self.startdownload_function(self.book_list) class AboutDialog(QDialog): - def __init__(self, icon, text, parent=None): + def __init__(self, parent, icon, text): QDialog.__init__(self, parent) self.resize(400, 250) self.l = QGridLayout() @@ -267,7 +273,7 @@ class AboutDialog(QDialog): self.label = QLabel(text) self.label.setOpenExternalLinks(True) self.label.setWordWrap(True) - self.setWindowTitle(_('Update available!')) + self.setWindowTitle(_('About FanFictionDownLoader')) self.setWindowIcon(icon) self.l.addWidget(self.logo, 0, 0) self.l.addWidget(self.label, 0, 1) @@ -277,10 +283,321 @@ class AboutDialog(QDialog): self.l.addWidget(self.bb, 2, 0, 1, -1) self.bb.accepted.connect(self.accept) - -class NotGoingToDownload(Exception): - def __init__(self,error): - self.error=error +class IconWidgetItem(ReadOnlyTextIconWidgetItem): + def __init__(self, text, icon, sort_key): + ReadOnlyTextIconWidgetItem.__init__(self, text, icon) + self.sort_key = sort_key - def __str__(self): - return self.error + #Qt uses a simple < check for sorting items, override this to use the sortKey + def __lt__(self, other): + return self.sort_key < other.sort_key + +class AuthorTableWidgetItem(ReadOnlyTableWidgetItem): + def __init__(self, text, sort_key): + ReadOnlyTableWidgetItem.__init__(self, text) + self.sort_key = sort_key + + #Qt uses a simple < check for sorting items, override this to use the sortKey + def __lt__(self, other): + return self.sort_key < other.sort_key + +class UpdateExistingDialog(SizePersistedDialog): + def __init__(self, gui, header, prefs, icon, books, + save_size_name='FanFictionDownLoader plugin:update list dialog'): + SizePersistedDialog.__init__(self, gui, save_size_name) + self.gui = gui + + self.setWindowTitle(header) + self.setWindowIcon(icon) + + layout = QVBoxLayout(self) + self.setLayout(layout) + title_layout = ImageTitleLayout(self, 'images/icon.png', + header) + layout.addLayout(title_layout) + books_layout = QHBoxLayout() + layout.addLayout(books_layout) + + self.books_table = StoryListTableWidget(self) + books_layout.addWidget(self.books_table) + + button_layout = QVBoxLayout() + books_layout.addLayout(button_layout) + # self.move_up_button = QtGui.QToolButton(self) + # self.move_up_button.setToolTip('Move selected books up the list') + # self.move_up_button.setIcon(QIcon(I('arrow-up.png'))) + # self.move_up_button.clicked.connect(self.books_table.move_rows_up) + # button_layout.addWidget(self.move_up_button) + spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding) + button_layout.addItem(spacerItem) + self.remove_button = QtGui.QToolButton(self) + self.remove_button.setToolTip('Remove selected books from the list') + self.remove_button.setIcon(get_icon('list_remove.png')) + self.remove_button.clicked.connect(self.remove_from_list) + button_layout.addWidget(self.remove_button) + spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding) + button_layout.addItem(spacerItem1) + # self.move_down_button = QtGui.QToolButton(self) + # self.move_down_button.setToolTip('Move selected books down the list') + # self.move_down_button.setIcon(QIcon(I('arrow-down.png'))) + # self.move_down_button.clicked.connect(self.books_table.move_rows_down) + # button_layout.addWidget(self.move_down_button) + + options_layout = QHBoxLayout() + + label = QLabel('Output &Format:') + options_layout.addWidget(label) + self.fileform = QComboBox(self) + self.fileform.addItem('epub') + self.fileform.addItem('mobi') + self.fileform.addItem('html') + self.fileform.addItem('txt') + self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform'])) + self.fileform.setToolTip('Choose output format to create. May set default from plugin configuration.') + label.setBuddy(self.fileform) + options_layout.addWidget(self.fileform) + + label = QLabel('Update Mode:') + label.setToolTip("What sort of update to perform. May set default from plugin configuration.") + options_layout.addWidget(label) + self.collision = QComboBox(self) + for o in collision_order: + if o not in [ADDNEW,SKIP]: + self.collision.addItem(o) + self.collision.setCurrentIndex(self.collision.findText(prefs['collision'])) + # self.collision.setToolTip('Overwrite will replace the existing story. Add New will create a new story with the same title and author.') + label.setBuddy(self.collision) + options_layout.addWidget(self.collision) + + self.updatemeta = QCheckBox('Update Calibre &Metadata?',self) + self.updatemeta.setToolTip('Update metadata for story in Calibre from web site? May set default from plugin configuration.') + self.updatemeta.setChecked(prefs['updatemeta']) + options_layout.addWidget(self.updatemeta) + + button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) + button_box.accepted.connect(self.accept) + button_box.rejected.connect(self.reject) + options_layout.addWidget(button_box) + + layout.addLayout(options_layout) + + # Cause our dialog size to be restored from prefs or created on first usage + self.resize_dialog() + self.books_table.populate_table(books) + + def remove_from_list(self): + self.books_table.remove_selected_rows() + + def get_books(self): + return self.books_table.get_books() + + def get_ffdl_options(self): + return { + 'fileform': unicode(self.fileform.currentText()), + 'collision': unicode(self.collision.currentText()), + 'updatemeta': unicode(self.updatemeta.isChecked()), + } + +class DisplayStoryListDialog(SizePersistedDialog): + def __init__(self, gui, header, prefs, icon, books, + label_text='', + save_size_name='FanFictionDownLoader plugin:display list dialog'): + SizePersistedDialog.__init__(self, gui, save_size_name) + # UpdateExistingDialog.__init__(self, gui, header, prefs, icon, books, + # save_size_name='FanFictionDownLoader plugin:display list dialog') + + self.gui = gui + + self.setWindowTitle(header) + self.setWindowIcon(icon) + + layout = QVBoxLayout(self) + self.setLayout(layout) + title_layout = ImageTitleLayout(self, 'images/icon.png', + header) + layout.addLayout(title_layout) + + self.books_table = StoryListTableWidget(self) + layout.addWidget(self.books_table) + + options_layout = QHBoxLayout() + self.label = QLabel(label_text) + #self.label.setOpenExternalLinks(True) + #self.label.setWordWrap(True) + options_layout.addWidget(self.label) + + button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) + button_box.accepted.connect(self.accept) + button_box.rejected.connect(self.reject) + + options_layout.addWidget(button_box) + + layout.addLayout(options_layout) + + # Cause our dialog size to be restored from prefs or created on first usage + self.resize_dialog() + self.books_table.populate_table(books) + + def get_books(self): + return self.books_table.get_books() + +class StoryListTableWidget(QTableWidget): + + def __init__(self, parent): + QTableWidget.__init__(self, parent) + self.setSelectionBehavior(QAbstractItemView.SelectRows) + + def populate_table(self, books): + self.clear() + self.setAlternatingRowColors(True) + self.setRowCount(len(books)) + header_labels = ['','Title', 'Author', 'URL', 'Comment'] + self.setColumnCount(len(header_labels)) + self.setHorizontalHeaderLabels(header_labels) + self.horizontalHeader().setStretchLastSection(True) + #self.verticalHeader().setDefaultSectionSize(24) + self.verticalHeader().hide() + + self.books={} + for row, book in enumerate(books): + self.populate_table_row(row, book) + self.books[row] = book + + # turning True breaks up/down. Do we need either sorting or up/down? + self.setSortingEnabled(True) + self.resizeColumnsToContents() + self.setMinimumColumnWidth(1, 100) + self.setMinimumColumnWidth(2, 100) + self.setMinimumColumnWidth(3, 100) + self.setMinimumSize(300, 0) + # if len(books) > 0: + # self.selectRow(0) + self.sortItems(1) + self.sortItems(0) + + def setMinimumColumnWidth(self, col, minimum): + if self.columnWidth(col) < minimum: + self.setColumnWidth(col, minimum) + + def populate_table_row(self, row, book): + if book['good']: + icon = get_icon('ok.png') + val = 0 + else: + icon = get_icon('minus.png') + val = 1 + if 'icon' in book: + icon = get_icon(book['icon']) + + status_cell = IconWidgetItem(None,icon,val) + status_cell.setData(Qt.UserRole, QVariant(val)) + self.setItem(row, 0, status_cell) + + title_cell = ReadOnlyTableWidgetItem(book['title']) + title_cell.setData(Qt.UserRole, QVariant(row)) + self.setItem(row, 1, title_cell) + + self.setItem(row, 2, AuthorTableWidgetItem(book['author'], book['author_sort'])) + + url_cell = ReadOnlyTableWidgetItem(book['url']) + #url_cell.setData(Qt.UserRole, QVariant(book['url'])) + self.setItem(row, 3, url_cell) + + comment_cell = ReadOnlyTableWidgetItem(book['comment']) + #comment_cell.setData(Qt.UserRole, QVariant(book)) + self.setItem(row, 4, comment_cell) + + # def get_calibre_ids(self): + # ids = [] + # for row in range(self.rowCount()): + # ids.append(self.item(row, 1).data(Qt.UserRole).toPyObject()) + # return ids + + # def get_urls(self): + # urls = [] + # for row in range(self.rowCount()): + # urls.append(self.item(row, 2).data(Qt.UserRole).toPyObject()) + # return urls + + def get_books(self): + books = [] + #print("=========================\nbooks:%s"%self.books) + for row in range(self.rowCount()): + rnum = self.item(row, 1).data(Qt.UserRole).toPyObject() + #print("get_books rnum:%s"%rnum) + book = self.books[rnum] + #if book['good']: + books.append(book) + return books + + def remove_selected_rows(self): + self.setFocus() + rows = self.selectionModel().selectedRows() + if len(rows) == 0: + return + message = '

Are you sure you want to remove this book from the list?' + if len(rows) > 1: + message = '

Are you sure you want to remove the selected %d books from the list?'%len(rows) + if not confirm(message,'fanfictiondownloader_delete_item', self): + return + first_sel_row = self.currentRow() + for selrow in reversed(rows): + self.removeRow(selrow.row()) + if first_sel_row < self.rowCount(): + self.select_and_scroll_to_row(first_sel_row) + elif self.rowCount() > 0: + self.select_and_scroll_to_row(first_sel_row - 1) + + def select_and_scroll_to_row(self, row): + self.selectRow(row) + self.scrollToItem(self.currentItem()) + + def move_rows_up(self): + self.setFocus() + rows = self.selectionModel().selectedRows() + if len(rows) == 0: + return + first_sel_row = rows[0].row() + if first_sel_row <= 0: + return + # Workaround for strange selection bug in Qt which "alters" the selection + # in certain circumstances which meant move down only worked properly "once" + selrows = [] + for row in rows: + selrows.append(row.row()) + selrows.sort() + for selrow in selrows: + self.swap_row_widgets(selrow - 1, selrow + 1) + scroll_to_row = first_sel_row - 1 + if scroll_to_row > 0: + scroll_to_row = scroll_to_row - 1 + self.scrollToItem(self.item(scroll_to_row, 0)) + + def move_rows_down(self): + self.setFocus() + rows = self.selectionModel().selectedRows() + if len(rows) == 0: + return + last_sel_row = rows[-1].row() + if last_sel_row == self.rowCount() - 1: + return + # Workaround for strange selection bug in Qt which "alters" the selection + # in certain circumstances which meant move down only worked properly "once" + selrows = [] + for row in rows: + selrows.append(row.row()) + selrows.sort() + for selrow in reversed(selrows): + self.swap_row_widgets(selrow + 2, selrow) + scroll_to_row = last_sel_row + 1 + if scroll_to_row < self.rowCount() - 1: + scroll_to_row = scroll_to_row + 1 + self.scrollToItem(self.item(scroll_to_row, 0)) + + def swap_row_widgets(self, src_row, dest_row): + self.blockSignals(True) + self.insertRow(dest_row) + for col in range(0, self.columnCount()): + self.setItem(dest_row, col, self.takeItem(src_row, col)) + self.removeRow(src_row) + self.blockSignals(False) diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py index 4adbea7..b6b1f65 100644 --- a/calibre-plugin/ffdl_plugin.py +++ b/calibre-plugin/ffdl_plugin.py @@ -4,31 +4,37 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) __license__ = 'GPL v3' -__copyright__ = '2011, Jim Miller' +__copyright__ = '2012, Jim Miller' __docformat__ = 'restructuredtext en' -import ConfigParser, os +import time, os +from ConfigParser import SafeConfigParser from StringIO import StringIO from functools import partial from datetime import datetime -from PyQt4.Qt import (QApplication) +from PyQt4.Qt import (QApplication, QMenu, QToolButton) -# The class that all interface action plugins must inherit from -from calibre.ptempfile import PersistentTemporaryFile -from calibre.ebooks.metadata import MetaInformation +from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir +from calibre.ebooks.metadata import MetaInformation, authors_to_string from calibre.ebooks.metadata.meta import get_metadata from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog +# The class that all interface action plugins must inherit from from calibre.gui2.actions import InterfaceAction from calibre.gui2.threaded_jobs import ThreadedJob +from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin_icon_resources, get_icon, + create_menu_action_unique, get_library_uuid) + from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge -from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, CLIP, SELECTED) +from calibre_plugins.fanfictiondownloader_plugin.config import (prefs) from calibre_plugins.fanfictiondownloader_plugin.dialogs import ( - DownloadDialog, MetadataProgressDialog, UserPassDialog, - OVERWRITE, UPDATE, ADDNEW, SKIP, CALIBREONLY, NotGoingToDownload ) + AddNewDialog, UpdateExistingDialog, DisplayStoryListDialog, + MetadataProgressDialog, UserPassDialog, AboutDialog, + OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, + NotGoingToDownload ) # because calibre immediately transforms html into zip and don't want # to have an 'if html'. db.has_format is cool with the case mismatch, @@ -40,6 +46,8 @@ formmapping = { 'txt':'TXT' } +PLUGIN_ICONS = ['images/icon.png'] + class FanFictionDownLoaderPlugin(InterfaceAction): name = 'FanFictionDownLoader' @@ -50,172 +58,236 @@ class FanFictionDownLoaderPlugin(InterfaceAction): # keyboard shortcuts, so try to use an unusual/unused shortcut. # (text, icon_path, tooltip, keyboard shortcut) # icon_path isn't in the zip--icon loaded below. - action_spec = ('FanFictionDownLoader', None, + action_spec = (name, None, 'Download FanFiction stories from various web sites', None) action_type = 'global' - + # make button menu drop down only + #popup_type = QToolButton.InstantPopup + def genesis(self): # This method is called once per plugin, do initial setup here + # Read the plugin icons and store for potential sharing with the config widget + icon_resources = self.load_resources(PLUGIN_ICONS) + set_plugin_icon_resources(self.name, icon_resources) + + # Show the config dialog + # The config dialog can also be shown from within + # Preferences->Plugins, which is why the do_user_config + # method is defined on the base plugin class + do_user_config = self.interface_action_base_plugin.do_user_config + # Set the icon for this interface action # The get_icons function is a builtin function defined for all your # plugin code. It loads icons from the plugin zip file. It returns # QIcon objects, if you want the actual data, use the analogous # get_resources builtin function. - # + # Note that if you are loading more than one icon, for performance, you # should pass a list of names to get_icons. In this case, get_icons # will return a dictionary mapping names to QIcons. Names that # are not found in the zip file will result in null QIcons. - icon = get_icons('images/icon.png') - + icon = get_icon('images/icon.png') + # The qaction is automatically created from the action_spec defined # above self.qaction.setIcon(icon) - # Call function when plugin triggered. - self.qaction.triggered.connect(self.show_dialog) - def show_dialog(self): - # The base plugin object defined in __init__.py - base_plugin_object = self.interface_action_base_plugin + # Call function when plugin triggered. + self.qaction.triggered.connect(self.plugin_button) + + # Assign our menu to this action + self.menu = QMenu(self.gui) + self.qaction.setMenu(self.menu) + + self.menu.aboutToShow.connect(self.about_to_show_menu) + + self.actions_unique_map = {} + + self.add_action = self.create_menu_item_ex(self.menu, '&Add New from URL(s)', image='plus.png', + unique_name='Add New FanFiction Book(s) from URL(s)', + shortcut_name='Add New FanFiction Book(s) from URL(s)', + triggered=self.add_dialog ) + + self.update_action = self.create_menu_item_ex(self.menu, '&Update Existing FanFiction Book(s)', image='plusplus.png', + unique_name='Update Existing FanFiction Book(s)', + shortcut_name='Update Existing FanFiction Book(s)', + triggered=self.update_existing) #partial(self._update_existing,'qwerty')) + + self.menu.addSeparator() + self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False, + image= 'config.png', + unique_name='Configure FanFictionDownLoader', + shortcut_name='Configure FanFictionDownLoader', + triggered=partial(do_user_config,parent=self.gui)) + + self.config_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False, + image= 'images/icon.png', + unique_name='About FanFictionDownLoader', + shortcut_name='About FanFictionDownLoader', + triggered=self.about) + + + def about_to_show_menu(self): + self.update_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 ) + + def about(self): + # Get the about text from a file inside the plugin zip file + # The get_resources function is a builtin function defined for all your + # plugin code. It loads files from the plugin zip file. It returns + # the bytes from the specified file. + # + # Note that if you are loading more than one file, for performance, you + # should pass a list of names to get_resources. In this case, + # get_resources will return a dictionary mapping names to bytes. Names that + # are not found in the zip file will not be in the returned dictionary. + text = get_resources('about.txt') +# QMessageBox.about(self, 'About the FanFictionDownLoader Plugin', + # text.decode('utf-8')) + AboutDialog(self.gui,self.qaction.icon(),text).exec_() + + def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None, + shortcut=None, triggered=None, is_checked=None, shortcut_name=None, + unique_name=None): + ac = create_menu_action_unique(self, parent_menu, menu_text, image, tooltip, + shortcut, triggered, is_checked, shortcut_name, unique_name) + self.actions_unique_map[ac.calibre_shortcut_unique_name] = ac.calibre_shortcut_unique_name + return ac + + def plugin_button(self): + if len(self.gui.library_view.get_selected_ids()) > 0 and prefs['updatedefault']: + self.update_existing() + else: + self.add_dialog() + + def add_dialog(self): + + #print("add_dialog()") + # Show the config dialog # The config dialog can also be shown from within # Preferences->Plugins, which is why the do_user_config # method is defined on the base plugin class - do_user_config = base_plugin_object.do_user_config - - # The current database shown in the GUI - # db is an instance of the class LibraryDatabase2 from database.py - # This class has many, many methods that allow you to do a lot of - # things. - self.db = self.gui.current_db - - # pre-pop urls from selected stories or clipboard. but with - # configurable priority and on/off option on each - if prefs['urlsfrompriority'] == SELECTED: - from_first_func = self.get_urls_select - from_second_func = self.get_urls_clip - elif prefs['urlsfrompriority'] == CLIP: - from_first_func = self.get_urls_clip - from_second_func = self.get_urls_select - - url_list_text = from_first_func() - if not url_list_text: - url_list_text = from_second_func() + do_user_config = self.interface_action_base_plugin.do_user_config + url_list = self.get_urls_clip() + url_list_text = "\n".join(url_list) + # self.gui is the main calibre GUI. It acts as the gateway to access # all the elements of the calibre user interface, it should also be the # parent of the dialog - # DownloadDialog just collects URLs, format and presents buttons. - d = DownloadDialog(self.gui, - prefs, - self.qaction.icon(), - url_list_text, - do_user_config, # method for config button - self.start_downloads, # method to start downloads - ) - d.show() + # AddNewDialog just collects URLs, format and presents buttons. + d = AddNewDialog(self.gui, + prefs, + self.qaction.icon(), + url_list_text, + ) + d.exec_() + if d.result() != d.Accepted: + return + + url_list = get_url_list(d.get_urlstext()) + add_books = self._convert_urls_to_books(url_list) + #print("add_books:%s"%add_books) + #print("options:%s"%d.get_ffdl_options()) - ## if there's rows selected, try to find a source URL from - ## either identifier in the metadata, or from the epub - ## metadata. - def get_urls_select(self): - url_list = [] - rows = self.gui.library_view.selectionModel().selectedRows() - if rows and prefs['urlsfromselected']: - book_ids = self.gui.library_view.get_selected_ids() - #print("book_ids: %s"%book_ids) - for book_id in book_ids: - #print("book_on_device:%s"%self.db.book_on_device(book_id)) - identifiers = self.db.get_identifiers(book_id,index_is_id=True) - if 'url' in identifiers: - # identifiers have :->| in url. - #print("url from book:"+identifiers['url'].replace('|',':')) - url_list.append(identifiers['url'].replace('|',':')) - else: - ## only epub has that in it. - if self.db.has_format(book_id,'EPUB',index_is_id=True): - existingepub = self.db.format(book_id,'EPUB',index_is_id=True, as_file=True) - mi = get_metadata(existingepub,'EPUB') - #print("mi:%s"%mi) - identifiers = mi.get_identifiers() - if 'url' in identifiers: - #print("url from epub:"+identifiers['url'].replace('|',':')) - url_list.append(identifiers['url'].replace('|',':')) - return self.get_valid_urls(url_list) + self.start_downloads( d.get_ffdl_options(),add_books ) + + def update_existing(self): + #print("update_existing()") + previous = self.gui.library_view.currentIndex() + db = self.gui.current_db + book_ids = self.gui.library_view.get_selected_ids() + books = self._convert_calibre_ids_to_books(db, book_ids) + #print("update books:%s"%books) + + d = UpdateExistingDialog(self.gui, + 'Update Existing List', + prefs, + self.qaction.icon(), + books, + ) + d.exec_() + if d.result() != d.Accepted: + return + + update_books = d.get_books() + + #print("update_books:%s"%update_books) + #print("options:%s"%d.get_ffdl_options()) + # only if there's some good ones. + if 0 < len(filter(lambda x : x['good'], update_books)): + self.start_downloads( d.get_ffdl_options(), update_books ) def get_urls_clip(self): url_list = [] if prefs['urlsfromclip']: - # no rows selected, check for valid URLs in the clipboard. - cliptext = unicode(QApplication.instance().clipboard().text()) - url_list.extend(cliptext.split()) - return self.get_valid_urls(url_list) + for url in unicode(QApplication.instance().clipboard().text()).split(): + if( self._is_good_downloader_url(url) ): + url_list.append(url) + return url_list - def get_valid_urls(self,url_list): - url_list_text = "" - - # Check and make sure the URLs are valid ffdl URLs. - if url_list: - # this is the accepted way to 'check for existance'? really? - try: - self.dummyconfig - except AttributeError: - self.dummyconfig = ConfigParser.SafeConfigParser() - - alreadyin=[] - for url in url_list: - if url in alreadyin: - continue - alreadyin.append(url) - # pulling up an adapter is pretty low over-head. If - # it fails, it's a bad url. - try: - adapters.getAdapter(self.dummyconfig,url) - except: - pass - else: - if url_list_text: - url_list_text += "\n" - url_list_text += url - - return url_list_text - def apply_settings(self): # No need to do anything with perfs here, but we could. prefs - def start_downloads(self,urls,fileform, - collision,updatemeta,onlyoverwriteifnewer): + def start_downloads(self, options, books): - url_list = get_url_list(urls) + #print("start_downloads:%s"%books) - self.fetchmeta_qpd = \ - MetadataProgressDialog(self.gui, - url_list, - fileform, - partial(self.get_adapter_for_story, collision=collision,onlyoverwriteifnewer=onlyoverwriteifnewer), - partial(self.download_list,collision=collision,updatemeta=updatemeta,onlyoverwriteifnewer=onlyoverwriteifnewer), - self.db) - - def get_adapter_for_story(self,url,fileform, - collision=SKIP, - onlyoverwriteifnewer=False): + # create and pass temp dir. + tdir = PersistentTemporaryDirectory(prefix='fanfictiondownloader_') + options['tdir']=tdir + + self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000) + + MetadataProgressDialog(self.gui, + books, + options, + partial(self.get_metadata_for_book, options = options), + partial(self.start_download_list, options = options)) + # MetadataProgressDialog calls get_adapter_for_story for each 'good' story, + # get_adapter_for_story updates book for each, + # MetadataProgressDialog calls download_list at the end which goes into the BG. + # Not sure yet how I get back to display the 'do updates?' list. + + def get_metadata_for_book(self,book, + options={'fileform':'epub', + 'collision':ADDNEW, + 'updatemeta':True}): ''' - Returns adapter object for story at URL. To be called from - MetadataProgressDialog 'loop' to build up list of adapters. Also - pops dialogs for is adult, user/pass, duplicate + Update passed in book dict with metadata from website and + necessary data. To be called from MetadataProgressDialog + 'loop'. Also pops dialogs for is adult, user/pass. ''' - print("URL:"+url) + # The current database shown in the GUI + # db is an instance of the class LibraryDatabase2 from database.py + # This class has many, many methods that allow you to do a lot of + # things. + db = self.gui.current_db + + fileform = options['fileform'] + collision = options['collision'] + updatemeta= options['updatemeta'] + + if not book['good']: + # book has already been flagged bad for whatever reason. + return + + url = book['url'] + ## was self.ffdlconfig, but we need to be able to change it ## when doing epub update. - ffdlconfig = ConfigParser.SafeConfigParser() + ffdlconfig = SafeConfigParser() ffdlconfig.readfp(StringIO(get_resources("defaults.ini"))) ffdlconfig.readfp(StringIO(prefs['personal.ini'])) adapter = adapters.getAdapter(ffdlconfig,url) + #options['defaults.ini'] = get_resources("defaults.ini") + options['personal.ini'] = prefs['personal.ini'] + try: adapter.getStoryMetadataOnly() except exceptions.FailedToLogin: @@ -225,240 +297,612 @@ class FanFictionDownLoaderPlugin(InterfaceAction): if userpass.status: adapter.username = userpass.user.text() adapter.password = userpass.passwd.text() - # else: - # del adapter - # return + except exceptions.AdultCheckRequired: if question_dialog(self.gui, 'Are You Adult?', '

'+ "%s requires that you be an adult. Please confirm you are an adult in your locale:"%url, show_copy_button=False): adapter.is_adult=True - # else: - # del adapter - # return - # let exceptions percolate up. + # let other exceptions percolate up. story = adapter.getStoryMetadataOnly() + writer = writers.getWriter(options['fileform'],adapter.config,adapter) - if collision != ADDNEW: - mi = MetaInformation(story.getMetadata("title"), - (story.getMetadata("author"),)) # author is a list. + book['title'] = story.getMetadata("title", removeallentities=True) + book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True) + book['publisher'] = story.getMetadata("site") + book['tags'] = writer.getTags() + book['pubdate'] = story.getMetadataRaw('datePublished') + book['timestamp'] = story.getMetadataRaw('dateCreated') + book['comments'] = story.getMetadata("description") #, removeallentities=True) comments handles entities better. + + # adapter.opener is the element with a threadlock. But del + # adapter.opener doesn't work--subproc fails when it tries + # to pull in the adapter object that hasn't been imported yet. + # book['adapter'] = adapter + + book['is_adult'] = adapter.is_adult + book['username'] = adapter.username + book['password'] = adapter.password - identicalbooks = self.db.find_identical_books(mi) - print(identicalbooks) - ## more than one match will need to be handled differently. - if identicalbooks: + book['icon'] = 'plus.png' + + if collision in (CALIBREONLY): + book['icon'] = 'metadata.png' + + # XXX should really do a 'you can't do that' dialog when the + # hit 'OK' for this case. + if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub': + raise NotGoingToDownload("Cannot update non-epub format.") + + book_id = None + + if book['calibre_id'] != None: + # updating an existing book. Update mode applies. + print("update id:%s"%book['calibre_id']) + book_id = book['calibre_id'] + # No handling needed: OVERWRITEALWAYS,CALIBREONLY + + # only care about collisions when not ADDNEW + elif collision != ADDNEW: + # 'new' book from URL. collision handling applies. + print("from URL") + + # find dups + mi = MetaInformation(story.getMetadata("title", removeallentities=True), + (story.getMetadata("author", removeallentities=True),)) # author is a list. + identicalbooks = db.find_identical_books(mi) + ## removed for being overkill. + # for ib in db.find_identical_books(mi): + # # only *really* identical if URL matches, too. + # # XXX make an option? + # if self._get_story_url(db,ib) == url: + # identicalbooks.append(ib) + print("identicalbooks:%s"%identicalbooks) + + if collision == SKIP and identicalbooks: + raise NotGoingToDownload("Skipping duplicate story.","list_remove.png") + + if len(identicalbooks) > 1: + raise NotGoingToDownload("More than one identical book--can't tell which to update/overwrite.","minusminus.png") + + if collision == CALIBREONLY and not identicalbooks: + raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png") + + if len(identicalbooks)>0: book_id = identicalbooks.pop() - if collision == SKIP: - raise NotGoingToDownload("Skipping duplicate story.") + book['calibre_id'] = book_id + book['icon'] = 'edit-redo.png' - # print("Attempting to add to list.") - # rl_plugin = self.gui.iactions['Reading List'] - # rl_plugin.remove_books_from_list('Send',[book_id],refresh_screen=False) - # rl_plugin.add_books_to_list('Send',[book_id],refresh_screen=False) - #rl_plugin.view_list('Send') - - if collision == OVERWRITE and len(identicalbooks) > 1: - raise NotGoingToDownload("More than one identical books--can't tell which to overwrite.") - - if collision == OVERWRITE and \ - onlyoverwriteifnewer and \ - self.db.has_format(book_id,fileform,index_is_id=True): - # check make sure incoming is newer. - lastupdated=story.getMetadataRaw('dateUpdated').date() - fileupdated=datetime.fromtimestamp(os.stat(self.db.format_abspath(book_id, fileform, index_is_id=True))[8]).date() - if fileupdated > lastupdated: - raise NotGoingToDownload("Not Overwriting, story is not newer.") - - if collision == UPDATE: - if fileform != 'epub': - raise NotGoingToDownload("Not updating non-epub format.") - # 'book' can exist without epub. If there's no existing epub, - # let it go and it will download it. - if self.db.has_format(book_id,fileform,index_is_id=True): - toupdateio = StringIO() - (epuburl,chaptercount) = doMerge(toupdateio, - [StringIO(self.db.format(book_id,'EPUB', - index_is_id=True))], - titlenavpoints=False, - striptitletoc=True, - forceunique=False) - - urlchaptercount = int(story.getMetadata('numChapters')) - if chaptercount == urlchaptercount: # and not onlyoverwriteifnewer: - raise NotGoingToDownload("%s already contains %d chapters." % (url,chaptercount)) - elif chaptercount > urlchaptercount: - raise NotGoingToDownload("%s contains %d chapters, more than epub." % (url,chaptercount)) - else: - print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)) - - else: # not identicalbooks - if collision == CALIBREONLY: - raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.") - - return adapter - - def download_list(self,adaptertuple_list,fileform, - collision=ADDNEW, - updatemeta=True, - onlyoverwriteifnewer=True): - ''' - Called by MetadataProgressDialog to start story downloads BG processing. - adapter_list is a list of tuples of (url,adapter) - ''' - print("download_list") - - job = ThreadedJob('FanFictionDownload', - 'Downloading FanFiction Stories', - func=self.do_story_downloads, - args=(adaptertuple_list, fileform, self.db), - kwargs={'collision':collision,'updatemeta':updatemeta, - 'onlyoverwriteifnewer':onlyoverwriteifnewer}, - callback=self._get_stories_completed) - - - self.gui.job_manager.run_threaded_job(job) - - self.gui.status_bar.show_message('Downloading %d stories'%len(adaptertuple_list)) - - def _get_stories_completed(self, job): - print("_get_stories_completed") - - def do_story_downloads(self, adaptertuple_list, fileform, db, - **kwargs): - ''' - Master job, loop to download this list of stories - ''' - print("do_story_downloads") - abort = kwargs['abort'] - notifications=kwargs['notifications'] - log = kwargs['log'] - notifications.put((0.01, 'Start Downloading Stories')) - count = 0.01 - total = len(adaptertuple_list) - # Queue all the jobs - for (url,adapter) in adaptertuple_list: - if abort.is_set(): - notifications.put(1.0,'Aborting...') + if book_id != None and collision != ADDNEW: + if options['collision'] in (CALIBREONLY): + book['comment'] = 'Metadata collected.' + # don't need temp file created below. return - notifications.put((float(count)/total, - 'Downloading %s'%adapter.getStoryMetadataOnly().getMetadata("title"))) - log.prints(log.INFO,'Downloading %s'%adapter.getStoryMetadataOnly().getMetadata("title")) - try: - self.do_story_download(adapter,fileform,db, - kwargs['collision'],kwargs['updatemeta'],kwargs['onlyoverwriteifnewer']) - except Exception as e: - log.prints(log.ERROR,'Failed Downloading %s: %s'% - (adapter.getStoryMetadataOnly().getMetadata("title"),e)) - - count = count + 1 - return - - def do_story_download(self,adapter,fileform,db,collision, - updatemeta,onlyoverwriteifnewer): - print("do_story_download") - - story = adapter.getStoryMetadataOnly() - - mi = MetaInformation(story.getMetadata("title"), - (story.getMetadata("author"),)) # author is a list. - - writer = writers.getWriter(fileform,adapter.config,adapter) - tmp = PersistentTemporaryFile("."+fileform) - titleauth = "%s by %s"%(story.getMetadata("title"), story.getMetadata("author")) - url = story.getMetadata("storyUrl") - print(titleauth) - print("tmp: "+tmp.name) - - mi.set_identifiers({'url':story.getMetadata("storyUrl")}) - mi.publisher = story.getMetadata("site") - mi.tags = writer.getTags() - mi.languages = ['en'] - mi.pubdate = story.getMetadataRaw('datePublished') - mi.timestamp = story.getMetadataRaw('dateCreated') - mi.comments = story.getMetadata("description") - - identicalbooks = self.db.find_identical_books(mi) - #print(identicalbooks) - addedcount=0 - if identicalbooks and collision != ADDNEW: - ## more than one match? add to first off the list. - ## Shouldn't happen--we checked above. - book_id = identicalbooks.pop() - + + ## newer/chaptercount checks are the same for both: + # Update epub, but only if more chapters. if collision == UPDATE: - if self.db.has_format(book_id,fileform,index_is_id=True): - urlchaptercount = int(story.getMetadata('numChapters')) - ## First, get existing epub with titlepage and tocpage stripped. - updateio = StringIO() - (epuburl,chaptercount) = doMerge(updateio, - [StringIO(self.db.format(book_id,'EPUB', + # 'book' can exist without epub. If there's no existing epub, + # let it go and it will download it. + if db.has_format(book_id,fileform,index_is_id=True): + toupdateio = StringIO() + (epuburl,chaptercount) = doMerge(toupdateio, + [StringIO(db.format(book_id,'EPUB', index_is_id=True))], titlenavpoints=False, striptitletoc=True, forceunique=False) - print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)) - - ## Get updated title page/metadata by itself in an epub. - ## Even if the title page isn't included, this carries the metadata. - titleio = StringIO() - writer.writeStory(outstream=titleio,metaonly=True) - - newchaptersio = None - if urlchaptercount > chaptercount : - ## Go get the new chapters only in another epub. - newchaptersio = StringIO() - adapter.setChaptersRange(chaptercount+1,urlchaptercount) - - adapter.config.set("overrides",'include_tocpage','false') - adapter.config.set("overrides",'include_titlepage','false') - writer.writeStory(outstream=newchaptersio) - - ## Merge the three epubs together. - doMerge(tmp, - [titleio,updateio,newchaptersio], - fromfirst=True, - titlenavpoints=False, - striptitletoc=False, - forceunique=False) - - - else: # update, but there's no epub extant, so do overwrite. - collision = OVERWRITE - - if collision == OVERWRITE: - writer.writeStory(tmp) - - db.add_format_with_hooks(book_id, fileform, tmp, index_is_id=True) - - # get all formats. - if prefs['deleteotherforms'] and collision in (OVERWRITE, UPDATE): - fmts = set([x.lower() for x in db.formats(book_id, index_is_id=True).split(',')]) - for fmt in fmts: - if fmt != fileform: - print("remove f:"+fmt) - db.remove_format(book_id, fmt,index_is_id=True)#, notify=False - - if updatemeta or collision == CALIBREONLY: - db.set_metadata(book_id,mi) - - else: # no matching, adding new. - writer.writeStory(tmp) - (notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True) - - - # Otherwise list of books doesn't update right away. - if addedcount: - self.gui.library_view.model().books_added(addedcount) - - self.gui.library_view.model().refresh() - del adapter - del writer - -def f(x): - if x.strip(): return True - else: return False + urlchaptercount = int(story.getMetadata('numChapters')) + if chaptercount == urlchaptercount: # and not onlyoverwriteifnewer: + raise NotGoingToDownload("Already contains %d chapters."%chaptercount,'edit-undo.png') + elif chaptercount > urlchaptercount: + raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d." % (chaptercount,urlchaptercount),'dialog_error.png') + if collision == OVERWRITE and \ + db.has_format(book_id,formmapping[fileform],index_is_id=True): + # check make sure incoming is newer. + lastupdated=story.getMetadataRaw('dateUpdated').date() + fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8]).date() + if fileupdated > lastupdated: + raise NotGoingToDownload("Not Overwriting, web site is not newer.",'edit-undo.png') + + # For update, provide a tmp file copy of the existing epub so + # it can't change underneath us. + if collision in (UPDATE,UPDATEALWAYS) and \ + db.has_format(book['calibre_id'],'EPUB',index_is_id=True): + tmp = PersistentTemporaryFile(prefix='old-%s-'%book['calibre_id'], + suffix='.epub', + dir=options['tdir']) + db.copy_format_to(book_id,fileform,tmp,index_is_id=True) + print("existing epub tmp:"+tmp.name) + book['epub_for_update'] = tmp.name + + if book['good']: # there shouldn't be any !'good' books at this point. + # if still 'good', make a temp file to write the output to. + tmp = PersistentTemporaryFile(prefix='new-%s-'%book['calibre_id'], + suffix='.'+options['fileform'], + dir=options['tdir']) + print("outfile:"+tmp.name) + book['outfile'] = tmp.name + + return + + # if collision != ADDNEW: + # mi = MetaInformation(story.getMetadata("title"), + # (story.getMetadata("author"),)) # author is a list. + + # identicalbooks = self.db.find_identical_books(mi) + # ## more than one match will need to be handled differently. + # if identicalbooks: + # book_id = identicalbooks.pop() + # if collision == SKIP: + # raise NotGoingToDownload("Skipping duplicate story.") + + # if collision == OVERWRITE and len(identicalbooks) > 1: + # raise NotGoingToDownload("More than one identical books--can't tell which to overwrite.") + + # if collision == OVERWRITE and \ + # onlyoverwriteifnewer and \ + # self.db.has_format(book_id,fileform,index_is_id=True): + # # check make sure incoming is newer. + # lastupdated=story.getMetadataRaw('dateUpdated').date() + # fileupdated=datetime.fromtimestamp(os.stat(self.db.format_abspath(book_id, fileform, index_is_id=True))[8]).date() + # if fileupdated > lastupdated: + # raise NotGoingToDownload("Not Overwriting, story is not newer.") + + # if collision == UPDATE: + # if fileform != 'epub': + # raise NotGoingToDownload("Not updating non-epub format.") + # # 'book' can exist without epub. If there's no existing epub, + # # let it go and it will download it. + # if self.db.has_format(book_id,fileform,index_is_id=True): + # toupdateio = StringIO() + # (epuburl,chaptercount) = doMerge(toupdateio, + # [StringIO(self.db.format(book_id,'EPUB', + # index_is_id=True))], + # titlenavpoints=False, + # striptitletoc=True, + # forceunique=False) + + # urlchaptercount = int(story.getMetadata('numChapters')) + # if chaptercount == urlchaptercount: # and not onlyoverwriteifnewer: + # raise NotGoingToDownload("%s already contains %d chapters." % (url,chaptercount)) + # elif chaptercount > urlchaptercount: + # raise NotGoingToDownload("%s contains %d chapters, more than epub." % (url,chaptercount)) + # else: + # print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)) + + # else: # not identicalbooks + # if collision == CALIBREONLY: + # raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.") + + # return adapter + + def start_download_list(self,book_list, + options={'fileform':'epub', + 'collision':ADDNEW, + 'updatemeta':True}): + ''' + Called by MetadataProgressDialog to start story downloads BG processing. + adapter_list is a list of tuples of (url,adapter) + ''' + #print("start_download_list:book_list:%s"%book_list) + + ## No need to BG process when CALIBREONLY! Fake it. + if options['collision'] in (CALIBREONLY): + class NotJob(object): + def __init__(self,result): + self.failed=False + self.result=result + notjob = NotJob(book_list) + self.download_list_completed(notjob,options=options) + return + + for book in book_list: + if book['good']: + break + else: + ## No good stories to try to download, go straight to + ## list. + d = DisplayStoryListDialog(self.gui, + 'Nothing to Download', + prefs, + self.qaction.icon(), + book_list, + label_text='None of the URLs/stories given can be/need to be downloaded.' + ) + d.exec_() + return + + func = 'arbitrary_n' + cpus = self.gui.job_manager.server.pool_size + #print("cpus:%s <<<======================================"%cpus) + # XXX - option for parallel downloads? + args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', 'do_download_worker', + (book_list, options, cpus)] + desc = 'Download FanFiction Book' + job = self.gui.job_manager.run_job( + self.Dispatcher(partial(self.download_list_completed,options=options)), + func, args=args, + description=desc) + + self.gui.status_bar.show_message('Starting %d FanFictionDownLoads'%len(book_list),3000) + + def download_list_completed(self, job, options={}): + if job.failed: + self.gui.job_exception(job, dialog_title='Failed to Download Stories') + return + #print("download_list_completed job:%s"%job.result) + # for b in job.result: + # print("job.result: %s"%b['title']) + + db = self.gui.current_db + + d = DisplayStoryListDialog(self.gui, + 'Downloads finished, confirm to update Calibre', + prefs, + self.qaction.icon(), + job.result, + label_text='Stories will not be added or updated in Calibre without confirmation.' + ) + d.exec_() + if d.result() == d.Accepted: + + ## in case the user removed any from the list. + book_list = d.get_books() + # for b in book_list: + # print("d list: %s"%b['title']) + + update_list = filter(lambda x : x['good'] and x['calibre_id'] != None, + book_list) + + add_list = filter(lambda x : x['good'] and x['calibre_id'] == None, + book_list) + + total_good = len(update_list)+len(add_list) + + self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good), 3000) + + print("==================================================") + addfiles,addfileforms,addmis=[],[],[] + + added=0 + for book in add_list: + print("adding %s %s"%(book['title'],book['url'])) + addfiles.append(book['outfile']) + addfileforms.append(options['fileform']) + addmis.append(self._make_mi_from_book(book)) + + (notadded,added)=db.add_books(addfiles,addfileforms,addmis, + add_duplicates=True) + + print("==================================================") + updated=0 + for book in update_list: + print("updating (%s)%s %s"%(book['calibre_id'],book['title'],book['url'])) + + if options['collision'] == CALIBREONLY: + updated += 1 + else: + if not db.add_format_with_hooks(book['calibre_id'], options['fileform'], book['outfile'], index_is_id=True): + book['comment'] = "Adding format to book failed for some reason..." + book['good']=False + book['icon']='dialog_error.png' + else: + updated += 1 + # get all formats. + if prefs['deleteotherforms']: + fmts = set([x.lower() for x in db.formats(book['calibre_id'], index_is_id=True).split(',')]) + for fmt in fmts: + if fmt != options['fileform']: + print("remove f:"+fmt) + db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False + + if options['collision'] == CALIBREONLY or \ + (options['updatemeta'] or book['good']) : + db.set_metadata(book['calibre_id'], + self._make_mi_from_book(book)) + + if added: + self.gui.library_view.model().books_added(added) + + # the refresh causes problems sometimes? Switching libraries + # and back cleared it? + self.gui.library_view.model().refresh() + + self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(updated + added)), 3000) + + if updated + added != total_good: + d = DisplayStoryListDialog(self.gui, + 'Updates completed, final status', + prefs, + self.qaction.icon(), + book_list, + label_text='Stories have be added or updated in Calibre, some had additional problems.' + ) + d.exec_() + + print("all done, remove temp dir.") + remove_dir(options['tdir']) + + + def _make_mi_from_book(self,book): + mi = MetaInformation(book['title'],(book['author'],)) # author is a list. + mi.set_identifiers({'url':book['url']}) + mi.publisher = book['publisher'] + mi.tags = book['tags'] + #mi.languages = ['en'] + mi.pubdate = book['pubdate'] + mi.timestamp = book['timestamp'] + mi.comments = book['comments'] + return mi + + + def _convert_urls_to_books(self, urls): + books = [] + for url in urls: + book = {} + book['good'] = True + book['calibre_id'] = None + book['title'] = 'Unknown' + book['author'] = 'Unknown' + book['author_sort'] = 'Unknown' + + book['comment'] = '' + book['url'] = '' + + self._set_book_url_and_comment(book,url) + + books.append(book) + return books + + def _convert_calibre_ids_to_books(self, db, ids): + books = [] + for book_id in ids: + mi = db.get_metadata(book_id, index_is_id=True) + book = {} + book['good'] = True + book['calibre_id'] = mi.id + book['title'] = mi.title + book['author'] = authors_to_string(mi.authors) + book['author_sort'] = mi.author_sort + # book['series'] = mi.series + # if mi.series: + # book['series_index'] = mi.series_index + # else: + # book['series_index'] = 0 + + book['comment'] = '' + book['url'] = "" + + url = self._get_story_url(db,book_id) + self._set_book_url_and_comment(book,url) + + books.append(book) + return books + + def _set_book_url_and_comment(self,book,url): + if not url: + book['comment'] = "No story URL found." + book['good'] = False + book['icon'] = 'search_delete_saved.png' + else: + # get normalized url or None. + book['url'] = self._is_good_downloader_url(url) + if book['url'] == None: + book['url'] = url + book['comment'] = "URL is not a valid story URL." + book['good'] = False + book['icon']='dialog_error.png' + + def _get_story_url(self, db, book_id): + identifiers = db.get_identifiers(book_id,index_is_id=True) + if 'url' in identifiers: + # identifiers have :->| in url. + #print("url from book:"+identifiers['url'].replace('|',':')) + return identifiers['url'].replace('|',':') + else: + ## only epub has URL in it. + if db.has_format(book_id,'EPUB',index_is_id=True): + existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True) + mi = get_metadata(existingepub,'EPUB') + #print("mi:%s"%mi) + identifiers = mi.get_identifiers() + if 'url' in identifiers: + #print("url from epub:"+identifiers['url'].replace('|',':')) + return identifiers['url'].replace('|',':') + return None + + def _is_good_downloader_url(self,url): + # this is the accepted way to 'check for existance'? really? + try: + self.dummyconfig + except AttributeError: + self.dummyconfig = SafeConfigParser() + # pulling up an adapter is pretty low over-head. If + # it fails, it's a bad url. + try: + adapter = adapters.getAdapter(self.dummyconfig,url) + url = adapter.url + del adapter + return url + except: + return None; + + + +def get_job_details(job): + ''' + Convert the job result into a set of parameters including a detail message + summarising the success of the extraction operation. + This is used by both the threaded and worker approaches to extraction + ''' + extracted_ids, same_isbn_ids, failed_ids, no_format_ids = job.result + if not hasattr(job, 'html_details'): + job.html_details = job.details + det_msg = [] + for i, title in failed_ids: + if i in no_format_ids: + msg = title + ' (No formats)' + else: + msg = title + ' (ISBN not found)' + det_msg.append(msg) + if same_isbn_ids: + if det_msg: + det_msg.append('----------------------------------') + for i, title in same_isbn_ids: + msg = title + ' (Same ISBN)' + det_msg.append(msg) + if len(extracted_ids) > 0: + if det_msg: + det_msg.append('----------------------------------') + for i, title, last_modified, isbn in extracted_ids: + msg = '%s (Extracted %s)'%(title, isbn) + det_msg.append(msg) + + det_msg = '\n'.join(det_msg) + return extracted_ids, same_isbn_ids, failed_ids, det_msg + def get_url_list(urls): - return filter(f,urls.strip().splitlines()) + def f(x): + if x.strip(): return True + else: return False + # set removes dups. + return set(filter(f,urls.strip().splitlines())) + +############################################################################3 + + # def do_story_downloads(self, adaptertuple_list, fileform, db, + # **kwargs): + # ''' + # Master job, loop to download this list of stories + # ''' + # print("do_story_downloads") + # abort = kwargs['abort'] + # notifications=kwargs['notifications'] + # log = kwargs['log'] + # notifications.put((0.01, 'Start Downloading Stories')) + # count = 0.01 + # total = len(adaptertuple_list) + # # Queue all the jobs + # for (url,adapter) in adaptertuple_list: + # if abort.is_set(): + # notifications.put(1.0,'Aborting...') + # return + # notifications.put((float(count)/total, + # 'Downloading %s'%adapter.getStoryMetadataOnly().getMetadata("title"))) + # log.prints(log.INFO,'Downloading %s'%adapter.getStoryMetadataOnly().getMetadata("title")) + # try: + # self.do_story_download(adapter,fileform,db, + # kwargs['collision'],kwargs['updatemeta'],kwargs['onlyoverwriteifnewer']) + # except Exception as e: + # log.prints(log.ERROR,'Failed Downloading %s: %s'% + # (adapter.getStoryMetadataOnly().getMetadata("title"),e)) + + # count = count + 1 + # return + + # def do_story_download(self,adapter,fileform,db,collision, + # updatemeta,onlyoverwriteifnewer): + # print("do_story_download") + + # story = adapter.getStoryMetadataOnly() + + # mi = MetaInformation(story.getMetadata("title"), + # (story.getMetadata("author"),)) # author is a list. + + # writer = writers.getWriter(fileform,adapter.config,adapter) + # tmp = PersistentTemporaryFile("."+fileform) + # titleauth = "%s by %s"%(story.getMetadata("title"), story.getMetadata("author")) + # url = story.getMetadata("storyUrl") + # print(titleauth) + # print("tmp: "+tmp.name) + + # mi.set_identifiers({'url':story.getMetadata("storyUrl")}) + # mi.publisher = story.getMetadata("site") + # mi.tags = writer.getTags() + # mi.languages = ['en'] + # mi.pubdate = story.getMetadataRaw('datePublished') + # mi.timestamp = story.getMetadataRaw('dateCreated') + # mi.comments = story.getMetadata("description") + + # identicalbooks = self.db.find_identical_books(mi) + # #print(identicalbooks) + # added=0 + # if identicalbooks and collision != ADDNEW: + # ## more than one match? add to first off the list. + # ## Shouldn't happen--we checked above. + # book_id = identicalbooks.pop() + + # if collision == UPDATE: + # if self.db.has_format(book_id,fileform,index_is_id=True): + # urlchaptercount = int(story.getMetadata('numChapters')) + # ## First, get existing epub with titlepage and tocpage stripped. + # updateio = StringIO() + # (epuburl,chaptercount) = doMerge(updateio, + # [StringIO(self.db.format(book_id,'EPUB', + # index_is_id=True))], + # titlenavpoints=False, + # striptitletoc=True, + # forceunique=False) + # print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)) + + # ## Get updated title page/metadata by itself in an epub. + # ## Even if the title page isn't included, this carries the metadata. + # titleio = StringIO() + # writer.writeStory(outstream=titleio,metaonly=True) + + # newchaptersio = None + # if urlchaptercount > chaptercount : + # ## Go get the new chapters only in another epub. + # newchaptersio = StringIO() + # adapter.setChaptersRange(chaptercount+1,urlchaptercount) + + # adapter.config.set("overrides",'include_tocpage','false') + # adapter.config.set("overrides",'include_titlepage','false') + # writer.writeStory(outstream=newchaptersio) + + # ## Merge the three epubs together. + # doMerge(tmp, + # [titleio,updateio,newchaptersio], + # fromfirst=True, + # titlenavpoints=False, + # striptitletoc=False, + # forceunique=False) + + + # else: # update, but there's no epub extant, so do overwrite. + # collision = OVERWRITE + + # if collision == OVERWRITE: + # writer.writeStory(tmp) + + # db.add_format_with_hooks(book_id, fileform, tmp, index_is_id=True) + + # # get all formats. + # if prefs['deleteotherforms'] and collision in (OVERWRITE, UPDATE): + # fmts = set([x.lower() for x in db.formats(book_id, index_is_id=True).split(',')]) + # for fmt in fmts: + # if fmt != fileform: + # print("remove f:"+fmt) + # db.remove_format(book_id, fmt,index_is_id=True)#, notify=False + + # if updatemeta or collision == CALIBREONLY: + # db.set_metadata(book_id,mi) + + # else: # no matching, adding new. + # writer.writeStory(tmp) + # (notadded,added)=db.add_books([tmp],[fileform],[mi], add_duplicates=True) + + # # Otherwise list of books doesn't update right away. + # if added: + # self.gui.library_view.model().books_added(added) + + # # the refresh causes problems sometimes? Switching libraries + # # and back cleared it? + # self.gui.library_view.model().refresh() + + # del adapter + # del writer diff --git a/calibre-plugin/images/icon.png b/calibre-plugin/images/icon.png index 75b2806..e971530 100644 Binary files a/calibre-plugin/images/icon.png and b/calibre-plugin/images/icon.png differ diff --git a/calibre-plugin/images/icon.xcf b/calibre-plugin/images/icon.xcf index 36c63a5..76d7c0c 100644 Binary files a/calibre-plugin/images/icon.xcf and b/calibre-plugin/images/icon.xcf differ diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py new file mode 100644 index 0000000..0c0f632 --- /dev/null +++ b/calibre-plugin/jobs.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Jim Miller' +__copyright__ = '2011, Grant Drake ' +__docformat__ = 'restructuredtext en' + +import time, os, traceback + +from ConfigParser import SafeConfigParser +from StringIO import StringIO +#from itertools import izip +#from threading import Event + +#from calibre.gui2.convert.single import sort_formats_by_preference +from calibre.utils.config import prefs +from calibre.utils.ipc.server import Server +from calibre.utils.ipc.job import ParallelJob +from calibre.utils.logging import Log + +from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload, + OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY) +from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions +from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge + +# ------------------------------------------------------------------------------ +# +# Functions to perform downloads using worker jobs +# +# ------------------------------------------------------------------------------ + +def do_download_worker(book_list, options, + cpus, notification=lambda x,y:x): + ''' + Master job, to launch child jobs to extract ISBN for a set of books + This is run as a worker job in the background to keep the UI more + responsive and get around the memory leak issues as it will launch + a child job for each book as a worker process + ''' + server = Server(pool_size=cpus) + + total = 0 + # Queue all the jobs + for book in book_list: + if book['good']: + print("add job for %s"%book['url']) + total += 1 + args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', + 'do_download_for_worker', + (book,options)] + job = ParallelJob('arbitrary', + "url:(%s) id:(%s)"%(book['url'],book['calibre_id']), + done=None, + args=args) + job._book = book + # job._book_id = book_id + # job._title = title + # job._modified_date = modified_date + # job._existing_isbn = existing_isbn + server.add_job(job) + + # This server is an arbitrary_n job, so there is a notifier available. + # Set the % complete to a small number to avoid the 'unavailable' indicator + notification(0.01, 'Downloading FanFiction Stories') + + # dequeue the job results as they arrive, saving the results + count = 0 + while True: + job = server.changed_jobs_queue.get() + # A job can 'change' when it is not finished, for example if it + # produces a notification. Ignore these. + job.update() + if not job.is_finished: + continue + # A job really finished. Get the information. + output_book = job.result + #print("output_book:%s"%output_book) + book_list.remove(job._book) + book_list.append(job.result) + book_id = job._book['calibre_id'] + #title = job._title + count = count + 1 + notification(float(count)/total, 'Downloaded Story') + # Add this job's output to the current log + print('Logfile for book ID %s (%s)'%(book_id, book['title'])) + print(job.details) + + if count >= total: + # All done! + break + + server.close() + + # return the book list as the job result + return book_list + +def do_download_for_worker(book,options): + ''' + Child job, to extract isbn from formats for this specific book, + when run as a worker job + ''' + try: + print("do_download_for_worker") + # print("is_adult:%s"%book['is_adult']) + # print("personal.ini len:%s"%len(options['personal.ini'])) + # print("defaults.ini len:%s"%len(get_resources("defaults.ini"))) + #time.sleep(2.0) + book['comment'] = 'Download started...' + + ffdlconfig = SafeConfigParser() + ffdlconfig.readfp(StringIO(get_resources("defaults.ini"))) + ffdlconfig.readfp(StringIO(options['personal.ini'])) + + adapter = adapters.getAdapter(ffdlconfig,book['url']) + adapter.is_adult = book['is_adult'] + adapter.username = book['username'] + adapter.password = book['password'] + + story = adapter.getStoryMetadataOnly() + writer = writers.getWriter(options['fileform'],adapter.config,adapter) + + outfile = book['outfile'] + + ## No need to download at all. Shouldn't ever get down here. + if options['collision'] in (CALIBREONLY): + print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...") + book['comment'] = 'Metadata collected.' + + ## checks were done earlier, it's new or not dup or newer--just write it. + elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \ + ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)): + print("write to %s"%outfile) + writer.writeStory(outfilename=outfile) + book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters")) + + ## checks were done earlier, just update it. + elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS): + + urlchaptercount = int(story.getMetadata('numChapters')) + ## First, get existing epub with titlepage and tocpage stripped. + updateio = StringIO() + (epuburl,chaptercount) = doMerge(updateio, + [book['epub_for_update']], + titlenavpoints=False, + striptitletoc=True, + forceunique=False) + print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)) + + ## Get updated title page/metadata by itself in an epub. + ## Even if the title page isn't included, this carries the metadata. + titleio = StringIO() + writer.writeStory(outstream=titleio,metaonly=True) + + newchaptersio = None + if urlchaptercount > chaptercount : + ## Go get the new chapters + newchaptersio = StringIO() + adapter.setChaptersRange(chaptercount+1,urlchaptercount) + + adapter.config.set("overrides",'include_tocpage','false') + adapter.config.set("overrides",'include_titlepage','false') + writer.writeStory(outstream=newchaptersio) + + ## Merge the three epubs together. + doMerge(outfile, + [titleio,updateio,newchaptersio], + fromfirst=True, + titlenavpoints=False, + striptitletoc=False, + forceunique=False) + + book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\ + (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount) + + except NotGoingToDownload as d: + book['good']=False + book['comment']=unicode(d) + book['icon'] = d.icon + + except Exception as e: + book['good']=False + book['comment']=unicode(e) + print("%s:%s"%(book,unicode(e))) + # XXX trace for not-expected exceptions + traceback.print_exc() + + #time.sleep(10) + return book diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py index f00b963..a17c7a6 100644 --- a/fanficdownloader/adapters/adapter_test1.py +++ b/fanficdownloader/adapters/adapter_test1.py @@ -67,9 +67,11 @@ class TestSiteAdapter(BaseSiteAdapter): raise exceptions.FailedToLogin(self.url,self.username) if self.story.getMetadata('storyId') == '664': - self.story.setMetadata(u'title',"Test Story Title "+self.crazystring) + self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring) + self.story.setMetadata('author','Test Author aa bare amp(&) quote(') amp(&)') else: self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')) + self.story.setMetadata('author','Test Author aa') self.story.setMetadata('storyUrl',self.url) self.story.setMetadata('description',u'Description '+self.crazystring+u''' Done @@ -85,7 +87,6 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!" self.story.setMetadata('status','In-Completed') self.story.setMetadata('rating','Tweenie') - self.story.setMetadata('author','Test Author aa') self.story.setMetadata('authorId','98765') self.story.setMetadata('authorUrl','http://author/url') diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index 7241a90..53f2d66 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -17,7 +17,7 @@ import os -from htmlcleanup import conditionalRemoveEntities +from htmlcleanup import conditionalRemoveEntities, removeAllEntities class Story: @@ -30,15 +30,17 @@ class Story: self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists. def setMetadata(self, key, value): + ## still keeps < < and & self.metadata[key]=conditionalRemoveEntities(value) def getMetadataRaw(self,key): if self.metadata.has_key(key): return self.metadata[key] - def getMetadata(self, key): + def getMetadata(self, key, removeallentities=False): + value = None if self.getLists().has_key(key): - return ', '.join(self.getList(key)) + value = ', '.join(self.getList(key)) if self.metadata.has_key(key): value = self.metadata[key] if value: @@ -48,6 +50,10 @@ class Story: value = value.strftime("%Y-%m-%d %H:%M:%S") if key == "datePublished" or key == "dateUpdated": value = value.strftime("%Y-%m-%d") + + if removeallentities and value != None: + return removeAllEntities(value) + else: return value def getAllMetadata(self):