Clear Reject List cache when switching libraries, bump version PI only.

ribbon cover exclusion for dramione.org

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-33
+version: 4-4-35
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 6, 18)
+    version             = (1, 7, 1)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/common_utils.py

@@ -196,7 +196,7 @@ class ImageTitleLayout(QHBoxLayout):
     '''
     A reusable layout widget displaying an image followed by a title
     '''
-    def __init__(self, parent, icon_name, title):
+    def __init__(self, parent, icon_name, title, tooltip=None):
         QHBoxLayout.__init__(self)
         title_image_label = QLabel(parent)
         pixmap = get_pixmap(icon_name)
@@ -217,6 +217,9 @@ class ImageTitleLayout(QHBoxLayout):
         self.addWidget(shelf_label)
         self.insertStretch(-1)
 
+        if tooltip:
+            title_image_label.setToolTip(tooltip)
+            shelf_label.setToolTip(tooltip)
 
 class SizePersistedDialog(QDialog):
     '''

calibre-plugin/config.py

@@ -7,20 +7,24 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import traceback, copy
+import traceback, copy, threading
 from collections import OrderedDict
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont, QWidget,
-                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea)
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
+                      QDialogButtonBox )
 
 from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
-    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order)
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getConfigSections
+    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
+            EditTextDialog)
+    
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
+    import (getConfigSections, getNormalStoryURL)
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
@@ -34,6 +38,10 @@ PREFS_KEY_SETTINGS = 'settings'
 # take from here.
 default_prefs = {}
 default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['rejecturls'] = ''
+default_prefs['rejectreasons'] = '''Sucked
+Boring
+Dup from another site'''
 
 default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False
@@ -137,8 +145,82 @@ class PrefsFacade():
     def save_to_db(self):
         set_library_config(self._get_prefs())
         
-
 prefs = PrefsFacade(default_prefs)
+
+class RejectURLList:
+    def __init__(self,prefs):
+        self.prefs = prefs
+        self.sync_lock = threading.RLock()
+        self.listcache = None
+
+    def _read_list_from_text(self,text):
+        cache = {}
+        for line in text.splitlines():
+            if ',' in line:
+                (rejurl,note) = line.split(',',1)
+            else:
+                (rejurl,note) = (line,'')
+            rejurl = getNormalStoryURL(rejurl)
+            if rejurl:
+                cache[rejurl] = note
+        return cache
+        
+
+    def _get_listcache(self):
+        if self.listcache == None:
+            self.listcache = self._read_list_from_text(prefs['rejecturls'])
+        return self.listcache
+
+    def _save_list(self,listcache):
+        rejectlist = []
+        for url in listcache:
+            rejectlist.append("%s,%s"%(url,listcache[url]))
+            
+        self.prefs['rejecturls'] = '\n'.join(rejectlist)
+        self.prefs.save_to_db()
+        self.listcache = None
+        
+    def clear_cache(self):
+        self.listcache = None
+
+    def check(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                note = listcache[url]
+                return note
+            
+            # not found
+            return None
+
+    def remove(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                del listcache[url]
+                self._save_list(listcache)
+
+    def add_text(self,rejecttext):
+        self.add(self._read_list_from_text(rejecttext).items())
+            
+    def add(self,rejectlist,clear=False):
+        # rejectlist=list of (url,note) tuples.
+        with self.sync_lock:
+            if clear:
+                listcache={}
+            else:
+                listcache = self._get_listcache()
+            for (url,note) in rejectlist:
+                listcache[url]=note
+            self._save_list(listcache)
+
+    def get_list(self):
+        return copy.deepcopy(self._get_listcache())
+            
+    def get_reject_reasons(self):
+        return self.prefs['rejectreasons'].splitlines()
+
+rejecturllist = RejectURLList(prefs)
     
 class ConfigWidget(QWidget):
 
@@ -396,6 +478,27 @@ class BasicTab(QWidget):
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
+        self.l.addSpacing(10)        
+
+        horz = QHBoxLayout()
+        
+        self.rejectlist = QPushButton('Edit Reject URL List', self)
+        self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
+        self.rejectlist.clicked.connect(self.show_rejectlist)
+        horz.addWidget(self.rejectlist)
+        
+        self.reject_urls = QPushButton('Add Reject URLs', self)
+        self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
+        self.reject_urls.clicked.connect(self.add_reject_urls)
+        horz.addWidget(self.reject_urls)
+        
+        self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
+        self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
+        self.reject_reasons.clicked.connect(self.show_reject_reasons)
+        horz.addWidget(self.reject_reasons)
+        
+        self.l.addLayout(horz)
+        
         self.l.insertStretch(-1)
         
     def set_collisions(self):
@@ -411,7 +514,50 @@ class BasicTab(QWidget):
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+
+    def show_rejectlist(self):
+        rejectlist = []
+        for (url,note) in rejecturllist.get_list().items():
+            rejectlist.append((None,url,note,note))
+            
+        d = RejectListDialog(self,
+                             rejectlist,
+                             rejectreasons=rejecturllist.get_reject_reasons(),
+                             header="Edit Reject URLs List",
+                             show_delete=False)
+        d.exec_()
         
+        if d.result() != d.Accepted:
+            return
+        
+        rejectlist=[]
+        for (bookid,url,note) in d.get_reject_list():
+            rejectlist.append((url,note))
+
+        rejecturllist.add(rejectlist,clear=True)
+        
+    def show_reject_reasons(self):
+        d = EditTextDialog(self,
+                           prefs['rejectreasons'],
+                           icon=self.windowIcon(),
+                           title="Reject Reasons",
+                           label="Customize Reject List Reasons",
+                           tooltip="Customize the Reasons presented when Rejecting URLs")
+        d.exec_()
+        if d.result() == d.Accepted:
+            prefs['rejectreasons'] = d.get_plain_text()
+                
+    def add_reject_urls(self):
+        d = EditTextDialog(self,
+                           "http://example.com?story.php?sid=5,Reason why I rejected it",
+                           icon=self.windowIcon(),
+                           title="Add Reject URLs",
+                           label="Add Reject URLs. Use: <b>http://...,note</b>",
+                           tooltip="One URL per line, everything after <b>,</b> will be put in the note.")
+        d.exec_()
+        if d.result() == d.Accepted:
+            rejecturllist.add_text(d.get_plain_text())
+                
 class PersonalIniTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -451,7 +597,7 @@ class PersonalIniTab(QWidget):
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
-
+        
 class ShowDefaultsIniDialog(QDialog):
 
     def __init__(self, icon, text, parent=None):
@@ -916,3 +1062,4 @@ class StandardColumnsTab(QWidget):
             self.l.addLayout(horz)
         
         self.l.insertStretch(-1)
+

calibre-plugin/dialogs.py

@@ -8,15 +8,19 @@ __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import traceback
+from functools import partial
 
 from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, QTextCursor,
-                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
-                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt, QAbstractItemView )
+from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout,
+                      QGridLayout, QPushButton, QProgressDialog, QString, QLabel,
+                      QCheckBox, QIcon, QTextCursor, QTextEdit, QLineEdit, QInputDialog,
+                      QComboBox, QClipboard, QVariant, QProgressDialog, QTimer,
+                      QDialogButtonBox, QPixmap, Qt, QAbstractItemView, SIGNAL,
+                      QTableWidgetItem )
 
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.gui2.complete2 import EditWithComplete
 
 from calibre import confirm_config_name
 from calibre.gui2 import dynamic
@@ -712,3 +716,281 @@ class StoryListTableWidget(QTableWidget):
             self.setItem(dest_row, col, self.takeItem(src_row, col))
         self.removeRow(src_row)
         self.blockSignals(False)
+
+class RejectListTableWidget(QTableWidget):
+
+    def __init__(self, parent,rejectreasons=[]):
+        QTableWidget.__init__(self, parent)
+        self.setSelectionBehavior(QAbstractItemView.SelectRows)
+        self.rejectreasons = rejectreasons
+
+    def on_headersection_clicked(self):
+        self.setSortingEnabled(True)
+        
+    def populate_table(self, reject_list):
+        self.clear()
+        self.setAlternatingRowColors(True)
+        self.setRowCount(len(reject_list))
+        header_labels = ['URL', 'Note']
+        self.setColumnCount(len(header_labels))
+        self.setHorizontalHeaderLabels(header_labels)
+        self.horizontalHeader().setStretchLastSection(True)
+        #self.verticalHeader().setDefaultSectionSize(24)
+        self.verticalHeader().hide()
+
+        # need sortingEnbled to sort, but off to up & down.
+        self.connect(self.horizontalHeader(),
+                     SIGNAL('sectionClicked(int)'),
+                     self.on_headersection_clicked)
+
+        # row is just row number.
+        for row, rejectrow in enumerate(reject_list):
+            self.populate_table_row(row,rejectrow)
+
+        self.resizeColumnsToContents()
+        self.setMinimumColumnWidth(1, 100)
+        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumSize(300, 0)
+
+    def setMinimumColumnWidth(self, col, minimum):
+        if self.columnWidth(col) < minimum:
+            self.setColumnWidth(col, minimum)
+
+    def populate_table_row(self, row, rejectrow):
+
+        (bookid,url,titleauth,oldrejnote) = rejectrow
+        if oldrejnote:
+            noteprefix = note = oldrejnote
+            # incase the existing note ends with one of the known reasons.
+            for reason in self.rejectreasons:
+                if noteprefix.endswith(' - '+reason):
+                    noteprefix = noteprefix[:-len(' - '+reason)]
+                    break
+        else:
+            noteprefix = note = titleauth
+            
+        if len(noteprefix) > 0:
+            noteprefix = noteprefix+' - '
+            
+        url_cell = ReadOnlyTableWidgetItem(url)
+        url_cell.setData(Qt.UserRole, QVariant(bookid))
+        url_cell.setToolTip('URL to add to the Reject List.')
+        self.setItem(row, 0, url_cell)
+
+        note_cell = EditWithComplete(self)
+
+        # This is a more than slightly kludgey way to get
+        # EditWithComplete to *not* alpha-order the reasons, but leave
+        # them in the order entered.  If
+        # calibre.gui2.complete2.CompleteModel.set_items ever changes,
+        # this function will need to also.
+        def complete_model_set_items_kludge(self, items):
+            items = [unicode(x.strip()) for x in items]
+            items = [x for x in items if x]
+            items = tuple(items)
+            self.all_items = self.current_items = items
+            self.current_prefix = ''
+            self.reset()
+
+        note_cell.lineEdit().mcompleter.model().set_items = \
+            partial(complete_model_set_items_kludge,
+                    note_cell.lineEdit().mcompleter.model())
+        
+        items = [note]+[ noteprefix+x for x in self.rejectreasons ]
+        note_cell.update_items_cache(items)
+        note_cell.show_initial_value(note)
+        note_cell.set_separator(None)
+        note_cell.setToolTip('Select or Edit Reject Note.')
+        self.setCellWidget(row, 1, note_cell)
+        
+        # note_cell = QTableWidgetItem(note)
+        # note_cell.setToolTip('Double-click to edit note.')
+        # self.setItem(row, 1, note_cell)
+      
+    def get_reject_list(self):
+        rejectrows = []
+        for row in range(self.rowCount()):
+            bookid = self.item(row, 0).data(Qt.UserRole).toPyObject()
+            url = unicode(self.item(row, 0).text())
+            note = unicode(self.cellWidget(row, 1).currentText()).strip()
+            rejectrows.append((bookid,url,note))
+        return rejectrows
+
+    def remove_selected_rows(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        message = '<p>Are you sure you want to remove this URL from the list?'
+        if len(rows) > 1:
+            message = '<p>Are you sure you want to remove the %d selected URLs from the list?'%len(rows)
+        if not confirm(message,'ffdl_rejectlist_delete_item_again', self):
+            return
+        first_sel_row = self.currentRow()
+        for selrow in reversed(rows):
+            self.removeRow(selrow.row())
+        if first_sel_row < self.rowCount():
+            self.select_and_scroll_to_row(first_sel_row)
+        elif self.rowCount() > 0:
+            self.select_and_scroll_to_row(first_sel_row - 1)
+
+    def select_and_scroll_to_row(self, row):
+        self.selectRow(row)
+        self.scrollToItem(self.currentItem())
+
+    def move_rows_up(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        first_sel_row = rows[0].row()
+        if first_sel_row <= 0:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in selrows:
+            self.swap_row_widgets(selrow - 1, selrow + 1)
+        scroll_to_row = first_sel_row - 1
+        if scroll_to_row > 0:
+            scroll_to_row = scroll_to_row - 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def move_rows_down(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        last_sel_row = rows[-1].row()
+        if last_sel_row == self.rowCount() - 1:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in reversed(selrows):
+            self.swap_row_widgets(selrow + 2, selrow)
+        scroll_to_row = last_sel_row + 1
+        if scroll_to_row < self.rowCount() - 1:
+            scroll_to_row = scroll_to_row + 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def swap_row_widgets(self, src_row, dest_row):
+        self.blockSignals(True)
+        self.setSortingEnabled(False)
+        self.insertRow(dest_row)
+        for col in range(0, self.columnCount()):
+            self.setItem(dest_row, col, self.takeItem(src_row, col))
+        self.removeRow(src_row)
+        self.blockSignals(False)
+
+class RejectListDialog(SizePersistedDialog):
+    def __init__(self, gui, reject_list,
+                 rejectreasons=[],
+                 header="List of Books to Reject",
+                 icon='rotate-right.png',
+                 show_delete=True,
+                 save_size_name='ffdl:reject list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        self.gui = gui
+      
+        self.setWindowTitle(header)
+        self.setWindowIcon(get_icon(icon))
+      
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, icon, header,
+                                        '<i></i>FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.')
+        layout.addLayout(title_layout)
+        rejects_layout = QHBoxLayout()
+        layout.addLayout(rejects_layout)
+
+        self.rejects_table = RejectListTableWidget(self,rejectreasons=rejectreasons)
+        rejects_layout.addWidget(self.rejects_table)
+
+        button_layout = QVBoxLayout()
+        rejects_layout.addLayout(button_layout)
+        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem)
+        # self.move_up_button = QtGui.QToolButton(self)
+        # self.move_up_button.setToolTip('Move selected books up the list')
+        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
+        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
+        # button_layout.addWidget(self.move_up_button)
+        self.remove_button = QtGui.QToolButton(self)
+        self.remove_button.setToolTip('Remove selected URL(s) from the list')
+        self.remove_button.setIcon(get_icon('list_remove.png'))
+        self.remove_button.clicked.connect(self.remove_from_list)
+        button_layout.addWidget(self.remove_button)
+        # self.move_down_button = QtGui.QToolButton(self)
+        # self.move_down_button.setToolTip('Move selected books down the list')
+        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
+        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
+        # button_layout.addWidget(self.move_down_button)
+        spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem1)
+
+        options_layout = QHBoxLayout()
+
+        if show_delete:
+            self.deletebooks = QCheckBox('Delete Books (including books without FanFiction URLs)?',self)
+            self.deletebooks.setToolTip("Delete the selected books after adding them to the Rejected URLs list.")
+            self.deletebooks.setChecked(True)
+            options_layout.addWidget(self.deletebooks)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        options_layout.addWidget(button_box)
+      
+        layout.addLayout(options_layout)
+      
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.rejects_table.populate_table(reject_list)
+
+    def remove_from_list(self):
+        self.rejects_table.remove_selected_rows()
+
+    def get_reject_list(self):
+        return self.rejects_table.get_reject_list()
+
+    def get_deletebooks(self):
+        return self.deletebooks.isChecked()
+
+class EditTextDialog(QDialog):
+
+    def __init__(self, parent, text,
+                 icon=None, title=None, label=None, tooltip=None):
+        QDialog.__init__(self, parent)
+        self.resize(600, 500)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.label = QLabel(label)
+        if title:
+            self.setWindowTitle(title)
+        if icon:
+            self.setWindowIcon(icon)
+        self.l.addWidget(self.label)
+        
+        self.textedit = QTextEdit(self)
+        self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        self.textedit.setText(text)
+        self.l.addWidget(self.textedit)
+
+        if tooltip:
+            self.label.setToolTip(tooltip)
+            self.textedit.setToolTip(tooltip)
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
+
+    def get_plain_text(self):
+        return unicode(self.textedit.toPlainText())

calibre-plugin/ffdl_plugin.py

@@ -41,10 +41,10 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable i
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
 
-from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values)
+from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values, rejecturllist)
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
     AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog,
-    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, 
+    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
     NotGoingToDownload )
 
@@ -145,6 +145,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def library_changed(self, db):
         # We need to reset our menus after switching libraries
         self.rebuild_menus()
+        rejecturllist.clear_cache()
         
     def rebuild_menus(self):
         with self.menus_lock:
@@ -190,6 +191,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
                                                                 triggered=self.get_urls_from_page)
 
+            self.reject_list_action = self.create_menu_item_ex(self.menu, 'Reject Selected Books', image='rotate-right.png',
+                                                               triggered=self.reject_list_urls)
+
             # print("platform.system():%s"%platform.system())
             # print("platform.mac_ver()[0]:%s"%platform.mac_ver()[0])
             if not self.check_macmenuhack(): # not platform.mac_ver()[0]: # Some macs crash on these menu items for unknown reasons.
@@ -211,7 +215,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 if menu_id not in self.actions_unique_map:
                     self.gui.keyboard.unregister_shortcut(unique_name)
             self.old_actions_unique_map = self.actions_unique_map
-            self.gui.keyboard.finalize()            
+            self.gui.keyboard.finalize()
 
     def about(self):
         # Get the about text from a file inside the plugin zip file
@@ -238,15 +242,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         #print("create_menu_item_ex after %s"%menu_text)
         return ac
 
+    def is_library_view(self):
+        # 0 = library, 1 = main, 2 = card_a, 3 = card_b
+        return self.gui.stack.currentIndex() == 0
+    
     def plugin_button(self):
-        if len(self.gui.library_view.get_selected_ids()) > 0 and prefs['updatedefault']:
+        if self.is_library_view() and \
+                len(self.gui.library_view.get_selected_ids()) > 0 and \
+                prefs['updatedefault']:
             self.update_existing()
         else:
             self.add_dialog()
 
     def update_lists(self,add=True):
-        if len(self.gui.library_view.get_selected_ids()) > 0 and \
-                (prefs['addtolists'] or prefs['addtoreadlists']) :
+        if prefs['addtolists'] or prefs['addtoreadlists']:
+            if not self.is_library_view():
+                self.gui.status_bar.show_message(_('Cannot Update Reading Lists from Device View'), 3000)
+                return
+        
+            if len(self.gui.library_view.get_selected_ids()) == 0:
+                self.gui.status_bar.show_message(_('No Selected Books to Update Reading Lists'), 3000)
+                return
+
             self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)
 
     def get_urls_from_page(self):
@@ -284,19 +301,35 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
             
     def get_list_urls(self):
-        if len(self.gui.library_view.get_selected_ids()) > 0:
-            book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
+        if self.gui.current_view().selectionModel().selectedRows() == 0 :
+            self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
+                                             3000)
+            return
+            
+        if self.is_library_view():
+            book_list = map( partial(self._convert_id_to_book, good=False),
+                             self.gui.library_view.get_selected_ids() )
 
-            LoopProgressDialog(self.gui,
-                               book_list,
-                               partial(self._get_story_url_for_list, db=self.gui.current_db),
-                               self._finish_get_list_urls,
-                               init_label="Collecting URLs for stories...",
-                               win_title="Get URLs for stories",
-                               status_prefix="URL retrieved")
+        else: # device view, get from epubs on device.
+            view = self.gui.current_view()
+            rows = view.selectionModel().selectedRows()
+            # paths = view.model().paths(rows)
+            book_list = map( partial(self._convert_row_to_book, good=False), rows )
+            
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self._get_story_url_for_list, db=self.gui.current_db),
+                           self._finish_get_list_urls,
+                           init_label="Collecting URLs for stories...",
+                           win_title="Get URLs for stories",
+                           status_prefix="URL retrieved")
             
     def _get_story_url_for_list(self,book,db=None):
-        book['url'] = self._get_story_url(db,book['calibre_id'])
+        if book['calibre_id']:
+            book['url'] = self._get_story_url(db,book_id=book['calibre_id'])
+        elif book['path']:
+            book['url'] = self._get_story_url(db,path=book['path'])
+            
         if book['url'] == None:
             book['good']=False
         else:
@@ -314,6 +347,78 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         show=True,
                         show_copy_button=False)
                 
+    def reject_list_urls(self):
+        if self.is_library_view():
+            book_list = map( partial(self._convert_id_to_book, good=False),
+                             self.gui.library_view.get_selected_ids() )
+
+        else: # device view, get from epubs on device.
+            view = self.gui.current_view()
+            rows = view.selectionModel().selectedRows()
+            #paths = view.model().paths(rows)
+            book_list = map( partial(self._convert_row_to_book, good=False), rows )
+            
+        if len(book_list) == 0 :
+            self.gui.status_bar.show_message(_('No Selected Books have URLs to Reject'), 3000)
+            return
+            
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self._reject_story_url_for_list, db=self.gui.current_db),
+                           self._finish_reject_list_urls,
+                           init_label="Collecting URLs for Reject List...",
+                           win_title="Get URLs for Reject List",
+                           status_prefix="URL retrieved")
+            
+    def _reject_story_url_for_list(self,book,db=None): 
+        if book['calibre_id']:
+            # want title/author, too, for rejects.
+            self._populate_book_from_calibre_id(book,db)
+            book['url'] = self._get_story_url(db,book_id=book['calibre_id'])
+        elif book['path']:
+            book['url'] = self._get_story_url(db,path=book['path'])
+                
+        if book['url'] == None:
+            book['good']=False
+        else:
+            book['good']=True
+            # get existing note, if there is one.
+            book['oldrejnote']=rejecturllist.check(book['url'])
+            
+    def _finish_reject_list_urls(self, book_list):
+
+        # construct reject list of tuples:
+        # (calibre_id, url, "title, authors", old reject note).
+        reject_list = [ ( x['calibre_id'],x['url'],
+                          "%s by %s"%(x['title'],
+                                      ', '.join(x['author'])),
+                          x['oldrejnote'])
+                        for x in book_list if x['good'] ]
+        if reject_list:
+            d = RejectListDialog(self.gui,reject_list,
+                                 rejectreasons=rejecturllist.get_reject_reasons())
+            d.exec_()
+            
+            if d.result() != d.Accepted:
+                return
+
+            bookids=[]
+            rejectlist=[]
+            for (bookid,url,note) in d.get_reject_list():
+                bookids.append(bookid)
+                rejectlist.append((url,note))
+                print("Adding (%s) to Reject List: %s"%(url,note))
+                
+            rejecturllist.add(rejectlist)
+
+            if d.get_deletebooks():
+                self.gui.iactions['Remove Books'].delete_books()
+            
+        else:
+            message="<p>Rejecting FFDL URLs: None of the books selected have FanFiction URLs.</p><p>Proceed to Remove?</p>"
+            if confirm(message,'fanfictiondownloader_reject_non_fanfiction', self.gui):
+                self.gui.iactions['Remove Books'].delete_books()
+                
     def add_dialog(self):
 
         #print("add_dialog()")
@@ -346,7 +451,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         self.start_downloads( options, add_books )
         
     def update_existing(self):
+        if not self.is_library_view():
+            self.gui.status_bar.show_message(_('Cannot Update Books from Device View'), 3000)
+            return
+        
         if len(self.gui.library_view.get_selected_ids()) == 0:
+            self.gui.status_bar.show_message(_('No Selected Books to Update'), 3000)
             return
         #print("update_existing()")
         
@@ -431,6 +541,29 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         necessary data.  To be called from LoopProgressDialog
         'loop'.  Also pops dialogs for is adult, user/pass.
         '''
+
+        url = book['url']
+        print("url:%s"%url)
+
+        rejnote = rejecturllist.check(url)
+        if rejnote:
+            if question_dialog(self.gui, 'Reject URL?',
+                               '<p>Reject URL?</p>'+
+                               '<p>%s is on the Reject URL list:<br />"%s"</p>'%(url,rejnote)+
+                               "<p>Click 'No' to download anyway.</p>",
+                               show_copy_button=False):
+                book['comment'] = "Story on Reject URLs list (%s)."%rejnote
+                book['good']=False
+                book['icon']='rotate-right.png'
+                book['status'] = 'Rejected'
+                return
+            else:
+                if question_dialog(self.gui, 'Remove Reject URL?',
+                                   "<p>Remove URL from Reject List?</p>"+
+                                   '<p>%s is on the Reject URL list:<br />"%s"</p>'%(url,rejnote)+
+                                   "<p>Click 'Yes' to remove it from the list and download,<br /> 'No' to download, but leave it on the Reject list.</p>",
+                                   show_copy_button=False):
+                    rejecturllist.remove(url)
         
         # The current database shown in the GUI
         # db is an instance of the class LibraryDatabase2 from database.py
@@ -447,8 +580,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             # book has already been flagged bad for whatever reason.
             return
         
-        url = book['url']
-        print("url:%s"%url)
         skip_date_update = False
         
         options['personal.ini'] = prefs['personal.ini']
@@ -502,11 +633,6 @@ make_firstimage_cover:true
             book['comments']=''
         book['series'] = story.getMetadata("series", removeallentities=True)
         
-        # adapter.opener is the element with a threadlock.  But del
-        # adapter.opener doesn't work--subproc fails when it tries
-        # to pull in the adapter object that hasn't been imported yet.
-        # book['adapter'] = adapter
-        
         book['is_adult'] = adapter.is_adult
         book['username'] = adapter.username
         book['password'] = adapter.password
@@ -540,8 +666,8 @@ make_firstimage_cover:true
             # 'new' book from URL.  collision handling applies.
             print("from URL(%s)"%url)
 
-            # try to find by identifier url first.
-            searchstr = 'identifiers:"=url:=%s"'%url.replace(":","|")
+            # try to find by identifier url or uri first.
+            searchstr = 'identifiers:"~ur(i|l):=%s"'%url.replace(":","|")
             identicalbooks = db.search_getting_ids(searchstr, None)
             if len(identicalbooks) < 1:
                 # find dups
@@ -904,13 +1030,13 @@ make_firstimage_cover:true
             # mi.tags needs to be list, but set kills dups.
             mi.tags = list(set(list(old_tags)+mi.tags))
 
-        if 'langcode' in book['all_metadata']:
+        if book['all_metadata']['langcode']:
             mi.languages=[book['all_metadata']['langcode']]
         else:
             # Set language english, but only if not already set.
             if not oldmi.languages:
-                mi.languages=['eng']
-
+                mi.languages=['en']
+                
         if options['fileform'] == 'epub' and prefs['updatecover']:
             existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
             epubmi = get_metadata(existingepub,'EPUB')
@@ -1118,12 +1244,6 @@ make_firstimage_cover:true
                 message="<p>You configured FanFictionDownLoader to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?</p>"
                 confirm(message,'fanfictiondownloader_no_send_lists', self.gui)
 
-            # Quick demo of how an 'allow send' list might work.
-            # Issues: allow list per send list?  Naming convention?  "send(allow)"
-            # allow_list = rl_plugin.get_book_list("Allow Send to Device")
-            # # intersection of book_ids & allow_list
-            # add_book_ids = list(set(book_ids) & set(allow_list))
-
             for l in lists:
                 if l in rl_plugin.get_list_names():
                     #print("good send l:(%s)"%l)
@@ -1136,17 +1256,6 @@ make_firstimage_cover:true
                         message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
                         confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)
 
-    # def _find_existing_book_id(self,db,book,matchurl=True):
-    #     mi = MetaInformation(book["title"],book["author"]) # author is a list.
-    #     identicalbooks = db.find_identical_books(mi)
-    #     if matchurl: # only *really* identical if URL matches, too.
-    #         for ib in identicalbooks:
-    #             if self._get_story_url(db,ib) == book['url']:
-    #                 return ib
-    #     if identicalbooks:
-    #         return identicalbooks.pop()
-    #     return None
-    
     def _make_mi_from_book(self,book):
         mi = MetaInformation(book['title'],book['author']) # author is a list.
         mi.set_identifiers({'url':book['url']})
@@ -1208,7 +1317,25 @@ make_firstimage_cover:true
         book['comment'] = ''
         book['url'] = ''
         book['added'] = False
-        
+
+        return book
+            
+    def _convert_row_to_book(self, row, good=True):
+        book = {}
+        mi = self.gui.current_view().model().get_book_display_info(row.row())
+        book['title'] = mi.title
+        book['author'] = mi.authors
+        book['path'] = mi.path
+        book['author_sort'] = mi.author_sort
+        book['good'] = good
+        book['calibre_id'] = None
+        book['begin'] = None
+        book['end'] = None
+            
+        book['comment'] = ''
+        book['url'] = ''
+        book['added'] = False
+
         return book
             
     def _populate_book_from_calibre_id(self, book, db=None):
@@ -1243,44 +1370,46 @@ make_firstimage_cover:true
                 book['icon']='dialog_error.png'
                 book['status'] = 'Bad URL'
     
-    def _get_story_url(self, db, book_id):
-        identifiers = db.get_identifiers(book_id,index_is_id=True)
+    def _get_story_url(self, db, book_id=None, path=None):
+        if book_id == None:
+            identifiers={}
+        else:
+            identifiers = db.get_identifiers(book_id,index_is_id=True)
         if 'url' in identifiers:
             # identifiers have :->| in url.
-            #print("url from book:"+identifiers['url'].replace('|',':'))
+            # print("url from ident url:%s"%identifiers['url'].replace('|',':'))
             return identifiers['url'].replace('|',':')
+        elif 'uri' in identifiers:
+            # identifiers have :->| in uri.
+            # print("uri from ident uri:%s"%identifiers['uri'].replace('|',':'))
+            return identifiers['uri'].replace('|',':')
         else:
-            ## only epub has URL in it--at least where I can easily find it.
-            if db.has_format(book_id,'EPUB',index_is_id=True):
+            existingepub = None
+            if path == None and db.has_format(book_id,'EPUB',index_is_id=True):
                 existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
                 mi = get_metadata(existingepub,'EPUB')
                 identifiers = mi.get_identifiers()
                 if 'url' in identifiers:
-                    #print("url from epub:"+identifiers['url'].replace('|',':'))
+                    # print("url from get_metadata:%s"%identifiers['url'].replace('|',':'))
                     return identifiers['url'].replace('|',':')
-                # look for dc:source first, then scan HTML if 
+            elif path.lower().endswith('.epub'):
+                existingepub = path
+                
+            ## only epub has URL in it--at least where I can easily find it.
+            if existingepub:                
+                # look for dc:source first, then scan HTML if lookforurlinhtml
                 link = get_dcsource(existingepub)
                 if link:
+                    # print("url from get_dcsource:%s"%link)
                     return link
                 elif prefs['lookforurlinhtml']:
-                    return get_story_url_from_html(existingepub,self._is_good_downloader_url)
+                    link = get_story_url_from_html(existingepub,self._is_good_downloader_url)
+                    # print("url from get_story_url_from_html:%s"%link)
+                    return link
         return None
 
     def _is_good_downloader_url(self,url):
-        # this is the accepted way to 'check for existance of a class variable'?  really?
-        try:
-            self.dummyconfig
-        except AttributeError:
-            self.dummyconfig = Configuration("test1.com","EPUB")
-        # pulling up an adapter is pretty low over-head.  If
-        # it fails, it's a bad url.
-        try:
-            adapter = adapters.getAdapter(self.dummyconfig,url)
-            url = adapter.url
-            del adapter
-            return url
-        except:
-            return None;
+        return adapters.getNormalStoryURL(url)
 
 def get_url_list(urls):
     def f(x):

defaults.ini

@@ -386,9 +386,20 @@ output_css:
 image_max_size: 580, 725
 
 ## Change image to grayscale, if graphics library allows, to save
-## space.
+## space.  Transparency removed as if remove_transparency: true
 #grayscale_images: false
 
+## jpg or png
+## -- jpg produces smaller images, and may be supported by more
+## readers, but it's older and doesn't allow transparency.
+## Transparency removed as if remove_transparency: true
+## -- png is newer but does allow transparency, but only in CLI.
+## It doesn't work in calibre PI due to limitations of the API.
+convert_images_to: jpg
+
+## Remove transparency and fill with background_color if true.
+remove_transparency: true
+
 ## if the <img> tag doesn't have a div or a p around it, nook gets
 ## confused and displays it on every page after that under the text
 ## for the rest of the chapter.  I doubt adding a div around the img
@@ -458,14 +469,15 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks
+extra_valid_entries:fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
 fandoms_label:Fandoms
 freefromtags_label:Freeform Tags
 ao3categories_label:AO3 Categories
 comments_label:Comments
 kudos_label:Kudos
 hits_label:Hits
-bookmarks:Bookmarks
+collections_label:Collections
+bookmarks_label:Bookmarks
 
 ## adds to titlepage_entries instead of replacing it.
 #extra_titlepage_entries: fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks
@@ -559,6 +571,10 @@ extraships:Draco Malfoy/Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/.*?ribbon.gif
+
 ## Some adapters collect additional meta information beyond the
 ## standard ones.  They need to be defined in extra_valid_entries to
 ## tell the rest of the FFDL system about them.  They can be used in

fanficdownloader/adapters/__init__.py

@@ -23,6 +23,7 @@ import urlparse as up
 logger = logging.getLogger(__name__)
 
 from .. import exceptions as exceptions
+from ..configurable import Configuration
 
 ## must import each adapter here.
 
@@ -107,7 +108,6 @@ import adapter_jlaunlimitedcom
 import adapter_qafficcom
 import adapter_efpfanficnet
 
-
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
 ## to pick out the adapter.
@@ -125,6 +125,22 @@ for x in imports():
         #print x
         __class_list.append(sys.modules[x].getClass())
 
+def getNormalStoryURL(url):
+    if not getNormalStoryURL.__dummyconfig:
+        getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
+    # pulling up an adapter is pretty low over-head.  If
+    # it fails, it's a bad url.
+    try:
+        adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
+        url = adapter.url
+        del adapter
+        return url
+    except:
+        return None;
+
+# kludgey function static/singleton
+getNormalStoryURL.__dummyconfig = None
+
 def getAdapter(config,url):
 
     logger.debug("trying url:"+url)
@@ -155,10 +171,6 @@ def getClassFor(url):
     ## remove any trailing '#' locations.
     fixedurl = re.sub(r"#.*$","",fixedurl)
     
-    ## remove any trailing '&' parameters--?sid=999 will be left.
-    ## that's all that any of the current adapters need or want.
-    fixedurl = re.sub(r"&.*$","",fixedurl)
-    
     parsedUrl = up.urlparse(fixedurl)
     domain = parsedUrl.netloc.lower()
     if( domain != parsedUrl.netloc ):
@@ -175,6 +187,8 @@ def getClassFor(url):
         cls = getClassFromList("www."+domain)
         fixedurl = fixedurl.replace("http://","http://www.")
         
+    fixedurl = cls.stripURLParameters(fixedurl)
+    
     return (cls,fixedurl)
     
 def getClassFromList(domain):

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -233,13 +233,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chars = a.findAll('a',{'class':"tag"})
             for char in chars:
                 self.story.addToList('characters',char.string)
+                
         a = metasoup.find('dd',{'class':"relationship tags"})
         if a != None:
             ships = a.findAll('a',{'class':"tag"})
             for ship in ships:
                 self.story.addToList('ships',ship.string)
 		
-
+        a = metasoup.find('dd',{'class':"collections"})
+        if a != None:
+            collections = a.findAll('a')
+            for collection in collections:
+                self.story.addToList('collections',collection.string)
+		
         stats = metasoup.find('dl',{'class':'stats'})
         dt = stats.findAll('dt')
         dd = stats.findAll('dd')

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -158,6 +158,7 @@ class EFPFanFicNet(BaseSiteAdapter):
                 self.chapterUrls.append((title,url))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('language','Italian')
         
         # normalize story URL to first chapter if later chapter URL was given:
         url = self.chapterUrls[0][1].replace('&i=1','')
@@ -171,10 +172,11 @@ class EFPFanFicNet(BaseSiteAdapter):
 
         storya = None
         authsoup = None
+        storyblock = None
         authurl = self.story.getMetadata('authorUrl')
         
         ## author can have more than one page of stories.
-        while storya == None:
+        while storyblock == None:
 
             # no storya, but do have authsoup--we're looping on author pages.
             if authsoup != None:
@@ -186,10 +188,14 @@ class EFPFanFicNet(BaseSiteAdapter):
             logger.debug("fetching author page: (%s)"%authurl)
             authsoup = bs.BeautifulSoup(self._fetchUrl(authurl))
             #print("authsoup:%s"%authsoup)
-        
-            storya = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
-        
-        storyblock = storya.parent.parent.parent
+
+            storyas = authsoup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
+            for storya in storyas:
+                #print("======storya:%s"%storya)
+                storyblock = storya.findParent('div',{'class':'storybloc'})
+                #print("======storyblock:%s"%storyblock)
+                if storyblock != None:
+                    continue
 
         self.setDescription(url,storyblock.find('div', {'class':'introbloc'}))
 
@@ -293,4 +299,14 @@ class EFPFanFicNet(BaseSiteAdapter):
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return self.utf8FromSoup(url,div)
+        # remove any header and 'o:p' tags.
+        for tag in div.findAll("head") + div.findAll("o:p"):
+            tag.extract()
+
+        # change any html and body tags to div.
+        for tag in div.findAll("html") + div.findAll("body"):
+            tag.name='div'
+
+        # remove extra bogus doctype.
+        #<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+        return re.sub(r"<!DOCTYPE[^>]+>","",self.utf8FromSoup(url,div))

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -68,7 +68,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -215,9 +215,14 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         else:
             self.story.setMetadata('status', 'In-Progress')
 
-        img = soup.find('img',{'class':'cimage'})
-        if img:
-            self.setCoverImage(url,img['src'])
+        # Try the larger image first.
+        try:
+            img = soup.find('img',{'class':'lazy cimage'})
+            self.setCoverImage(url,img['data-original'])
+        except:
+            img = soup.find('img',{'class':'cimage'})
+            if img:
+                self.setCoverImage(url,img['src'])
             
         # Find the chapter selector 
         select = soup.find('select', { 'name' : 'chapter' } )

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -125,10 +125,12 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata("numChapters", len(self.chapterUrls))
         
         # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
-        # For the sake of bringing it in line with the other adapters, 'Incomplete' and 'On Hiatus' become 'In-Progress'
+        # For the sake of bringing it in line with the other adapters, 'Incomplete' becomes 'In-Progress'
         # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
         # status to leave unchanged.
-        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
+        # Nov2012 - 'On Hiatus' is now passed, too.  It's easy now for users to change/remove if they want
+        # with replace_metadata
+        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("Complete", "Completed")
         self.story.setMetadata("status", status)
         self.story.setMetadata("rating", storyMetadata["content_rating_text"])
             
@@ -144,8 +146,9 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 coverurl = storyMetadata["full_image"]
             else:
                 coverurl = storyMetadata["image"]
-            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
+            if coverurl.startswith('//'): # fix for img urls missing 'http:'
                 coverurl = "http:"+coverurl
+
             self.setCoverImage(self.url,coverurl)
 
         # the fimfic API gives bbcode for desc, not html.

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -76,7 +76,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.squidge.org'
     
-    @classmethod # must be @staticmethod, don't remove it.
+    @classmethod # must be @classmethod, don't remove it.
     def getConfigSection(cls):
         # The config section name.  Only override if != site domain.
         return cls.getSiteDomain()+'/peja'

fanficdownloader/adapters/base_adapter.py

@@ -255,6 +255,13 @@ class BaseSiteAdapter(Configurable):
         "Only needs to be overriden if != site domain."
         return cls.getSiteDomain()
     
+    @classmethod
+    def stripURLParameters(cls,url):
+        "Only needs to be overriden if URL contains more than one parameter"
+        ## remove any trailing '&' parameters--?sid=999 will be left.
+        ## that's all that any of the current adapters need or want.
+        return re.sub(r"&.*$","",url)
+    
     ## URL pattern validation is done *after* picking an adaptor based
     ## on domain instead of *as* the adaptor selector so we can offer
     ## the user example(s) for that particular site.

fanficdownloader/epubutils.py

@@ -22,7 +22,7 @@ def get_dcsource_chaptercount(inputio):
 def get_update_data(inputio,
                     getfilecount=True,
                     getsoups=True):
-    epub = ZipFile(inputio, 'r')
+    epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob
 
     ## Find the .opf file.
     container = epub.read("META-INF/container.xml")
@@ -153,7 +153,7 @@ def get_path_part(n):
 def get_story_url_from_html(inputio,_is_good_url=None):
 
     #print("get_story_url_from_html called")
-    epub = ZipFile(inputio, 'r')
+    epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob
 
     ## Find the .opf file.
     container = epub.read("META-INF/container.xml")

fanficdownloader/story.py

@@ -21,6 +21,7 @@ import string
 from math import floor
 from functools import partial
 import logging
+logger = logging.getLogger(__name__)
 import urlparse as up
 
 import exceptions
@@ -29,10 +30,21 @@ from configurable import Configurable
 
 # Create convert_image method depending on which graphics lib we can
 # load.  Preferred: calibre, PIL, none
+        
+imagetypes = {
+    'jpg':'image/jpeg',
+    'jpeg':'image/jpeg',
+    'png':'image/png',
+    'gif':'image/gif',
+    'svg':'image/svg+xml',
+    }
+
 try:
     from calibre.utils.magick import Image
+    convtype = {'jpg':'JPG', 'png':'PNG'}
 
-    def convert_image(url,data,sizes,grayscale):
+    def convert_image(url,data,sizes,grayscale,
+                      removetrans,imgtype="jpg",background='#ffffff'):
         export = False
         img = Image()
         img.load(data)
@@ -44,18 +56,25 @@ try:
             img.size = (nwidth, nheight)
             export = True
             
+        if normalize_format_name(img.format) != imgtype:
+            export = True
+
+        if removetrans and img.has_transparent_pixels():
+            canvas = Image()
+            canvas.create_canvas(int(img.size[0]), int(img.size[1]), str(background))
+            canvas.compose(img)
+            img = canvas
+            export = True
+            
         if grayscale and img.type != "GrayscaleType":
             img.type = "GrayscaleType"
             export = True
 
-        if normalize_format_name(img.format) != "jpg":
-            export = True
-
         if export:
-            return (img.export('JPG'),'jpg','image/jpeg')
+            return (img.export(convtype[imgtype]),imgtype,imagetypes[imgtype])
         else:
-            logging.debug("image used unchanged")
-            return (data,'jpg','image/jpeg')
+            logger.debug("image used unchanged")
+            return (data,imgtype,imagetypes[imgtype])
         
 except:
 
@@ -63,8 +82,9 @@ except:
     try:
         import Image
         from StringIO import StringIO
-        def convert_image(url,data,sizes,grayscale):
-            
+        convtype = {'jpg':'JPEG', 'png':'PNG'}
+        def convert_image(url,data,sizes,grayscale,
+                          removetrans,imgtype="jpg",background='#ffffff'):
             export = False
             img = Image.open(StringIO(data))
             
@@ -75,36 +95,36 @@ except:
                 img = img.resize((nwidth, nheight),Image.ANTIALIAS)
                 export = True
                 
-            if grayscale and img.mode != "L":
-                img = img.convert("L")
-                export = True
-
-            if normalize_format_name(img.format) != "jpg":
+            if normalize_format_name(img.format) != imgtype:
                 if img.mode == "P":
                     # convert pallete gifs to RGB so jpg save doesn't fail.
                     img = img.convert("RGB")
                 export = True
 
+            if removetrans and img.mode == "RGBA":
+                background = Image.new('RGBA', img.size, background)
+                # Paste the image on top of the background
+                background.paste(img, img)
+                img = background.convert('RGB')
+                export = True
+
+            if grayscale and img.mode != "L":
+                img = img.convert("L")
+                export = True
+
             if export:
                 outsio = StringIO()
-                img.save(outsio,'JPEG')
-                return (outsio.getvalue(),'jpg','image/jpeg')
+                img.save(outsio,convtype[imgtype])
+                return (outsio.getvalue(),imgtype,imagetypes[imgtype])
             else:
-                logging.debug("image used unchanged")
-                return (data,'jpg','image/jpeg')
+                logger.debug("image used unchanged")
+                return (data,imgtype,imagetypes[imgtype])
         
     except:
         # No calibre or PIL, simple pass through with mimetype.
-        def convert_image(url,data,sizes,grayscale):
+        def convert_image(url,data,sizes,grayscale,
+                          removetrans,imgtype="jpg",background='#ffffff'):
             return no_convert_image(url,data)
-        
-imagetypes = {
-    'jpg':'image/jpeg',
-    'jpeg':'image/jpeg',
-    'png':'image/png',
-    'gif':'image/gif',
-    'svg':'image/svg+xml',
-    }
 
 ## also used for explicit no image processing.
 def no_convert_image(url,data):
@@ -113,7 +133,7 @@ def no_convert_image(url,data):
     ext=parsedUrl.path[parsedUrl.path.rfind('.')+1:].lower()
     
     if ext not in imagetypes:
-        logging.debug("no_convert_image url:%s - no known extension"%url)
+        logger.debug("no_convert_image url:%s - no known extension"%url)
         # doesn't have extension? use jpg.
         ext='jpg'
         
@@ -240,7 +260,8 @@ class Story(Configurable):
             self.metadata[key]=value
         if key == "language":
             try:
-                self.metadata['langcode'] = langs[self.metadata[key]]
+                # getMetadata not just self.metadata[] to do replace_metadata.
+                self.metadata['langcode'] = langs[self.getMetadata(key)]
             except:
                 self.metadata['langcode'] = 'en'
         if key == 'dateUpdated':
@@ -518,18 +539,27 @@ class Story(Configurable):
             try:
                 if self.getConfig('no_image_processing'):
                     (data,ext,mime) = no_convert_image(imgurl,
-                                                   fetch(imgurl))
+                                                       fetch(imgurl))
                 else:
                     try:
                         sizes = [ int(x) for x in self.getConfigList('image_max_size') ]
                     except Exception, e:
                         raise exceptions.FailedToDownload("Failed to parse image_max_size from personal.ini:%s\nException: %s"%(self.getConfigList('image_max_size'),e))
+                    grayscale = self.getConfig('grayscale_images')
+                    imgtype = self.getConfig('convert_images_to')
+                    if not imgtype:
+                        imgtype = "jpg"
+                    removetrans = self.getConfig('remove_transparency')
+                    removetrans = removetrans or grayscale or imgtype=="jpg"
                     (data,ext,mime) = convert_image(imgurl,
                                                     fetch(imgurl),
                                                     sizes,
-                                                    self.getConfig('grayscale_images'))
+                                                    grayscale,
+                                                    removetrans,
+                                                    imgtype,
+                                                    background="#"+self.getConfig('background_color'))
             except Exception, e:
-                logging.info("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
+                logger.info("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
                 return "failedtoload"
             
             # explicit cover, make the first image.
@@ -564,7 +594,7 @@ class Story(Configurable):
                     ext)
                 self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                 
-            logging.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
+            logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
         else:
             newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc']
             

index.html

@@ -56,7 +56,7 @@
 	  <!-- put announcements here, h3 is a good title size. -->
 	  <h3>Fixes:</h3>
 	  <p>
-	    Minor fixes for fictionalley.org, thehexfiles.net, ponyfictionarchive.net, and potionsandsnitches.net.
+	    Set language to Italian for efpfanfic.net, allow replace_metadata to effect language metadata.
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -66,7 +66,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-32.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-33.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -433,14 +433,15 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks
+extra_valid_entries:fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
 fandoms_label:Fandoms
 freefromtags_label:Freeform Tags
 ao3categories_label:AO3 Categories
 comments_label:Comments
 kudos_label:Kudos
 hits_label:Hits
-bookmarks:Bookmarks
+collections_label:Collections
+bookmarks_label:Bookmarks
 
 ## adds to titlepage_entries instead of replacing it.
 #extra_titlepage_entries: fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks
@@ -534,6 +535,10 @@ extraships:Draco Malfoy/Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/.*?ribbon.gif
+
 ## Some adapters collect additional meta information beyond the
 ## standard ones.  They need to be defined in extra_valid_entries to
 ## tell the rest of the FFDL system about them.  They can be used in