Bump versions, jump PI to 1.7.0

langcode, default PI to 'en'.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-31
+version: 4-4-35
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 6, 16)
+    version             = (1, 7, 0)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/common_utils.py

@@ -196,7 +196,7 @@ class ImageTitleLayout(QHBoxLayout):
     '''
     A reusable layout widget displaying an image followed by a title
     '''
-    def __init__(self, parent, icon_name, title):
+    def __init__(self, parent, icon_name, title, tooltip=None):
         QHBoxLayout.__init__(self)
         title_image_label = QLabel(parent)
         pixmap = get_pixmap(icon_name)
@@ -217,6 +217,9 @@ class ImageTitleLayout(QHBoxLayout):
         self.addWidget(shelf_label)
         self.insertStretch(-1)
 
+        if tooltip:
+            title_image_label.setToolTip(tooltip)
+            shelf_label.setToolTip(tooltip)
 
 class SizePersistedDialog(QDialog):
     '''

calibre-plugin/config.py

@@ -7,20 +7,24 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import traceback, copy
+import traceback, copy, threading
 from collections import OrderedDict
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont, QWidget,
-                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea)
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
+                      QDialogButtonBox )
 
 from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
-    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order)
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getConfigSections
+    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
+            EditTextDialog)
+    
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
+    import (getConfigSections, getNormalStoryURL)
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
@@ -34,6 +38,10 @@ PREFS_KEY_SETTINGS = 'settings'
 # take from here.
 default_prefs = {}
 default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['rejecturls'] = ''
+default_prefs['rejectreasons'] = '''Sucked
+Boring
+Dup from another site'''
 
 default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False
@@ -137,8 +145,79 @@ class PrefsFacade():
     def save_to_db(self):
         set_library_config(self._get_prefs())
         
-
 prefs = PrefsFacade(default_prefs)
+
+class RejectURLList:
+    def __init__(self,prefs):
+        self.prefs = prefs
+        self.sync_lock = threading.RLock()
+        self.listcache = None
+
+    def _read_list_from_text(self,text):
+        cache = {}
+        for line in text.splitlines():
+            if ',' in line:
+                (rejurl,note) = line.split(',',1)
+            else:
+                (rejurl,note) = (line,'')
+            rejurl = getNormalStoryURL(rejurl)
+            if rejurl:
+                cache[rejurl] = note
+        return cache
+        
+
+    def _get_listcache(self):
+        if self.listcache == None:
+            self.listcache = self._read_list_from_text(prefs['rejecturls'])
+        return self.listcache
+
+    def _save_list(self,listcache):
+        rejectlist = []
+        for url in listcache:
+            rejectlist.append("%s,%s"%(url,listcache[url]))
+            
+        self.prefs['rejecturls'] = '\n'.join(rejectlist)
+        self.prefs.save_to_db()
+        self.listcache = None
+        
+    def check(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                note = listcache[url]
+                return note
+            
+            # not found
+            return None
+
+    def remove(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                del listcache[url]
+                self._save_list(listcache)
+
+    def add_text(self,rejecttext):
+        self.add(self._read_list_from_text(rejecttext).items())
+            
+    def add(self,rejectlist,clear=False):
+        # rejectlist=list of (url,note) tuples.
+        with self.sync_lock:
+            if clear:
+                listcache={}
+            else:
+                listcache = self._get_listcache()
+            for (url,note) in rejectlist:
+                listcache[url]=note
+            self._save_list(listcache)
+
+    def get_list(self):
+        return copy.deepcopy(self._get_listcache())
+            
+    def get_reject_reasons(self):
+        return self.prefs['rejectreasons'].splitlines()
+
+rejecturllist = RejectURLList(prefs)
     
 class ConfigWidget(QWidget):
 
@@ -162,6 +241,9 @@ class ConfigWidget(QWidget):
         self.personalini_tab = PersonalIniTab(self, plugin_action)
         tab_widget.addTab(self.personalini_tab, 'personal.ini')
         
+        # self.rejecturls_tab = RejectUrlsTab(self, plugin_action)
+        # tab_widget.addTab(self.rejecturls_tab, 'Reject URLs')
+        
         self.readinglist_tab = ReadingListTab(self, plugin_action)
         tab_widget.addTab(self.readinglist_tab, 'Reading Lists')
         if 'Reading List' not in plugin_action.gui.iactions:
@@ -396,6 +478,27 @@ class BasicTab(QWidget):
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
+        self.l.addSpacing(10)        
+
+        horz = QHBoxLayout()
+        
+        self.rejectlist = QPushButton('Edit Reject URL List', self)
+        self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
+        self.rejectlist.clicked.connect(self.show_rejectlist)
+        horz.addWidget(self.rejectlist)
+        
+        self.reject_urls = QPushButton('Add Reject URLs', self)
+        self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
+        self.reject_urls.clicked.connect(self.add_reject_urls)
+        horz.addWidget(self.reject_urls)
+        
+        self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
+        self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
+        self.reject_reasons.clicked.connect(self.show_reject_reasons)
+        horz.addWidget(self.reject_reasons)
+        
+        self.l.addLayout(horz)
+        
         self.l.insertStretch(-1)
         
     def set_collisions(self):
@@ -411,7 +514,50 @@ class BasicTab(QWidget):
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+
+    def show_rejectlist(self):
+        rejectlist = []
+        for (url,note) in rejecturllist.get_list().items():
+            rejectlist.append((None,url,note,note))
+            
+        d = RejectListDialog(self,
+                             rejectlist,
+                             rejectreasons=rejecturllist.get_reject_reasons(),
+                             header="Edit Reject URLs List",
+                             show_delete=False)
+        d.exec_()
         
+        if d.result() != d.Accepted:
+            return
+        
+        rejectlist=[]
+        for (bookid,url,note) in d.get_reject_list():
+            rejectlist.append((url,note))
+
+        rejecturllist.add(rejectlist,clear=True)
+        
+    def show_reject_reasons(self):
+        d = EditTextDialog(self,
+                           prefs['rejectreasons'],
+                           icon=self.windowIcon(),
+                           title="Reject Reasons",
+                           label="Customize Reject List Reasons",
+                           tooltip="Customize the Reasons presented when Rejecting URLs")
+        d.exec_()
+        if d.result() == d.Accepted:
+            prefs['rejectreasons'] = d.get_plain_text()
+                
+    def add_reject_urls(self):
+        d = EditTextDialog(self,
+                           "http://example.com?story.php?sid=5,Reason why I rejected it",
+                           icon=self.windowIcon(),
+                           title="Add Reject URLs",
+                           label="Add Reject URLs. Use: <b>http://...,note</b>",
+                           tooltip="One URL per line, everything after <b>,</b> will be put in the note.")
+        d.exec_()
+        if d.result() == d.Accepted:
+            rejecturllist.add_text(d.get_plain_text())
+                
 class PersonalIniTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -451,7 +597,7 @@ class PersonalIniTab(QWidget):
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
-
+        
 class ShowDefaultsIniDialog(QDialog):
 
     def __init__(self, icon, text, parent=None):
@@ -916,3 +1062,4 @@ class StandardColumnsTab(QWidget):
             self.l.addLayout(horz)
         
         self.l.insertStretch(-1)
+

calibre-plugin/dialogs.py

@@ -8,15 +8,19 @@ __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import traceback
+from functools import partial
 
 from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, QTextCursor,
-                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
-                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt, QAbstractItemView )
+from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout,
+                      QGridLayout, QPushButton, QProgressDialog, QString, QLabel,
+                      QCheckBox, QIcon, QTextCursor, QTextEdit, QLineEdit, QInputDialog,
+                      QComboBox, QClipboard, QVariant, QProgressDialog, QTimer,
+                      QDialogButtonBox, QPixmap, Qt, QAbstractItemView, SIGNAL,
+                      QTableWidgetItem )
 
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.gui2.complete2 import EditWithComplete
 
 from calibre import confirm_config_name
 from calibre.gui2 import dynamic
@@ -712,3 +716,281 @@ class StoryListTableWidget(QTableWidget):
             self.setItem(dest_row, col, self.takeItem(src_row, col))
         self.removeRow(src_row)
         self.blockSignals(False)
+
+class RejectListTableWidget(QTableWidget):
+
+    def __init__(self, parent,rejectreasons=[]):
+        QTableWidget.__init__(self, parent)
+        self.setSelectionBehavior(QAbstractItemView.SelectRows)
+        self.rejectreasons = rejectreasons
+
+    def on_headersection_clicked(self):
+        self.setSortingEnabled(True)
+        
+    def populate_table(self, reject_list):
+        self.clear()
+        self.setAlternatingRowColors(True)
+        self.setRowCount(len(reject_list))
+        header_labels = ['URL', 'Note']
+        self.setColumnCount(len(header_labels))
+        self.setHorizontalHeaderLabels(header_labels)
+        self.horizontalHeader().setStretchLastSection(True)
+        #self.verticalHeader().setDefaultSectionSize(24)
+        self.verticalHeader().hide()
+
+        # need sortingEnbled to sort, but off to up & down.
+        self.connect(self.horizontalHeader(),
+                     SIGNAL('sectionClicked(int)'),
+                     self.on_headersection_clicked)
+
+        # row is just row number.
+        for row, rejectrow in enumerate(reject_list):
+            self.populate_table_row(row,rejectrow)
+
+        self.resizeColumnsToContents()
+        self.setMinimumColumnWidth(1, 100)
+        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumSize(300, 0)
+
+    def setMinimumColumnWidth(self, col, minimum):
+        if self.columnWidth(col) < minimum:
+            self.setColumnWidth(col, minimum)
+
+    def populate_table_row(self, row, rejectrow):
+
+        (bookid,url,titleauth,oldrejnote) = rejectrow
+        if oldrejnote:
+            noteprefix = note = oldrejnote
+            # incase the existing note ends with one of the known reasons.
+            for reason in self.rejectreasons:
+                if noteprefix.endswith(' - '+reason):
+                    noteprefix = noteprefix[:-len(' - '+reason)]
+                    break
+        else:
+            noteprefix = note = titleauth
+            
+        if len(noteprefix) > 0:
+            noteprefix = noteprefix+' - '
+            
+        url_cell = ReadOnlyTableWidgetItem(url)
+        url_cell.setData(Qt.UserRole, QVariant(bookid))
+        url_cell.setToolTip('URL to add to the Reject List.')
+        self.setItem(row, 0, url_cell)
+
+        note_cell = EditWithComplete(self)
+
+        # This is a more than slightly kludgey way to get
+        # EditWithComplete to *not* alpha-order the reasons, but leave
+        # them in the order entered.  If
+        # calibre.gui2.complete2.CompleteModel.set_items ever changes,
+        # this function will need to also.
+        def complete_model_set_items_kludge(self, items):
+            items = [unicode(x.strip()) for x in items]
+            items = [x for x in items if x]
+            items = tuple(items)
+            self.all_items = self.current_items = items
+            self.current_prefix = ''
+            self.reset()
+
+        note_cell.lineEdit().mcompleter.model().set_items = \
+            partial(complete_model_set_items_kludge,
+                    note_cell.lineEdit().mcompleter.model())
+        
+        items = [note]+[ noteprefix+x for x in self.rejectreasons ]
+        note_cell.update_items_cache(items)
+        note_cell.show_initial_value(note)
+        note_cell.set_separator(None)
+        note_cell.setToolTip('Select or Edit Reject Note.')
+        self.setCellWidget(row, 1, note_cell)
+        
+        # note_cell = QTableWidgetItem(note)
+        # note_cell.setToolTip('Double-click to edit note.')
+        # self.setItem(row, 1, note_cell)
+      
+    def get_reject_list(self):
+        rejectrows = []
+        for row in range(self.rowCount()):
+            bookid = self.item(row, 0).data(Qt.UserRole).toPyObject()
+            url = unicode(self.item(row, 0).text())
+            note = unicode(self.cellWidget(row, 1).currentText()).strip()
+            rejectrows.append((bookid,url,note))
+        return rejectrows
+
+    def remove_selected_rows(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        message = '<p>Are you sure you want to remove this URL from the list?'
+        if len(rows) > 1:
+            message = '<p>Are you sure you want to remove the %d selected URLs from the list?'%len(rows)
+        if not confirm(message,'ffdl_rejectlist_delete_item_again', self):
+            return
+        first_sel_row = self.currentRow()
+        for selrow in reversed(rows):
+            self.removeRow(selrow.row())
+        if first_sel_row < self.rowCount():
+            self.select_and_scroll_to_row(first_sel_row)
+        elif self.rowCount() > 0:
+            self.select_and_scroll_to_row(first_sel_row - 1)
+
+    def select_and_scroll_to_row(self, row):
+        self.selectRow(row)
+        self.scrollToItem(self.currentItem())
+
+    def move_rows_up(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        first_sel_row = rows[0].row()
+        if first_sel_row <= 0:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in selrows:
+            self.swap_row_widgets(selrow - 1, selrow + 1)
+        scroll_to_row = first_sel_row - 1
+        if scroll_to_row > 0:
+            scroll_to_row = scroll_to_row - 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def move_rows_down(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        last_sel_row = rows[-1].row()
+        if last_sel_row == self.rowCount() - 1:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in reversed(selrows):
+            self.swap_row_widgets(selrow + 2, selrow)
+        scroll_to_row = last_sel_row + 1
+        if scroll_to_row < self.rowCount() - 1:
+            scroll_to_row = scroll_to_row + 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def swap_row_widgets(self, src_row, dest_row):
+        self.blockSignals(True)
+        self.setSortingEnabled(False)
+        self.insertRow(dest_row)
+        for col in range(0, self.columnCount()):
+            self.setItem(dest_row, col, self.takeItem(src_row, col))
+        self.removeRow(src_row)
+        self.blockSignals(False)
+
+class RejectListDialog(SizePersistedDialog):
+    def __init__(self, gui, reject_list,
+                 rejectreasons=[],
+                 header="List of Books to Reject",
+                 icon='rotate-right.png',
+                 show_delete=True,
+                 save_size_name='ffdl:reject list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        self.gui = gui
+      
+        self.setWindowTitle(header)
+        self.setWindowIcon(get_icon(icon))
+      
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, icon, header,
+                                        '<i></i>FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.')
+        layout.addLayout(title_layout)
+        rejects_layout = QHBoxLayout()
+        layout.addLayout(rejects_layout)
+
+        self.rejects_table = RejectListTableWidget(self,rejectreasons=rejectreasons)
+        rejects_layout.addWidget(self.rejects_table)
+
+        button_layout = QVBoxLayout()
+        rejects_layout.addLayout(button_layout)
+        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem)
+        # self.move_up_button = QtGui.QToolButton(self)
+        # self.move_up_button.setToolTip('Move selected books up the list')
+        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
+        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
+        # button_layout.addWidget(self.move_up_button)
+        self.remove_button = QtGui.QToolButton(self)
+        self.remove_button.setToolTip('Remove selected URL(s) from the list')
+        self.remove_button.setIcon(get_icon('list_remove.png'))
+        self.remove_button.clicked.connect(self.remove_from_list)
+        button_layout.addWidget(self.remove_button)
+        # self.move_down_button = QtGui.QToolButton(self)
+        # self.move_down_button.setToolTip('Move selected books down the list')
+        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
+        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
+        # button_layout.addWidget(self.move_down_button)
+        spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem1)
+
+        options_layout = QHBoxLayout()
+
+        if show_delete:
+            self.deletebooks = QCheckBox('Delete Books (including books without FanFiction URLs)?',self)
+            self.deletebooks.setToolTip("Delete the selected books after adding them to the Rejected URLs list.")
+            self.deletebooks.setChecked(True)
+            options_layout.addWidget(self.deletebooks)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        options_layout.addWidget(button_box)
+      
+        layout.addLayout(options_layout)
+      
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.rejects_table.populate_table(reject_list)
+
+    def remove_from_list(self):
+        self.rejects_table.remove_selected_rows()
+
+    def get_reject_list(self):
+        return self.rejects_table.get_reject_list()
+
+    def get_deletebooks(self):
+        return self.deletebooks.isChecked()
+
+class EditTextDialog(QDialog):
+
+    def __init__(self, parent, text,
+                 icon=None, title=None, label=None, tooltip=None):
+        QDialog.__init__(self, parent)
+        self.resize(600, 500)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.label = QLabel(label)
+        if title:
+            self.setWindowTitle(title)
+        if icon:
+            self.setWindowIcon(icon)
+        self.l.addWidget(self.label)
+        
+        self.textedit = QTextEdit(self)
+        self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        self.textedit.setText(text)
+        self.l.addWidget(self.textedit)
+
+        if tooltip:
+            self.label.setToolTip(tooltip)
+            self.textedit.setToolTip(tooltip)
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
+
+    def get_plain_text(self):
+        return unicode(self.textedit.toPlainText())

calibre-plugin/ffdl_plugin.py

@@ -41,10 +41,10 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable i
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
 
-from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values)
+from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values, rejecturllist)
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
     AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog,
-    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, 
+    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
     NotGoingToDownload )
 
@@ -190,6 +190,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
                                                                 triggered=self.get_urls_from_page)
 
+            self.reject_list_action = self.create_menu_item_ex(self.menu, 'Reject Selected Books', image='rotate-right.png',
+                                                               triggered=self.reject_list_urls)
+
             # print("platform.system():%s"%platform.system())
             # print("platform.mac_ver()[0]:%s"%platform.mac_ver()[0])
             if not self.check_macmenuhack(): # not platform.mac_ver()[0]: # Some macs crash on these menu items for unknown reasons.
@@ -211,7 +214,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 if menu_id not in self.actions_unique_map:
                     self.gui.keyboard.unregister_shortcut(unique_name)
             self.old_actions_unique_map = self.actions_unique_map
-            self.gui.keyboard.finalize()            
+            self.gui.keyboard.finalize()
 
     def about(self):
         # Get the about text from a file inside the plugin zip file
@@ -238,15 +241,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         #print("create_menu_item_ex after %s"%menu_text)
         return ac
 
+    def is_library_view(self):
+        # 0 = library, 1 = main, 2 = card_a, 3 = card_b
+        return self.gui.stack.currentIndex() == 0
+    
     def plugin_button(self):
-        if len(self.gui.library_view.get_selected_ids()) > 0 and prefs['updatedefault']:
+        if self.is_library_view() and \
+                len(self.gui.library_view.get_selected_ids()) > 0 and \
+                prefs['updatedefault']:
             self.update_existing()
         else:
             self.add_dialog()
 
     def update_lists(self,add=True):
-        if len(self.gui.library_view.get_selected_ids()) > 0 and \
-                (prefs['addtolists'] or prefs['addtoreadlists']) :
+        if prefs['addtolists'] or prefs['addtoreadlists']:
+            if not self.is_library_view():
+                self.gui.status_bar.show_message(_('Cannot Update Reading Lists from Device View'), 3000)
+                return
+        
+            if len(self.gui.library_view.get_selected_ids()) == 0:
+                self.gui.status_bar.show_message(_('No Selected Books to Update Reading Lists'), 3000)
+                return
+
             self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)
 
     def get_urls_from_page(self):
@@ -284,19 +300,35 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
             
     def get_list_urls(self):
-        if len(self.gui.library_view.get_selected_ids()) > 0:
-            book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
+        if self.gui.current_view().selectionModel().selectedRows() == 0 :
+            self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
+                                             3000)
+            return
+            
+        if self.is_library_view():
+            book_list = map( partial(self._convert_id_to_book, good=False),
+                             self.gui.library_view.get_selected_ids() )
 
-            LoopProgressDialog(self.gui,
-                               book_list,
-                               partial(self._get_story_url_for_list, db=self.gui.current_db),
-                               self._finish_get_list_urls,
-                               init_label="Collecting URLs for stories...",
-                               win_title="Get URLs for stories",
-                               status_prefix="URL retrieved")
+        else: # device view, get from epubs on device.
+            view = self.gui.current_view()
+            rows = view.selectionModel().selectedRows()
+            # paths = view.model().paths(rows)
+            book_list = map( partial(self._convert_row_to_book, good=False), rows )
+            
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self._get_story_url_for_list, db=self.gui.current_db),
+                           self._finish_get_list_urls,
+                           init_label="Collecting URLs for stories...",
+                           win_title="Get URLs for stories",
+                           status_prefix="URL retrieved")
             
     def _get_story_url_for_list(self,book,db=None):
-        book['url'] = self._get_story_url(db,book['calibre_id'])
+        if book['calibre_id']:
+            book['url'] = self._get_story_url(db,book_id=book['calibre_id'])
+        elif book['path']:
+            book['url'] = self._get_story_url(db,path=book['path'])
+            
         if book['url'] == None:
             book['good']=False
         else:
@@ -314,6 +346,78 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         show=True,
                         show_copy_button=False)
                 
+    def reject_list_urls(self):
+        if self.is_library_view():
+            book_list = map( partial(self._convert_id_to_book, good=False),
+                             self.gui.library_view.get_selected_ids() )
+
+        else: # device view, get from epubs on device.
+            view = self.gui.current_view()
+            rows = view.selectionModel().selectedRows()
+            #paths = view.model().paths(rows)
+            book_list = map( partial(self._convert_row_to_book, good=False), rows )
+            
+        if len(book_list) == 0 :
+            self.gui.status_bar.show_message(_('No Selected Books have URLs to Reject'), 3000)
+            return
+            
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self._reject_story_url_for_list, db=self.gui.current_db),
+                           self._finish_reject_list_urls,
+                           init_label="Collecting URLs for Reject List...",
+                           win_title="Get URLs for Reject List",
+                           status_prefix="URL retrieved")
+            
+    def _reject_story_url_for_list(self,book,db=None): 
+        if book['calibre_id']:
+            # want title/author, too, for rejects.
+            self._populate_book_from_calibre_id(book,db)
+            book['url'] = self._get_story_url(db,book_id=book['calibre_id'])
+        elif book['path']:
+            book['url'] = self._get_story_url(db,path=book['path'])
+                
+        if book['url'] == None:
+            book['good']=False
+        else:
+            book['good']=True
+            # get existing note, if there is one.
+            book['oldrejnote']=rejecturllist.check(book['url'])
+            
+    def _finish_reject_list_urls(self, book_list):
+
+        # construct reject list of tuples:
+        # (calibre_id, url, "title, authors", old reject note).
+        reject_list = [ ( x['calibre_id'],x['url'],
+                          "%s by %s"%(x['title'],
+                                      ', '.join(x['author'])),
+                          x['oldrejnote'])
+                        for x in book_list if x['good'] ]
+        if reject_list:
+            d = RejectListDialog(self.gui,reject_list,
+                                 rejectreasons=rejecturllist.get_reject_reasons())
+            d.exec_()
+            
+            if d.result() != d.Accepted:
+                return
+
+            bookids=[]
+            rejectlist=[]
+            for (bookid,url,note) in d.get_reject_list():
+                bookids.append(bookid)
+                rejectlist.append((url,note))
+                print("Adding (%s) to Reject List: %s"%(url,note))
+                
+            rejecturllist.add(rejectlist)
+
+            if d.get_deletebooks():
+                self.gui.iactions['Remove Books'].delete_books()
+            
+        else:
+            message="<p>Rejecting FFDL URLs: None of the books selected have FanFiction URLs.</p><p>Proceed to Remove?</p>"
+            if confirm(message,'fanfictiondownloader_reject_non_fanfiction', self.gui):
+                self.gui.iactions['Remove Books'].delete_books()
+                
     def add_dialog(self):
 
         #print("add_dialog()")
@@ -346,7 +450,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         self.start_downloads( options, add_books )
         
     def update_existing(self):
+        if not self.is_library_view():
+            self.gui.status_bar.show_message(_('Cannot Update Books from Device View'), 3000)
+            return
+        
         if len(self.gui.library_view.get_selected_ids()) == 0:
+            self.gui.status_bar.show_message(_('No Selected Books to Update'), 3000)
             return
         #print("update_existing()")
         
@@ -431,6 +540,29 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         necessary data.  To be called from LoopProgressDialog
         'loop'.  Also pops dialogs for is adult, user/pass.
         '''
+
+        url = book['url']
+        print("url:%s"%url)
+
+        rejnote = rejecturllist.check(url)
+        if rejnote:
+            if question_dialog(self.gui, 'Reject URL?',
+                               '<p>Reject URL?</p>'+
+                               '<p>%s is on the Reject URL list:<br />"%s"</p>'%(url,rejnote)+
+                               "<p>Click 'No' to download anyway.</p>",
+                               show_copy_button=False):
+                book['comment'] = "Story on Reject URLs list (%s)."%rejnote
+                book['good']=False
+                book['icon']='rotate-right.png'
+                book['status'] = 'Rejected'
+                return
+            else:
+                if question_dialog(self.gui, 'Remove Reject URL?',
+                                   "<p>Remove URL from Reject List?</p>"+
+                                   '<p>%s is on the Reject URL list:<br />"%s"</p>'%(url,rejnote)+
+                                   "<p>Click 'Yes' to remove it from the list and download,<br /> 'No' to download, but leave it on the Reject list.</p>",
+                                   show_copy_button=False):
+                    rejecturllist.remove(url)
         
         # The current database shown in the GUI
         # db is an instance of the class LibraryDatabase2 from database.py
@@ -447,8 +579,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             # book has already been flagged bad for whatever reason.
             return
         
-        url = book['url']
-        print("url:%s"%url)
         skip_date_update = False
         
         options['personal.ini'] = prefs['personal.ini']
@@ -502,11 +632,6 @@ make_firstimage_cover:true
             book['comments']=''
         book['series'] = story.getMetadata("series", removeallentities=True)
         
-        # adapter.opener is the element with a threadlock.  But del
-        # adapter.opener doesn't work--subproc fails when it tries
-        # to pull in the adapter object that hasn't been imported yet.
-        # book['adapter'] = adapter
-        
         book['is_adult'] = adapter.is_adult
         book['username'] = adapter.username
         book['password'] = adapter.password
@@ -540,8 +665,8 @@ make_firstimage_cover:true
             # 'new' book from URL.  collision handling applies.
             print("from URL(%s)"%url)
 
-            # try to find by identifier url first.
-            searchstr = 'identifiers:"=url:=%s"'%url.replace(":","|")
+            # try to find by identifier url or uri first.
+            searchstr = 'identifiers:"~ur(i|l):=%s"'%url.replace(":","|")
             identicalbooks = db.search_getting_ids(searchstr, None)
             if len(identicalbooks) < 1:
                 # find dups
@@ -904,13 +1029,13 @@ make_firstimage_cover:true
             # mi.tags needs to be list, but set kills dups.
             mi.tags = list(set(list(old_tags)+mi.tags))
 
-        if 'langcode' in book['all_metadata']:
+        if book['all_metadata']['langcode']:
             mi.languages=[book['all_metadata']['langcode']]
         else:
             # Set language english, but only if not already set.
             if not oldmi.languages:
-                mi.languages=['eng']
-
+                mi.languages=['en']
+                
         if options['fileform'] == 'epub' and prefs['updatecover']:
             existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
             epubmi = get_metadata(existingepub,'EPUB')
@@ -1118,12 +1243,6 @@ make_firstimage_cover:true
                 message="<p>You configured FanFictionDownLoader to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?</p>"
                 confirm(message,'fanfictiondownloader_no_send_lists', self.gui)
 
-            # Quick demo of how an 'allow send' list might work.
-            # Issues: allow list per send list?  Naming convention?  "send(allow)"
-            # allow_list = rl_plugin.get_book_list("Allow Send to Device")
-            # # intersection of book_ids & allow_list
-            # add_book_ids = list(set(book_ids) & set(allow_list))
-
             for l in lists:
                 if l in rl_plugin.get_list_names():
                     #print("good send l:(%s)"%l)
@@ -1136,17 +1255,6 @@ make_firstimage_cover:true
                         message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
                         confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)
 
-    # def _find_existing_book_id(self,db,book,matchurl=True):
-    #     mi = MetaInformation(book["title"],book["author"]) # author is a list.
-    #     identicalbooks = db.find_identical_books(mi)
-    #     if matchurl: # only *really* identical if URL matches, too.
-    #         for ib in identicalbooks:
-    #             if self._get_story_url(db,ib) == book['url']:
-    #                 return ib
-    #     if identicalbooks:
-    #         return identicalbooks.pop()
-    #     return None
-    
     def _make_mi_from_book(self,book):
         mi = MetaInformation(book['title'],book['author']) # author is a list.
         mi.set_identifiers({'url':book['url']})
@@ -1208,7 +1316,25 @@ make_firstimage_cover:true
         book['comment'] = ''
         book['url'] = ''
         book['added'] = False
-        
+
+        return book
+            
+    def _convert_row_to_book(self, row, good=True):
+        book = {}
+        mi = self.gui.current_view().model().get_book_display_info(row.row())
+        book['title'] = mi.title
+        book['author'] = mi.authors
+        book['path'] = mi.path
+        book['author_sort'] = mi.author_sort
+        book['good'] = good
+        book['calibre_id'] = None
+        book['begin'] = None
+        book['end'] = None
+            
+        book['comment'] = ''
+        book['url'] = ''
+        book['added'] = False
+
         return book
             
     def _populate_book_from_calibre_id(self, book, db=None):
@@ -1243,44 +1369,46 @@ make_firstimage_cover:true
                 book['icon']='dialog_error.png'
                 book['status'] = 'Bad URL'
     
-    def _get_story_url(self, db, book_id):
-        identifiers = db.get_identifiers(book_id,index_is_id=True)
+    def _get_story_url(self, db, book_id=None, path=None):
+        if book_id == None:
+            identifiers={}
+        else:
+            identifiers = db.get_identifiers(book_id,index_is_id=True)
         if 'url' in identifiers:
             # identifiers have :->| in url.
-            #print("url from book:"+identifiers['url'].replace('|',':'))
+            # print("url from ident url:%s"%identifiers['url'].replace('|',':'))
             return identifiers['url'].replace('|',':')
+        elif 'uri' in identifiers:
+            # identifiers have :->| in uri.
+            # print("uri from ident uri:%s"%identifiers['uri'].replace('|',':'))
+            return identifiers['uri'].replace('|',':')
         else:
-            ## only epub has URL in it--at least where I can easily find it.
-            if db.has_format(book_id,'EPUB',index_is_id=True):
+            existingepub = None
+            if path == None and db.has_format(book_id,'EPUB',index_is_id=True):
                 existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
                 mi = get_metadata(existingepub,'EPUB')
                 identifiers = mi.get_identifiers()
                 if 'url' in identifiers:
-                    #print("url from epub:"+identifiers['url'].replace('|',':'))
+                    # print("url from get_metadata:%s"%identifiers['url'].replace('|',':'))
                     return identifiers['url'].replace('|',':')
-                # look for dc:source first, then scan HTML if 
+            elif path.lower().endswith('.epub'):
+                existingepub = path
+                
+            ## only epub has URL in it--at least where I can easily find it.
+            if existingepub:                
+                # look for dc:source first, then scan HTML if lookforurlinhtml
                 link = get_dcsource(existingepub)
                 if link:
+                    # print("url from get_dcsource:%s"%link)
                     return link
                 elif prefs['lookforurlinhtml']:
-                    return get_story_url_from_html(existingepub,self._is_good_downloader_url)
+                    link = get_story_url_from_html(existingepub,self._is_good_downloader_url)
+                    # print("url from get_story_url_from_html:%s"%link)
+                    return link
         return None
 
     def _is_good_downloader_url(self,url):
-        # this is the accepted way to 'check for existance of a class variable'?  really?
-        try:
-            self.dummyconfig
-        except AttributeError:
-            self.dummyconfig = Configuration("test1.com","EPUB")
-        # pulling up an adapter is pretty low over-head.  If
-        # it fails, it's a bad url.
-        try:
-            adapter = adapters.getAdapter(self.dummyconfig,url)
-            url = adapter.url
-            del adapter
-            return url
-        except:
-            return None;
+        return adapters.getNormalStoryURL(url)
 
 def get_url_list(urls):
     def f(x):

calibre-plugin/jobs.py

@@ -126,6 +126,7 @@ def do_download_for_worker(book,options):
         adapter.is_adult = book['is_adult'] 
         adapter.username = book['username'] 
         adapter.password = book['password']
+        adapter.setChaptersRange(book['begin'],book['end'])
         
         story = adapter.getStoryMetadataOnly()
         if 'calibre_series' in book:
@@ -148,8 +149,6 @@ def do_download_for_worker(book,options):
         elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                 ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
 
-            adapter.setChaptersRange(book['begin'],book['end'])
-            
             print("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))

defaults.ini

@@ -905,6 +905,21 @@ extracategories:InuYasha
 extracharacters:Sesshoumaru,Kagome
 extraships:Sesshoumaru/Kagome
 
+[www.efpfanfic.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:notes,context,type
+notes_label:Notes
+context_label:Context
+type_label:Type of Couple
+
 [www.fanfiction.net]
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.

downloader.py

@@ -158,6 +158,8 @@ def main():
 
        adapter = adapters.getAdapter(configuration,url)
 
+       adapter.setChaptersRange(options.begin,options.end)
+           
        ## Check for include_images and absence of PIL, give warning.
        if adapter.getConfig('include_images'):
            try:
@@ -221,8 +223,6 @@ def main():
            if options.metaonly:
                print adapter.getStoryMetadataOnly()
            
-           adapter.setChaptersRange(options.begin,options.end)
-           
            output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
        
        if not options.metaonly and adapter.getConfig("post_process_cmd"):

fanficdownloader/adapters/__init__.py

@@ -23,6 +23,7 @@ import urlparse as up
 logger = logging.getLogger(__name__)
 
 from .. import exceptions as exceptions
+from ..configurable import Configuration
 
 ## must import each adapter here.
 
@@ -105,6 +106,7 @@ import adapter_bloodtiesfancom
 import adapter_indeathnet
 import adapter_jlaunlimitedcom
 import adapter_qafficcom
+import adapter_efpfanficnet
 
 
 ## This bit of complexity allows adapters to be added by just adding
@@ -124,6 +126,22 @@ for x in imports():
         #print x
         __class_list.append(sys.modules[x].getClass())
 
+def getNormalStoryURL(url):
+    if not getNormalStoryURL.__dummyconfig:
+        getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
+    # pulling up an adapter is pretty low over-head.  If
+    # it fails, it's a bad url.
+    try:
+        adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
+        url = adapter.url
+        del adapter
+        return url
+    except:
+        return None;
+
+# kludgey function static/singleton
+getNormalStoryURL.__dummyconfig = None
+
 def getAdapter(config,url):
 
     logger.debug("trying url:"+url)

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -0,0 +1,312 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return EFPFanFicNet
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class EFPFanFicNet(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','efp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.efpfanfic.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Fai il login e leggi la storia!' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Invia'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?sid='+self.story.getMetadata('storyId')
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if '<a class="menu" href="newaccount.php">' in d : # register for new account link
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+        #     raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'sid' } )
+    	 
+        if select is None:
+    	   # no selector found, so it's a one-chapter story.
+    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')})
+            for o in allOptions:
+                url = u'http://%s/%s' % ( self.getSiteDomain(),
+                                          o['value'])
+                # just in case there's tags, like <i> in chapter titles.
+                title = stripHTML(o)
+                self.chapterUrls.append((title,url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('language','Italian')
+        
+        # normalize story URL to first chapter if later chapter URL was given:
+        url = self.chapterUrls[0][1].replace('&i=1','')
+        logger.debug("Normalizing to URL: "+url)
+        self._setURL(url)
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        storya = None
+        authsoup = None
+        storyblock = None
+        authurl = self.story.getMetadata('authorUrl')
+        
+        ## author can have more than one page of stories.
+        while storyblock == None:
+
+            # no storya, but do have authsoup--we're looping on author pages.
+            if authsoup != None:
+                # last author link with offset should be the 'next' link.
+                authurl = u'http://%s/%s' % ( self.getSiteDomain(),
+                                              authsoup.findAll('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
+            
+            # Need author page for most of the metadata.
+            logger.debug("fetching author page: (%s)"%authurl)
+            authsoup = bs.BeautifulSoup(self._fetchUrl(authurl))
+            #print("authsoup:%s"%authsoup)
+
+            storyas = authsoup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
+            for storya in storyas:
+                #print("======storya:%s"%storya)
+                storyblock = storya.findParent('div',{'class':'storybloc'})
+                #print("======storyblock:%s"%storyblock)
+                if storyblock != None:
+                    continue
+
+        self.setDescription(url,storyblock.find('div', {'class':'introbloc'}))
+
+        noteblock = storyblock.find('div', {'class':'notebloc'})
+        #print("%s"%noteblock)
+        notetext = ("%s" % noteblock).replace("<br />"," |")
+        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />␍
+        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />␍
+        # Categoria: <a href="categories.php?catid=1&amp;parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&amp;parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&amp;a=">3</a> recensioni</div>
+
+        cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+            
+        for item in notetext.split("|"):
+            if ":" in item:
+                (label,value) = item.split(":")
+                label=label.strip()
+                value=value.strip()
+            else:
+                label=value=item.strip()
+
+            if 'Pubblicata' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Aggiornata' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            if label == "Completa":
+                self.story.setMetadata('status', 'Completed')
+            
+            if label == "In corso":
+                self.story.setMetadata('status', 'In-Progress')
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Personaggi' in label:
+                for val in value.split(","):
+                    self.story.addToList('characters',val)
+                    
+            if 'Genere' in label:
+                for val in value.split(","):
+                    self.story.addToList('genre',val)
+                    
+            if 'Coppie' in label:
+                for val in value.split(","):
+                    self.story.addToList('ships',val)
+
+            if 'Avvertimenti' in label:
+                for val in value.split(","):
+                    if val != "None":
+                        self.story.addToList('warnings',val)
+
+            # 'extra' metadata for this adapter:
+                    
+            if 'Tipo di coppia' in label:
+                for val in value.split(","):
+                    self.story.addToList('type',val)
+                    
+            if 'Note' in label:
+                for val in value.split(","):
+                    if val != "None":
+                        self.story.addToList('notes',val)
+                    
+            if 'Contesto' in label:
+                self.story.setMetadata('context', value)
+
+            ## Note--efp doesn't provide word count.
+                
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?ssid=\d+&i=1"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'class' : 'storia'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        # remove any header and 'o:p' tags.
+        for tag in div.findAll("head") + div.findAll("o:p"):
+            tag.extract()
+
+        # change any html and body tags to div.
+        for tag in div.findAll("html") + div.findAll("body"):
+            tag.name='div'
+
+        # remove extra bogus doctype.
+        #<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+        return re.sub(r"<!DOCTYPE[^>]+>","",self.utf8FromSoup(url,div))

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -223,7 +223,16 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             
         if not data or not text:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
+        # not sure how, but we can get html, etc tags still in some
+        # stories.  That breaks later updates because it confuses
+        # epubutils.py
+        for tag in text.findAll('head'):
+            tag.extract()
+        
+        for tag in text.findAll('body') + text.findAll('html'):
+            tag.name = 'div'
+            
         return self.utf8FromSoup(url,text)
 
 def getClass():

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -125,10 +125,12 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata("numChapters", len(self.chapterUrls))
         
         # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
-        # For the sake of bringing it in line with the other adapters, 'Incomplete' and 'On Hiatus' become 'In-Progress'
+        # For the sake of bringing it in line with the other adapters, 'Incomplete' becomes 'In-Progress'
         # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
         # status to leave unchanged.
-        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
+        # Nov2012 - 'On Hiatus' is now passed, too.  It's easy now for users to change/remove if they want
+        # with replace_metadata
+        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("Complete", "Completed")
         self.story.setMetadata("status", status)
         self.story.setMetadata("rating", storyMetadata["content_rating_text"])
             
@@ -144,8 +146,9 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 coverurl = storyMetadata["full_image"]
             else:
                 coverurl = storyMetadata["image"]
-            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
+            if coverurl.startswith('//'): # fix for img urls missing 'http:'
                 coverurl = "http:"+coverurl
+
             self.setCoverImage(self.url,coverurl)
 
         # the fimfic API gives bbcode for desc, not html.

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -237,8 +237,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url)) # otherwise soup eats the br/hr tags.
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -162,12 +162,12 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%b %d %Y"))
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
             
             if 'Updated' in label:
                 # there's a stray [ at the end.
                 #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
 
         try:
             # Find Series name from series URL.

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -147,6 +147,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
             self.story.setMetadata('status', 'In-Progress')
 						
         summary = info.find('td', {'class' : 'summary'})
+        summary.name='div' # change td to div so it doesn't mess up the display when using table titlepage.
         self.setDescription(url,summary)
 		
         rating=stripHTML(info.find('td', {'align' : 'left'})).split('(')[1].split(')')[0]

fanficdownloader/epubutils.py

@@ -22,7 +22,7 @@ def get_dcsource_chaptercount(inputio):
 def get_update_data(inputio,
                     getfilecount=True,
                     getsoups=True):
-    epub = ZipFile(inputio, 'r')
+    epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob
 
     ## Find the .opf file.
     container = epub.read("META-INF/container.xml")
@@ -153,7 +153,7 @@ def get_path_part(n):
 def get_story_url_from_html(inputio,_is_good_url=None):
 
     #print("get_story_url_from_html called")
-    epub = ZipFile(inputio, 'r')
+    epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob
 
     ## Find the .opf file.
     container = epub.read("META-INF/container.xml")

fanficdownloader/story.py

@@ -240,7 +240,8 @@ class Story(Configurable):
             self.metadata[key]=value
         if key == "language":
             try:
-                self.metadata['langcode'] = langs[self.metadata[key]]
+                # getMetadata not just self.metadata[] to do replace_metadata.
+                self.metadata['langcode'] = langs[self.getMetadata(key)]
             except:
                 self.metadata['langcode'] = 'en'
         if key == 'dateUpdated':
@@ -257,31 +258,36 @@ class Story(Configurable):
     ## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp]
     def setReplace(self,replace):
         for line in replace.splitlines():
+            (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None)
             if "&&" in line:
                 (line,conditional) = map( lambda x: x.strip(), line.split("&&") )
-                condparts = map( lambda x: x.strip(), conditional.split("=>") )
-            else:
-                condparts=[None,None]
+                (condkey,condregexp) = map( lambda x: x.strip(), conditional.split("=>") )
             if "=>" in line:
                 parts = map( lambda x: x.strip(), line.split("=>") )
                 if len(parts) > 2:
-                    parts[0] = map( lambda x: x.strip(), parts[0].split(",") )
-                    self.replacements.append(parts+condparts)
+                    metakeys = map( lambda x: x.strip(), parts[0].split(",") )
+                    (regexp,replacement)=parts[1:]
                 else:
-                    self.replacements.append([None]+parts+condparts)
+                    (regexp,replacement)=parts
+
+            if regexp:
+                regexp = re.compile(regexp)
+                if condregexp:
+                    condregexp = re.compile(condregexp)
+                self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
     
     def doReplacments(self,value,key):
-        for (keys,regexp,replacement,condkey,condregexp) in self.replacements:
-            if (keys == None or key in keys) \
+        for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
+            if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
-                    and re.search(regexp,value):
+                    and regexp.search(value):
                 doreplace=True
                 if condkey and condkey != key: # prevent infinite recursion.
                     condval = self.getMetadata(condkey)
-                    doreplace = condval != None and re.search(condregexp,condval)
+                    doreplace = condval != None and condregexp.search(condval)
                     
                 if doreplace:
-                    value = re.sub(regexp,replacement,value)
+                    value = regexp.sub(replacement,value)
         return value
         
     def getMetadataRaw(self,key):

index.html

@@ -54,21 +54,9 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>New Sites:</h3>
+	  <h3>Fixes:</h3>
 	  <p>
-	    Two new sites thanks to besnef:
-	    <ul>
-	      <li><a href="http://indeath.net">indeath.net</a></li>
-	      <li><a href="http://www.jlaunlimited.com">www.jlaunlimited.com</a></li>
-	    </ul>
-	  </p>
-	  <h3>New Fixes:</h3>
-	  <p>
-	    <ul>
-	      <li>Fixes for changes to fanfiktion.de formatting.</li>
-	      <li>Fix for bad author on twiwritenet with some skins.</li>
-	      <li>Change from www.ncisfiction.com to www.ncisfiction.net due to the ncisfiction.com domain expiring.</li>
-	    </ul>
+	    Set language to Italian for efpfanfic.net, allow replace_metadata to effect language metadata.
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -78,7 +66,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-28.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-33.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -570,8 +558,12 @@
 	    Use the URL of the story's first chapter, such as
 	    <br /><a href="http://www.jlaunlimited.com/eFiction1.1/viewstory.php?sid=1234">http://www.jlaunlimited.com/eFiction1.1/viewstory.php?sid=1234</a>
 	  </dd>
+	  <dt>www.efpfanfic.net</dt>
+	  <dd>
+	    Use the URL of any story chapter, such as
+	    <br /><a href="http://www.efpfanfic.net/viewstory.php?sid=12345">http://www.efpfanfic.net/viewstory.php?sid=12345</a>
+	  </dd>
 	</dl>
-		    
 
 	<p>
 	  A few additional things to know, which will make your life substantially easier:

plugin-defaults.ini

@@ -896,6 +896,21 @@ extracategories:InuYasha
 extracharacters:Sesshoumaru,Kagome
 extraships:Sesshoumaru/Kagome
 
+[www.efpfanfic.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:notes,context,type
+notes_label:Notes
+context_label:Context
+type_label:Type of Couple
+
 [www.fanfiction.net]
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.