Update CLI download zip. - Version reset to 2.0.00.

to Windows-1252, and issue 78, chapterless fimf stories.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-5-06
+version: 2-0-00
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 8, 25)
+    version             = (2, 0, 0)
     minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/common_utils.py

@@ -8,12 +8,19 @@ __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import os
-from PyQt4 import QtGui
-from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
-                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
-                      QTextEdit,
-                      QListWidget, QAbstractItemView)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+    
 from calibre.constants import iswindows
 from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
 from calibre.gui2.actions import menu_action_unique_name

calibre-plugin/config.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging
@@ -13,10 +13,31 @@ logger = logging.getLogger(__name__)
 import traceback, copy, threading
 from collections import OrderedDict
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
-                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
-                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
-                      QDialogButtonBox, QGroupBox )
+try:
+    from PyQt5.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                          QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
+                          QDialogButtonBox, QGroupBox )
+except ImportError as e:
+    from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                          QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
+                          QDialogButtonBox, QGroupBox )
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()
 
 from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog
@@ -60,7 +81,7 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
 
-from calibre.gui2.complete import MultiCompleteLineEdit
+from calibre.gui2.complete2 import EditWithComplete  #MultiCompleteLineEdit
 
 class RejectURLList:
     def __init__(self,prefs):
@@ -220,6 +241,7 @@ class ConfigWidget(QWidget):
         prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
         prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
+        prefs['reject_always'] = self.basic_tab.reject_always.isChecked()
 
         if self.readinglist_tab:
             # lists
@@ -243,7 +265,7 @@ class ConfigWidget(QWidget):
         prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
         gc_site_settings = {}
         for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
             if val != 'none':
                 gc_site_settings[site] = val
                 #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
@@ -275,12 +297,12 @@ class ConfigWidget(QWidget):
 
         # Custom Columns tab
         # error column
-        prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())
+        prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
 
         # cust cols tab
         colsmap = {}
         for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
             if val != 'none':
                 colsmap[col] = val
                 #print("colsmap[%s]:%s"%(col,colsmap[col]))
@@ -482,6 +504,11 @@ class BasicTab(QWidget):
         self.reject_reasons.clicked.connect(self.show_reject_reasons)
         self.l.addWidget(self.reject_reasons)
 
+        self.reject_always = QCheckBox(_('Reject Without Confirmation?'),self)
+        self.reject_always.setToolTip(_("Always reject URLs on the Reject List without stopping and asking."))
+        self.reject_always.setChecked(prefs['reject_always'])
+        self.l.addWidget(self.reject_always)
+
         topl.addWidget(defs_gb)
 
         horz = QHBoxLayout()
@@ -649,7 +676,7 @@ class ReadingListTab(QWidget):
         label = QLabel(_('"Send to Device" Reading Lists'))
         label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
-        self.send_lists_box = MultiCompleteLineEdit(self)
+        self.send_lists_box = EditWithComplete(self)
         self.send_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.send_lists_box.update_items_cache(reading_lists)
         self.send_lists_box.setText(prefs['send_lists'])
@@ -665,7 +692,7 @@ class ReadingListTab(QWidget):
         label = QLabel(_('"To Read" Reading Lists'))
         label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
-        self.read_lists_box = MultiCompleteLineEdit(self)
+        self.read_lists_box = EditWithComplete(self)
         self.read_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.read_lists_box.update_items_cache(reading_lists)
         self.read_lists_box.setText(prefs['read_lists'])
@@ -727,17 +754,17 @@ class GenerateCoverTab(QWidget):
             horz.addWidget(label)
             dropdown = QComboBox(self)
             dropdown.setToolTip(s)
-            dropdown.addItem('',QVariant('none'))
+            dropdown.addItem('','none')
             for setting in gc_settings:
-                dropdown.addItem(setting,QVariant(setting))
+                dropdown.addItem(setting,setting)
             if site == _("Default"):
                 self.gc_dropdowns["Default"] = dropdown
                 if 'Default' in prefs['gc_site_settings']:
-                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings']['Default'])))
+                    dropdown.setCurrentIndex(dropdown.findData(prefs['gc_site_settings']['Default']))
             else:
                 self.gc_dropdowns[site] = dropdown
             if site in prefs['gc_site_settings']:
-                dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings'][site])))
+                dropdown.setCurrentIndex(dropdown.findData(prefs['gc_site_settings'][site]))
 
             horz.addWidget(dropdown)
             self.sl.addLayout(horz)
@@ -966,12 +993,12 @@ class CustomColumnsTab(QWidget):
                 label.setToolTip(_("Update this %s column(%s) with...")%(key,column['datatype']))
                 horz.addWidget(label)
                 dropdown = QComboBox(self)
-                dropdown.addItem('',QVariant('none'))
+                dropdown.addItem('','none')
                 for md in permitted_values[column['datatype']]:
-                    dropdown.addItem(titleLabels[md],QVariant(md))
+                    dropdown.addItem(titleLabels[md],md)
                 self.custcol_dropdowns[key] = dropdown
                 if key in prefs['custom_cols']:
-                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
+                    dropdown.setCurrentIndex(dropdown.findData(prefs['custom_cols'][key]))
                 if column['datatype'] == 'enumeration':
                     dropdown.setToolTip(_("Metadata values valid for this type of column.")+"\n"+_("Values that aren't valid for this enumeration column will be ignored."))
                 else:
@@ -1007,11 +1034,11 @@ class CustomColumnsTab(QWidget):
         horz.addWidget(label)
         self.errorcol = QComboBox(self)
         self.errorcol.setToolTip(tooltip)
-        self.errorcol.addItem('',QVariant('none'))
+        self.errorcol.addItem('','none')
         for key, column in custom_columns.iteritems():
             if column['datatype'] in ('text','comments'):
-                self.errorcol.addItem(column['name'],QVariant(key))
-        self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
+                self.errorcol.addItem(column['name'],key)
+        self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
         horz.addWidget(self.errorcol)
         self.l.addLayout(horz)
 

calibre-plugin/dialogs.py

@@ -19,12 +19,36 @@ logger = logging.getLogger(__name__)
 import urllib
 import email
 
-from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
-                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
-                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal,
-                      QGroupBox, QFrame)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame)
+
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
@@ -146,19 +170,6 @@ class RejectUrlEntry:
             
         return retval
 
-# This is a more than slightly kludgey way to get
-# EditWithComplete to *not* alpha-order the reasons, but leave
-# them in the order entered.  If
-# calibre.gui2.complete2.CompleteModel.set_items ever changes,
-# this function will need to also.
-def complete_model_set_items_kludge(self, items):
-    items = [unicode(x.strip()) for x in items]
-    items = [x for x in items if x]
-    items = tuple(items)
-    self.all_items = self.current_items = items
-    self.current_prefix = ''
-    self.reset()
-    
 class NotGoingToDownload(Exception):
     def __init__(self,error,icon='dialog_error.png'):
         self.error=error
@@ -196,9 +207,9 @@ class DroppableQTextEdit(QTextEdit):
                             urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                 else:
                     urllist.extend(get_urls_from_text("%s"%msg))
-        
         if urllist:
             self.append("\n".join(urllist))
+            return None
         return QTextEdit.dropEvent(self,event)
         
     def canInsertFromMimeData(self, source):
@@ -559,7 +570,7 @@ class LoopProgressDialog(QProgressDialog):
                  status_prefix=_("Fetched metadata for")):
         QProgressDialog.__init__(self,
                                  init_label,
-                                 QString(), 0, len(book_list), gui)
+                                 _('Cancel'), 0, len(book_list), gui)
         self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
         self.book_list = book_list
@@ -829,11 +840,11 @@ class StoryListTableWidget(QTableWidget):
             icon = get_icon(book['icon'])
 
         status_cell = IconWidgetItem(None,icon,val)
-        status_cell.setData(Qt.UserRole, QVariant(val))
+        status_cell.setData(Qt.UserRole, val)
         self.setItem(row, 0, status_cell)
         
         title_cell = ReadOnlyTableWidgetItem(book['title'])
-        title_cell.setData(Qt.UserRole, QVariant(row))
+        title_cell.setData(Qt.UserRole, row)
         self.setItem(row, 1, title_cell)
         
         self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
@@ -848,7 +859,7 @@ class StoryListTableWidget(QTableWidget):
         books = []
         #print("=========================\nbooks:%s"%self.books)
         for row in range(self.rowCount()):
-            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
+            rnum = convert_qvariant(self.item(row, 1).data(Qt.UserRole))
             book = self.books[rnum]
             books.append(book)
         return books
@@ -914,15 +925,12 @@ class RejectListTableWidget(QTableWidget):
     def populate_table_row(self, row, rej):
 
         url_cell = ReadOnlyTableWidgetItem(rej.url)
-        url_cell.setData(Qt.UserRole, QVariant(rej.book_id))
+        url_cell.setData(Qt.UserRole, rej.book_id)
         self.setItem(row, 0, url_cell)
         self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
         self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
         
-        note_cell = EditWithComplete(self)
-        note_cell.lineEdit().mcompleter.model().set_items = \
-            partial(complete_model_set_items_kludge,
-                    note_cell.lineEdit().mcompleter.model())
+        note_cell = EditWithComplete(self,sort_func=lambda x:1)
         
         items = [rej.note]+self.rejectreasons
         note_cell.update_items_cache(items)
@@ -992,10 +1000,7 @@ class RejectListDialog(SizePersistedDialog):
         button_layout.addItem(spacerItem1)
 
         if show_all_reasons:
-            self.reason_edit = EditWithComplete(self)
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
             
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)
@@ -1037,7 +1042,7 @@ class RejectListDialog(SizePersistedDialog):
         rejectrows = []
         for row in range(self.rejects_table.rowCount()):
             url = unicode(self.rejects_table.item(row, 0).text()).strip()
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id =convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
             title = unicode(self.rejects_table.item(row, 1).text()).strip()
             auth = unicode(self.rejects_table.item(row, 2).text()).strip()
             note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
@@ -1047,7 +1052,7 @@ class RejectListDialog(SizePersistedDialog):
     def get_reject_list_ids(self):
         rejectrows = []
         for row in range(self.rejects_table.rowCount()):
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id = convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
             if book_id:
                 rejectrows.append(book_id)
         return rejectrows
@@ -1089,11 +1094,7 @@ class EditTextDialog(QDialog):
             self.textedit.setToolTip(tooltip)
 
         if rejectreasons or reasonslabel:
-            self.reason_edit = EditWithComplete(self)
-
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
             
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)

calibre-plugin/ffdl_plugin.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging
@@ -19,10 +19,12 @@ import urllib
 import email
 import traceback
 
-from PyQt4.Qt import (QApplication, QMenu, QToolButton, QTimer)
-
-from PyQt4.Qt import QPixmap, Qt
-from PyQt4.QtCore import QBuffer
+try:
+    from PyQt5.Qt import (QApplication, QMenu, QTimer)
+    from PyQt5.QtCore import QBuffer
+except ImportError as e:
+    from PyQt4.Qt import (QApplication, QMenu, QTimer)
+    from PyQt4.QtCore import QBuffer
 
 from calibre.constants import numeric_version as calibre_version
 
@@ -207,7 +209,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 #print("text/plain:%s"%event.mimeData().data(mimetype))
                 urllist.extend(get_urls_from_text(event.mimeData().data(mimetype)))
 
-        #print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
+        # print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
         if urllist or dropped_ids:
             QTimer.singleShot(1, partial(self.do_drop,
                                          dropped_ids=dropped_ids,
@@ -764,7 +766,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if not merge: # skip reject list when merging.
             if rejecturllist.check(url):
                 rejnote = rejecturllist.get_full_note(url)
-                if question_dialog(self.gui, _('Reject URL?'),'''
+                if prefs['reject_always'] or question_dialog(self.gui, _('Reject URL?'),'''
                           <h3>%s</h3>
                           <p>%s</p>
                           <p>"<b>%s</b>"</p>

calibre-plugin/jobs.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 

calibre-plugin/prefs.py

@@ -25,6 +25,7 @@ default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
 Dup from another site'''
+default_prefs['reject_always'] = False
 
 default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False

defaults.ini

@@ -634,7 +634,7 @@ extracategories:The Sentinel
 ## explicitly set the encoding and order if you need to.  The special
 ## value 'auto' will call chardet and use the encoding it reports if
 ## it has +90% confidence.  'auto' is not reliable.
-website_encodings:ISO-8859-1,auto
+website_encodings:Windows-1252,ISO-8859-1,auto
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1383,8 +1383,11 @@ type_label:Type of Couple
 [www.fanfiction.net]
 user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
-## Change this to false to use them anyway.
-never_make_cover: true
+## Set this to true to never use them.
+#never_make_cover: false
+
+## fanfiction.net shows the user's 
+cover_exclusion_regexp:/imageu/
 
 ## fanfiction.net is blocking people more aggressively.  If you
 ## download fewer stories less often you can likely get by with
@@ -1808,6 +1811,9 @@ extracharacters:Wolverine,Rogue
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
 
+extra_valid_entries:reviews
+reviews_label:Reviews
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -74,7 +74,8 @@ class EFPFanFicNet(BaseSiteAdapter):
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
-        if 'Fai il login e leggi la storia!' in data:
+        if( 'Fai il login e leggi la storia!' in data or
+            'Questa storia presenta contenuti non adatti ai minori' in data ):
             return True
         else:
             return False

fanficdownloader/adapters/adapter_literotica.py

@@ -201,8 +201,10 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('description', soup1.find('meta', {'name': 'description'})['content'])
         
         # li tags inside div class b-s-story-tag-list
-        for li in soup1.find('div', {'class':'b-s-story-tag-list'}).findAll('a'):
-            self.story.addToList('eroticatags',stripHTML(li))
+        taglist = soup1.find('div', {'class':'b-s-story-tag-list'})
+        if taglist:
+            for li in taglist.findAll('a'):
+                self.story.addToList('eroticatags',stripHTML(li))
 
         return
 

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -70,14 +70,14 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
-        if 'Free Registration' in data \
+        if self.needToLogin \
+                or 'Free Registration' in data \
                 or "Invalid Password!" in data \
                 or "Invalid User Name!" in data \
                 or "Log In" in data \
                 or "Access to unlinked chapters requires" in data:
-            return True
-        else:
-            return False
+            self.needToLogin = True
+        return self.needToLogin
         
     def performLogin(self, url):
         params = {}
@@ -114,11 +114,15 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         url = self.url
         logger.debug("URL: "+url)
 
+        self.needToLogin = False
         try:
             data = self._fetchUrl(url+":i")
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
+            elif e.code == 401:
+                self.needToLogin = True
+                data = ''
             else:
                 raise e
 

fanficdownloader/adapters/adapter_wraithbaitcom.py

@@ -125,6 +125,10 @@ class WraithBaitComAdapter(BaseSiteAdapter):
         rating=pt.text.split('[')[1].split(']')[0]
         self.story.setMetadata('rating', rating)
 
+        st = soup.find('div', {'class' : 'storytitle'})
+        a = st.findAll('a', href=re.compile(r'reviews.php\?type=ST&item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
+        self.story.setMetadata('reviews',stripHTML(a))
+        
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # include author on chapters if multiple authors.

fanficdownloader/adapters/base_adapter.py

@@ -328,7 +328,8 @@ class BaseSiteAdapter(Configurable):
 
     def setCoverImage(self,storyurl,imgurl):
         if self.getConfig('include_images'):
-            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True)
+            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True,
+                                 coverexclusion=self.getConfig('cover_exclusion_regexp'))
 
     # This gives us a unicode object, not just a string containing bytes.
     # (I gave soup a unicode string, you'd think it could give it back...)

fanficdownloader/epubutils.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging

fanficdownloader/story.py

@@ -324,7 +324,7 @@ class Story(Configurable):
                 self.in_ex_cludes[ie] = self.set_in_ex_clude(ies)
 
     def join_list(self, key, vallist):
-        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(vallist)
+        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist))
                 
     def setMetadata(self, key, value, condremoveentities=True):
 
@@ -345,7 +345,7 @@ class Story(Configurable):
             except:
                 self.setMetadata('langcode','en')
                 
-        if key == 'dateUpdated':
+        if key == 'dateUpdated' and value:
             # Last Update tags for Bill.
             self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
@@ -649,7 +649,7 @@ class Story(Configurable):
         # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
         # use replace_metadata to change separator first if needed.
         # ships=>[ ]*(/|&|&)[ ]*=>/
-        if listname == 'ships' and self.getConfig('sort_ships'):
+        if listname == 'ships' and self.getConfig('sort_ships') and retlist:
             retlist = [ '/'.join(sorted(x.split('/'))) for x in retlist ]
 
         if retlist:
@@ -768,6 +768,10 @@ class Story(Configurable):
                      '','',''))
                 #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
 
+        # apply coverexclusion to explicit covers, too.  Primarily for ffnet imageu.
+        if cover and coverexclusion and re.search(coverexclusion,imgurl):
+            return
+
         prefix='ffdl'
         if imgurl not in self.imgurls:
             parsedUrl = urlparse.urlparse(imgurl)
@@ -799,7 +803,7 @@ class Story(Configurable):
                 return "failedtoload"
 
             # explicit cover, make the first image.
-            if cover and not self.getConfig('never_make_cover'):
+            if cover:
                 if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']:
                     # remove existing cover, if there is one.
                     del self.imgurls[0]
@@ -818,7 +822,7 @@ class Story(Configurable):
                 if self.cover == None and \
                         self.getConfig('make_firstimage_cover') and \
                         not self.getConfig('never_make_cover') and \
-                        (not coverexclusion or not re.search(coverexclusion,imgurl)):
+                        not (coverexclusion and re.search(coverexclusion,imgurl)):
                     newsrc = "images/cover.%s"%ext
                     self.cover=newsrc
                     self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})

index.html

@@ -60,10 +60,9 @@
 	  </p>
 	  <p>
 	    <ul>
-              <li>New site: fictionmania.tv -- Thanks, cryzed!</li>
-              <li>Fix for site recognizer to handle with/without www. using https URLs.</li>
-              <li>Fix for some utf8 descriptions on fimf.</li>
-              <li>Site specific metadata 'eroticatags' for literotica.com.</li>
+              <li>Known issue: Password protected FimFiction.net stories aren't working.  FimF changed API access.</li>
+              <li>Known issue: Specific metadata 'eroticatags' for literotica.com doesn't work on all stories.</li>
+              <li>Reset version number to align with caliber plugin version.</li>
 	    </ul>
 	  </p>
 	  <p>
@@ -74,7 +73,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-5-05.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-5-07.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

makezip.py

@@ -1,9 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-# epubmerge.py 1.0
-
-# Copyright 2011, Jim Miller
+# Copyright 2014, Jim Miller
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

plugin-defaults.ini

@@ -610,7 +610,7 @@ extracategories:The Sentinel
 ## explicitly set the encoding and order if you need to.  The special
 ## value 'auto' will call chardet and use the encoding it reports if
 ## it has +90% confidence.  'auto' is not reliable.
-website_encodings:ISO-8859-1,auto
+website_encodings:Windows-1252,ISO-8859-1,auto
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1377,8 +1377,11 @@ type_label:Type of Couple
 [www.fanfiction.net]
 user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
-## Change this to false to use them anyway.
-never_make_cover: true
+## Set this to true to never use them.
+#never_make_cover: false
+
+## fanfiction.net shows the user's 
+cover_exclusion_regexp:/imageu/
 
 ## fanfiction.net is blocking people more aggressively.  If you
 ## download fewer stories less often you can likely get by with
@@ -1808,6 +1811,9 @@ extracharacters:Wolverine,Rogue
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
 
+extra_valid_entries:reviews
+reviews_label:Reviews
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For