Update CLI download zip.

to Windows-1252, and issue 78, chapterless fimf stories.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-5-04
+version: 2-0-01
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 8, 23)
+    version             = (2, 0, 1)
     minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/common_utils.py

@@ -8,12 +8,19 @@ __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import os
-from PyQt4 import QtGui
-from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
-                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
-                      QTextEdit,
-                      QListWidget, QAbstractItemView)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+    
 from calibre.constants import iswindows
 from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
 from calibre.gui2.actions import menu_action_unique_name

calibre-plugin/config.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging
@@ -13,10 +13,31 @@ logger = logging.getLogger(__name__)
 import traceback, copy, threading
 from collections import OrderedDict
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
-                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
-                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
-                      QDialogButtonBox, QGroupBox )
+try:
+    from PyQt5.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                          QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
+                          QDialogButtonBox, QGroupBox )
+except ImportError as e:
+    from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                          QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
+                          QDialogButtonBox, QGroupBox )
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()
 
 from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog
@@ -60,7 +81,7 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
 
-from calibre.gui2.complete import MultiCompleteLineEdit
+from calibre.gui2.complete2 import EditWithComplete  #MultiCompleteLineEdit
 
 class RejectURLList:
     def __init__(self,prefs):
@@ -220,6 +241,7 @@ class ConfigWidget(QWidget):
         prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
         prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
+        prefs['reject_always'] = self.basic_tab.reject_always.isChecked()
 
         if self.readinglist_tab:
             # lists
@@ -243,7 +265,7 @@ class ConfigWidget(QWidget):
         prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
         gc_site_settings = {}
         for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
             if val != 'none':
                 gc_site_settings[site] = val
                 #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
@@ -275,12 +297,12 @@ class ConfigWidget(QWidget):
 
         # Custom Columns tab
         # error column
-        prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())
+        prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
 
         # cust cols tab
         colsmap = {}
         for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
             if val != 'none':
                 colsmap[col] = val
                 #print("colsmap[%s]:%s"%(col,colsmap[col]))
@@ -482,6 +504,11 @@ class BasicTab(QWidget):
         self.reject_reasons.clicked.connect(self.show_reject_reasons)
         self.l.addWidget(self.reject_reasons)
 
+        self.reject_always = QCheckBox(_('Reject Without Confirmation?'),self)
+        self.reject_always.setToolTip(_("Always reject URLs on the Reject List without stopping and asking."))
+        self.reject_always.setChecked(prefs['reject_always'])
+        self.l.addWidget(self.reject_always)
+
         topl.addWidget(defs_gb)
 
         horz = QHBoxLayout()
@@ -649,7 +676,7 @@ class ReadingListTab(QWidget):
         label = QLabel(_('"Send to Device" Reading Lists'))
         label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
-        self.send_lists_box = MultiCompleteLineEdit(self)
+        self.send_lists_box = EditWithComplete(self)
         self.send_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.send_lists_box.update_items_cache(reading_lists)
         self.send_lists_box.setText(prefs['send_lists'])
@@ -665,7 +692,7 @@ class ReadingListTab(QWidget):
         label = QLabel(_('"To Read" Reading Lists'))
         label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
-        self.read_lists_box = MultiCompleteLineEdit(self)
+        self.read_lists_box = EditWithComplete(self)
         self.read_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.read_lists_box.update_items_cache(reading_lists)
         self.read_lists_box.setText(prefs['read_lists'])
@@ -727,17 +754,17 @@ class GenerateCoverTab(QWidget):
             horz.addWidget(label)
             dropdown = QComboBox(self)
             dropdown.setToolTip(s)
-            dropdown.addItem('',QVariant('none'))
+            dropdown.addItem('','none')
             for setting in gc_settings:
-                dropdown.addItem(setting,QVariant(setting))
+                dropdown.addItem(setting,setting)
             if site == _("Default"):
                 self.gc_dropdowns["Default"] = dropdown
                 if 'Default' in prefs['gc_site_settings']:
-                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings']['Default'])))
+                    dropdown.setCurrentIndex(dropdown.findData(prefs['gc_site_settings']['Default']))
             else:
                 self.gc_dropdowns[site] = dropdown
             if site in prefs['gc_site_settings']:
-                dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings'][site])))
+                dropdown.setCurrentIndex(dropdown.findData(prefs['gc_site_settings'][site]))
 
             horz.addWidget(dropdown)
             self.sl.addLayout(horz)
@@ -966,12 +993,12 @@ class CustomColumnsTab(QWidget):
                 label.setToolTip(_("Update this %s column(%s) with...")%(key,column['datatype']))
                 horz.addWidget(label)
                 dropdown = QComboBox(self)
-                dropdown.addItem('',QVariant('none'))
+                dropdown.addItem('','none')
                 for md in permitted_values[column['datatype']]:
-                    dropdown.addItem(titleLabels[md],QVariant(md))
+                    dropdown.addItem(titleLabels[md],md)
                 self.custcol_dropdowns[key] = dropdown
                 if key in prefs['custom_cols']:
-                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
+                    dropdown.setCurrentIndex(dropdown.findData(prefs['custom_cols'][key]))
                 if column['datatype'] == 'enumeration':
                     dropdown.setToolTip(_("Metadata values valid for this type of column.")+"\n"+_("Values that aren't valid for this enumeration column will be ignored."))
                 else:
@@ -1007,11 +1034,11 @@ class CustomColumnsTab(QWidget):
         horz.addWidget(label)
         self.errorcol = QComboBox(self)
         self.errorcol.setToolTip(tooltip)
-        self.errorcol.addItem('',QVariant('none'))
+        self.errorcol.addItem('','none')
         for key, column in custom_columns.iteritems():
             if column['datatype'] in ('text','comments'):
-                self.errorcol.addItem(column['name'],QVariant(key))
-        self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
+                self.errorcol.addItem(column['name'],key)
+        self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
         horz.addWidget(self.errorcol)
         self.l.addLayout(horz)
 

calibre-plugin/dialogs.py

@@ -19,12 +19,36 @@ logger = logging.getLogger(__name__)
 import urllib
 import email
 
-from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
-                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
-                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal,
-                      QGroupBox, QFrame)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame)
+
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
@@ -146,19 +170,6 @@ class RejectUrlEntry:
             
         return retval
 
-# This is a more than slightly kludgey way to get
-# EditWithComplete to *not* alpha-order the reasons, but leave
-# them in the order entered.  If
-# calibre.gui2.complete2.CompleteModel.set_items ever changes,
-# this function will need to also.
-def complete_model_set_items_kludge(self, items):
-    items = [unicode(x.strip()) for x in items]
-    items = [x for x in items if x]
-    items = tuple(items)
-    self.all_items = self.current_items = items
-    self.current_prefix = ''
-    self.reset()
-    
 class NotGoingToDownload(Exception):
     def __init__(self,error,icon='dialog_error.png'):
         self.error=error
@@ -196,9 +207,9 @@ class DroppableQTextEdit(QTextEdit):
                             urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                 else:
                     urllist.extend(get_urls_from_text("%s"%msg))
-        
         if urllist:
             self.append("\n".join(urllist))
+            return None
         return QTextEdit.dropEvent(self,event)
         
     def canInsertFromMimeData(self, source):
@@ -559,7 +570,7 @@ class LoopProgressDialog(QProgressDialog):
                  status_prefix=_("Fetched metadata for")):
         QProgressDialog.__init__(self,
                                  init_label,
-                                 QString(), 0, len(book_list), gui)
+                                 _('Cancel'), 0, len(book_list), gui)
         self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
         self.book_list = book_list
@@ -829,11 +840,11 @@ class StoryListTableWidget(QTableWidget):
             icon = get_icon(book['icon'])
 
         status_cell = IconWidgetItem(None,icon,val)
-        status_cell.setData(Qt.UserRole, QVariant(val))
+        status_cell.setData(Qt.UserRole, val)
         self.setItem(row, 0, status_cell)
         
         title_cell = ReadOnlyTableWidgetItem(book['title'])
-        title_cell.setData(Qt.UserRole, QVariant(row))
+        title_cell.setData(Qt.UserRole, row)
         self.setItem(row, 1, title_cell)
         
         self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
@@ -848,7 +859,7 @@ class StoryListTableWidget(QTableWidget):
         books = []
         #print("=========================\nbooks:%s"%self.books)
         for row in range(self.rowCount()):
-            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
+            rnum = convert_qvariant(self.item(row, 1).data(Qt.UserRole))
             book = self.books[rnum]
             books.append(book)
         return books
@@ -914,15 +925,12 @@ class RejectListTableWidget(QTableWidget):
     def populate_table_row(self, row, rej):
 
         url_cell = ReadOnlyTableWidgetItem(rej.url)
-        url_cell.setData(Qt.UserRole, QVariant(rej.book_id))
+        url_cell.setData(Qt.UserRole, rej.book_id)
         self.setItem(row, 0, url_cell)
         self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
         self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
         
-        note_cell = EditWithComplete(self)
-        note_cell.lineEdit().mcompleter.model().set_items = \
-            partial(complete_model_set_items_kludge,
-                    note_cell.lineEdit().mcompleter.model())
+        note_cell = EditWithComplete(self,sort_func=lambda x:1)
         
         items = [rej.note]+self.rejectreasons
         note_cell.update_items_cache(items)
@@ -992,10 +1000,7 @@ class RejectListDialog(SizePersistedDialog):
         button_layout.addItem(spacerItem1)
 
         if show_all_reasons:
-            self.reason_edit = EditWithComplete(self)
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
             
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)
@@ -1037,7 +1042,7 @@ class RejectListDialog(SizePersistedDialog):
         rejectrows = []
         for row in range(self.rejects_table.rowCount()):
             url = unicode(self.rejects_table.item(row, 0).text()).strip()
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id =convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
             title = unicode(self.rejects_table.item(row, 1).text()).strip()
             auth = unicode(self.rejects_table.item(row, 2).text()).strip()
             note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
@@ -1047,7 +1052,7 @@ class RejectListDialog(SizePersistedDialog):
     def get_reject_list_ids(self):
         rejectrows = []
         for row in range(self.rejects_table.rowCount()):
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id = convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
             if book_id:
                 rejectrows.append(book_id)
         return rejectrows
@@ -1089,11 +1094,7 @@ class EditTextDialog(QDialog):
             self.textedit.setToolTip(tooltip)
 
         if rejectreasons or reasonslabel:
-            self.reason_edit = EditWithComplete(self)
-
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
             
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)

calibre-plugin/ffdl_plugin.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging
@@ -19,10 +19,12 @@ import urllib
 import email
 import traceback
 
-from PyQt4.Qt import (QApplication, QMenu, QToolButton, QTimer)
-
-from PyQt4.Qt import QPixmap, Qt
-from PyQt4.QtCore import QBuffer
+try:
+    from PyQt5.Qt import (QApplication, QMenu, QTimer)
+    from PyQt5.QtCore import QBuffer
+except ImportError as e:
+    from PyQt4.Qt import (QApplication, QMenu, QTimer)
+    from PyQt4.QtCore import QBuffer
 
 from calibre.constants import numeric_version as calibre_version
 
@@ -207,7 +209,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 #print("text/plain:%s"%event.mimeData().data(mimetype))
                 urllist.extend(get_urls_from_text(event.mimeData().data(mimetype)))
 
-        #print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
+        # print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
         if urllist or dropped_ids:
             QTimer.singleShot(1, partial(self.do_drop,
                                          dropped_ids=dropped_ids,
@@ -712,6 +714,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # No need to do anything with perfs here, but we could.
         prefs
 
+    def make_id_searchstr(self,url):
+        # older idents can be uri vs url and have | instead of : after
+        # http, plus many sites are now switching to https.
+        return 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url))
+
     def prep_downloads(self, options, books, merge=False, extrapayload=None):
         '''Fetch metadata for stories from servers, launch BG job when done.'''
 
@@ -764,7 +771,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if not merge: # skip reject list when merging.
             if rejecturllist.check(url):
                 rejnote = rejecturllist.get_full_note(url)
-                if question_dialog(self.gui, _('Reject URL?'),'''
+                if prefs['reject_always'] or question_dialog(self.gui, _('Reject URL?'),'''
                           <h3>%s</h3>
                           <p>%s</p>
                           <p>"<b>%s</b>"</p>
@@ -857,7 +864,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         series = story.getMetadata('series')
         if not merge and series and prefs['checkforseriesurlid']:
             # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
+            searchstr = self.make_id_searchstr(story.getMetadata('seriesUrl'))
             identicalbooks = db.search_getting_ids(searchstr, None)
             # print("searchstr:%s"%searchstr)
             # print("identicalbooks:%s"%identicalbooks)
@@ -889,7 +896,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         # all_metadata duplicates some data, but also includes extra_entries, etc.
         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-        
+
         book['title'] = story.getMetadata("title", removeallentities=True)
         book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
@@ -934,7 +941,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 logger.debug("from URL(%s)"%url)
     
                 # try to find by identifier url or uri first.
-                searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\:','https?(\:|\|)',url)
+                searchstr = self.make_id_searchstr(url)
                 identicalbooks = db.search_getting_ids(searchstr, None)
                 # print("searchstr:%s"%searchstr)
                 # print("identicalbooks:%s"%identicalbooks)
@@ -1617,7 +1624,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                             pass
 
                         if val:
-                            vallist = [val]
+                            vallist.append(val)
                             
                         db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)
 
@@ -1797,6 +1804,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['url'] = ''
         book['site'] = ''
         book['added'] = False
+        book['pubdate'] = None
         return book
     
     def convert_urls_to_books(self, urls):
@@ -1962,9 +1970,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         
             # fill from first of each if not already present:
             for k in ('pubdate', 'timestamp', 'updatedate'):
-                if k not in b: # not in this book?  Skip it.
+                if k not in b or not b[k]: # not in this book?  Skip it.
                     continue
-                if k not in book: # first is good enough for publisher.
+                if k not in book or not book[k]: # first is good enough for publisher.
                     book[k]=b[k]
 
                 # Do these even on first to get the all_metadata settings.
@@ -1985,11 +1993,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             for (k,v) in b['all_metadata'].iteritems():
                 #print("merge_meta_books v:%s k:%s"%(v,k))
                 if k in ('numChapters','numWords'):
-                    if k not in book['all_metadata']:
-                        book['all_metadata'][k] = b['all_metadata'][k]
-                    else:
-                        # lot of work for a simple add.
-                        book['all_metadata'][k] = unicode(int(book['all_metadata'][k].replace(',',''))+int(b['all_metadata'][k].replace(',','')))
+                    if k in b['all_metadata'] and b['all_metadata'][k]:
+                        if k not in book['all_metadata']:
+                            book['all_metadata'][k] = b['all_metadata'][k]
+                        else:
+                            # lot of work for a simple add.
+                            book['all_metadata'][k] = unicode(int(book['all_metadata'][k].replace(',',''))+int(b['all_metadata'][k].replace(',','')))
                 elif k in ('dateUpdated','datePublished','dateCreated',
                            'series','status','title'):
                     pass # handled above, below or skip these for now, not going to do anything with them.
@@ -2012,8 +2021,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             book['comments'] = existingbook['comments']
         else:
             book['title'] = deftitle = book_list[0]['title']
-            book['comments'] = _("Anthology containing:")+"\n" + \
-                "\n".join([ _("%s by %s")%(b['title'],', '.join(b['author'])) for b in book_list ])
+            if len(book['author']) > 1:
+                book['comments'] = _("Anthology containing:")+"\n" + \
+                    "\n".join([ _("%s by %s")%(b['title'],', '.join(b['author'])) for b in book_list ])
+            else:
+                book['comments'] = _("Anthology containing:")+"\n" + \
+                    "\n".join([ b['title'] for b in book_list ])
             # book['all_metadata']['description']
         
             # if all same series, use series for name.  But only if all and not previous named

calibre-plugin/jobs.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 

calibre-plugin/prefs.py

@@ -25,6 +25,7 @@ default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
 Dup from another site'''
+default_prefs['reject_always'] = False
 
 default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False

defaults.ini

@@ -626,6 +626,12 @@ extracategories:The Sentinel
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[bdsm-geschichten.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [bloodshedverse.com]
 ## website encoding(s) In theory, each website reports the character
 ## encoding they use for each page.  In practice, some sites report it
@@ -634,7 +640,7 @@ extracategories:The Sentinel
 ## explicitly set the encoding and order if you need to.  The special
 ## value 'auto' will call chardet and use the encoding it reports if
 ## it has +90% confidence.  'auto' is not reliable.
-website_encodings:ISO-8859-1,auto
+website_encodings:Windows-1252,ISO-8859-1,auto
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -790,6 +796,38 @@ extracategories:Harry Potter
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
+[fanfiction.csodaidok.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-2,auto
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,challenge
+reviews_label:Reviews
+challenge_label:Challenge
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+[fanfic.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfiction.mugglenet.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -848,6 +886,45 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[fictionmania.tv]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## items to include in the log page Empty metadata entries, or those
+## that haven't changed since the last update, will *not* appear, even
+## if in the list.  You can include extra text or HTML that will be
+## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
+## summary,<br />,...
+## Don't include numChapters since all stories are a single "chapter", there's
+## no way to reliably find the next chapter
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,series,title,author,description,category,genre,rating,warnings
+
+## items to include in the title page
+## Empty metadata entries will *not* appear, even if in the list.
+## You can include extra text or HTML that will be included as-is in
+## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
+## All current formats already include title and author.
+## Don't include numChapters since all stories are a single "chapter", there's
+## no way to reliably find the next chapter
+titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numWords,site,description
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:fileName,fileSize,oldName,newName,keyWords,mainCharactersAge,readings
+
+## Turns all space characters into "&nbsp" HTML entities to forcefully preserve
+## formatting with spaces. Enabling this will blow up the filesize quite a bit
+## and is probably not a good idea, unless you absolutely need the story
+## formatting.
+## Specific to fictionmania.tv
+non_breaking_spaces:false
+
 [fictionpad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -873,36 +950,6 @@ dislikes_label:Dislikes
 #username:YourName
 #password:yourpassword
 
-[storiesonline.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Clear FanFiction from defaults, site is original fiction.
-extratags:
-
-extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
-#extra_titlepage_entries:size,universeHTML,codes,notice
-
-size_label:Size
-universe_label:Universe
-universeUrl_label:Universe URL
-universeHTML_label:Universe
-codes_label:Codes
-notice_label:Notice
-
-## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
-## make entryHTML.
-make_linkhtml_entries:universe
-
-## storiesonline.net stories can be in a series or a universe, but not
-## both.  By default, universe will be populated in 'series' with
-## index=0
-universe_as_series: true
-
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -955,6 +1002,11 @@ extracategories:Star Trek
 extracharacters:Kirk,Spock
 extraships:Kirk/Spock
 
+[literotica.com]
+extra_valid_entries:eroticatags
+eroticatags_label:Erotica Tags
+#extra_titlepage_entries: eroticatags
+
 [lumos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1004,6 +1056,17 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[nocturnal-light.net]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:readings,reviews
+readings_label:Readings
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
 [occlumency.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1108,6 +1171,36 @@ extracategories:Buffy the Vampire Slayer
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
 
+[storiesonline.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Clear FanFiction from defaults, site is original fiction.
+extratags:
+
+extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
+#extra_titlepage_entries:size,universeHTML,codes,notice
+
+size_label:Size
+universe_label:Universe
+universeUrl_label:Universe URL
+universeHTML_label:Universe
+codes_label:Codes
+notice_label:Notice
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:universe
+
+## storiesonline.net stories can be in a series or a universe, but not
+## both.  By default, universe will be populated in 'series' with
+## index=0
+universe_as_series: true
+
 [svufiction.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1181,6 +1274,10 @@ extracategories:Harry Potter
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1
 
+[tolkienfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [trekiverse.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1296,8 +1393,11 @@ type_label:Type of Couple
 [www.fanfiction.net]
 user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
-## Change this to false to use them anyway.
-never_make_cover: true
+## Set this to true to never use them.
+#never_make_cover: false
+
+## fanfiction.net shows the user's 
+cover_exclusion_regexp:/imageu/
 
 ## fanfiction.net is blocking people more aggressively.  If you
 ## download fewer stories less often you can likely get by with
@@ -1721,6 +1821,9 @@ extracharacters:Wolverine,Rogue
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
 
+extra_valid_entries:reviews
+reviews_label:Reviews
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

downloader.py

@@ -227,9 +227,9 @@ def main(argv,
            except:
                options.update = False
                pass
-       
-       ## Check for include_images and absence of PIL, give warning.
-       if adapter.getConfig('include_images'):
+
+       ## Check for include_images without no_image_processing. In absence of PIL, give warning.
+       if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
            try:
                from calibre.utils.magick import Image
                logging.debug("Using calibre.utils.magick")

fanficdownloader/adapters/__init__.py

@@ -126,11 +126,17 @@ import adapter_literotica
 import adapter_voracity2eficcom
 import adapter_spikeluvercom
 import adapter_bloodshedversecom
+import adapter_nocturnallightnet
+import adapter_fanfichu
+import adapter_fanfictioncsodaidokhu
+import adapter_fictionmaniatv
+import adapter_bdsmgeschichten
+import adapter_tolkienfanfiction
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
 ## to pick out the adapter.
-    
+
 ## List of registered site adapters.
 __class_list = []
 __domain_map = {}
@@ -198,13 +204,15 @@ def getConfigSectionFor(url):
     (cls,fixedurl) = getClassFor(url)
     if cls:
         return cls.getConfigSection()
-    
+
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
 
 def getClassFor(url):
     ## fix up leading protocol.
     fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
+    if fixedurl.startswith("//"):
+        fixedurl = "http:%s"%url
     if not fixedurl.startswith("http"):
         fixedurl = "http://%s"%url
     ## remove any trailing '#' locations.
@@ -220,17 +228,17 @@ def getClassFor(url):
         domain = domain.replace("www.","")
         #logger.debug("trying site:without www: "+domain)
         cls = getClassFromList(domain)
-        fixedurl = fixedurl.replace("http://www.","http://")
+        fixedurl = re.sub(r"^http(s?)://www\.",r"http\1://",fixedurl)
     if not cls:
         #logger.debug("trying site:www."+domain)
         cls = getClassFromList("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
+        fixedurl = re.sub(r"^http(s?)://",r"http\1://www.",fixedurl)
 
     if cls:
         fixedurl = cls.stripURLParameters(fixedurl)
-    
+
     return (cls,fixedurl)
-    
+
 def getClassFromList(domain):
     try:
         return __domain_map[domain]

fanficdownloader/adapters/adapter_bdsmgeschichten.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+class BdsmGeschichtenAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+
+        self.story.setMetadata('siteabbrev','bdsmgesch')
+
+        self.firstPagUrl = re.sub("-\d+$", "-1", url)
+
+        # normalize to just the series name
+        storyid = urlparse.urlparse(self.firstPagUrl).path.split('/',)[0]
+        self.story.setMetadata('storyId', storyid)
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = '%d. %m %Y - %H:%M'
+
+    @staticmethod
+    def getSiteDomain():
+        return 'bdsm-geschichten.net'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.bdsm-geschichten.net']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://www.bdsm-geschichten.net/title-of-story-1"
+
+    def getSiteURLPattern(self):
+        return r"https?://www.bdsm-geschichten.net/([a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+
+        try:
+            data1 = self._fetchUrl(self.firstPagUrl)
+            soup = bs.BeautifulSoup(data1)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.firstPagUrl)
+            else:
+                raise e
+
+        #strip comments from soup
+        [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # Cache the soups so we won't have to redownload in getChapterText later
+        self.soupsCache = {}
+        self.soupsCache[self.firstPagUrl] = soup
+
+        # author
+        authorDiv = soup.find("div", "author-pane-line author-name")
+        authorId = authorDiv.string.strip()
+        self.story.setMetadata('authorId', authorId)
+        self.story.setMetadata('author', authorId)
+        # TODO not really true need to be loggedin for this to work or fetch userid
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+authorId)
+
+        # TODO better metadata
+        date = soup.find("div", {"class": "submitted"}).string.strip()
+        date = re.sub(" —.*", "", date)
+        fullmon = {"Januar":"01",
+                   "Februar":"02",
+                   u"März":"03",
+                   "April":"04",
+                   "Mai":"05",
+                   "Juni":"06",
+                   "Juli":"07",
+                   "August":"08",
+                   "September":"09",
+                   "Oktober":"10",
+                   "November":"11",
+                   "Dezember":"12"}
+        for (name,num) in fullmon.items():
+            if name in date:
+                date = date.replace(name,num)
+        self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+        title1 = soup.find("h1", {'class': 'title'}).string
+        storyTitle = re.sub(" Teil .*$", "", title1)
+        self.chapterUrls = [(title1, self.firstPagUrl)]
+        self.story.setMetadata('title', storyTitle)
+
+        for tagLink in soup.find("ul", "taxonomy").findAll("a"):
+            self.story.addToList('category', tagLink.string)
+
+        nextLinkDiv = soup.find("div", "field-field-naechster-teil")
+
+        while nextLinkDiv is not None:
+            nextLink = 'http://www.bdsm-geschichten.net' + nextLinkDiv.find("a")['href']
+            try:
+                logger.debug("Grabbing next chapter URL " + nextLink)
+                data2 = self._fetchUrl(nextLink)
+                soup2 = bs.BeautifulSoup(data2)
+                self.soupsCache[nextLink] = soup2
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+                nextLinkDiv = soup2.find("div", "field-field-naechster-teil")
+                title2 = soup2.find("h1", {'class': 'title'}).string
+                self.chapterUrls.append((title2, nextLink))
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(nextLink)
+                else:
+                    raise e
+
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+        logger.debug("Chapter URLS: " + repr(self.chapterUrls))
+
+        # normalize on first chapter URL.
+        self._setURL(self.chapterUrls[0][1])
+
+    def getChapterText(self, url):
+
+        if url in self.soupsCache:
+            logger.debug('Getting chapter <%s> from cache' % url)
+            soup = self.soupsCache[url]
+        else:
+            time.sleep(0.5)
+            logger.debug('Downloading chapter <%s>' % url)
+            data1 = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data1)
+            #strip comments from soup
+            [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        storyDiv1 = bs.Tag(soup, "div")
+        for para in soup.find("div", "full-node").find('div', 'content').findAll("p"):
+            storyDiv1.append(para)
+        storyDiv1.append('<br />')
+        storytext = self.utf8FromSoup(url,storyDiv1)
+
+        return storytext
+
+
+def getClass():
+    return BdsmGeschichtenAdapter

fanficdownloader/adapters/adapter_bloodshedversecom.py

@@ -4,6 +4,7 @@ import urllib2
 import urlparse
 
 from .. import BeautifulSoup
+from ..htmlcleanup import stripHTML
 
 from base_adapter import BaseSiteAdapter, makeDate
 from .. import exceptions
@@ -39,11 +40,16 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
         self._setURL(self.READ_URL_TEMPLATE % story_no)
         self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
 
-    def _customized_fetch_url(self, url, exception, parameters=None):
-        try:
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
             data = self._fetchUrl(url, parameters)
-        except urllib2.HTTPError:
-            raise exception(self.url)
 
         return BeautifulSoup.BeautifulSoup(data)
 
@@ -64,25 +70,23 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
         return url
 
     def extractChapterUrlsAndMetadata(self):
-        # Use FailedToDownload exception because that's the only error that can
-        # occur here, missing stories don't return a 404 header
-        soup = self._customized_fetch_url(self.url, exceptions.FailedToDownload)
+        soup = self._customized_fetch_url(self.url)
 
         # Since no 404 error code we have to raise the exception ourselves.
         # A title that is just 'by' indicates that there is no author name
         # and no story title available.
-        if soup.title.string.strip() == 'by':
+        if stripHTML(soup.title) == 'by':
             raise exceptions.StoryDoesNotExist(self.url)
 
         for option in soup.find('select', {'name': 'chapter'}):
-            title = option.string.strip()
+            title = stripHTML(option)
             url = self.READ_URL_TEMPLATE % option['value']
             self.chapterUrls.append((title, url))
 
         # Get the URL to the author's page and find the correct story entry to
         # scrape the metadata
         author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
-        soup = self._customized_fetch_url(author_url, exceptions.FailedToDownload)
+        soup = self._customized_fetch_url(author_url)
 
         story_no = self.story.getMetadata('storyId')
         # Ignore first list_box div, it only contains the author information
@@ -94,17 +98,19 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
             # parse the element
             if query_data['no'] == story_no:
                 break
+        else:
+            raise exceptions.FailedToDownload(self.url)
 
         title_anchor = list_box.find('a', {'class': 'fictitle'})
-        self.story.setMetadata('title', title_anchor.string.strip())
+        self.story.setMetadata('title', stripHTML(title_anchor))
 
         author_anchor = title_anchor.findNextSibling('a')
-        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('author', stripHTML(author_anchor))
         self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
         self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
 
         list_review = list_box.find('div', {'class': 'list_review'})
-        reviews = list_review.a.string.strip().split(' ', 1)[0]
+        reviews = stripHTML(list_review.a).split(' ', 1)[0]
         self.story.setMetadata('reviews', reviews)
 
         summary_div = list_box.find('div', {'class': 'list_summary'})
@@ -117,7 +123,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
 
         # I'm assuming this to be the category, not sure what else it could be
         first_listinfo = list_box.find('div', {'class': 'list_info'})
-        self.story.addToList('category', first_listinfo.a.string.strip())
+        self.story.addToList('category', stripHTML(first_listinfo.a))
 
         for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
             for b_tag in list_info('b'):
@@ -176,7 +182,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
             raise exceptions.AdultCheckRequired(self.url)
 
     def getChapterText(self, url):
-        soup = self._customized_fetch_url(url, exceptions.FailedToDownload)
+        soup = self._customized_fetch_url(url)
         storytext_div = soup.find('div', {'class': 'storytext'})
 
         if self.getConfig('strip_text_links'):

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -74,7 +74,8 @@ class EFPFanFicNet(BaseSiteAdapter):
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
-        if 'Fai il login e leggi la storia!' in data:
+        if( 'Fai il login e leggi la storia!' in data or
+            'Questa storia presenta contenuti non adatti ai minori' in data ):
             return True
         else:
             return False

fanficdownloader/adapters/adapter_fanfichu.py

@@ -0,0 +1,185 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanficHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class FanficHuAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'ffh'
+    SITE_DOMAIN = 'fanfic.hu'
+    SITE_LANGUAGE = 'Hungarian'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/merengo/'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%s'
+
+    DATE_FORMAT = '%m/%d/%Y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanficHuAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&i=1')
+
+        if soup.title.string.encode(_SOURCE_CODE_ENCODING).strip(' :') == 'írta':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        chapter_options = soup.find('form', action='viewstory.php').select('option')
+        # Remove redundant "Fejezetek" option
+        chapter_options.pop(0)
+
+        # If there is still more than one entry remove chapter overview entry
+        if len(chapter_options) > 1:
+            chapter_options.pop(0)
+
+        for option in chapter_options:
+            url = urlparse.urljoin(self.url, option['value'])
+            self.chapterUrls.append((option.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_id = self.story.getMetadata('storyId')
+        for table in soup('table', {'class': 'mainnav'}):
+            title_anchor = table.find('span', {'class': 'storytitle'}).a
+            href = title_anchor['href']
+            if href.startswith('javascript:'):
+                href = href.rsplit(' ', 1)[1].strip("'")
+            query_data = _get_query_data(href)
+
+            if query_data['sid'] == story_id:
+                break
+        else:
+            # This should never happen, the story must be found on the author's
+            # page.
+            raise exceptions.FailedToDownload(self.url)
+
+        self.story.setMetadata('title', title_anchor.string)
+
+        rows = table('tr')
+
+        anchors = rows[0].div('a')
+        author_anchor = anchors[1]
+        query_data = _get_query_data(author_anchor['href'])
+        self.story.setMetadata('author', author_anchor.string)
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.BASE_URL, author_anchor['href']))
+        self.story.setMetadata('reviews', anchors[3].string)
+
+        if self.getConfig('keep_summary_html'):
+            self.story.setMetadata('description', self.utf8FromSoup(author_url, rows[1].td))
+        else:
+            self.story.setMetadata('description', ''.join(rows[1].td(text=True)))
+
+        for row in rows[3:]:
+            index = 0
+            cells = row('td')
+
+            while index < len(cells):
+                cell = cells[index]
+                key = cell.b.string.encode(_SOURCE_CODE_ENCODING).strip(':')
+                try:
+                    value = cells[index+1].string.encode(_SOURCE_CODE_ENCODING)
+                except AttributeError:
+                    value = None
+
+                if key == 'Kategória':
+                    for anchor in cells[index+1]('a'):
+                        self.story.addToList('category', anchor.string)
+
+                elif key == 'Szereplõk':
+                    if cells[index+1].string:
+                        for name in cells[index+1].string.split(', '):
+                            self.story.addToList('character', name)
+
+                elif key == 'Korhatár':
+                    if value != 'nem korhatáros':
+                        self.story.setMetadata('rating', value)
+
+                elif key == 'Figyelmeztetések':
+                    for b_tag in cells[index+1]('b'):
+                        self.story.addToList('warnings', b_tag.string)
+
+                elif key == 'Jellemzõk':
+                    for genre in cells[index+1].string.split(', '):
+                        self.story.addToList('genre', genre)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Megjelenés':
+                    self.story.setMetadata('datePublished', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Frissítés':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Szavak':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                index += 2
+
+        if self.story.getMetadata('rating') == '18':
+            if not (self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        story_cell = soup.find('form', action='viewstory.php').parent.parent
+
+        for div in story_cell('div'):
+            div.extract()
+
+        return self.utf8FromSoup(url, story_cell)

fanficdownloader/adapters/adapter_fanfictioncsodaidokhu.py

@@ -0,0 +1,218 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanfictionCsodaidokHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FanfictionCsodaidokHuAdapter(BaseSiteAdapter):
+    _SITE_DOMAIN = 'fanfiction.csodaidok.hu'
+    _BASE_URL = 'http://' + _SITE_DOMAIN + '/'
+    _VIEW_STORY_URL_TEMPLATE = _BASE_URL + 'viewstory.php?sid=%s'
+    _VIEW_CHAPTER_URL_TEMPLATE = _VIEW_STORY_URL_TEMPLATE + '&chapter=%s'
+
+    _STORY_DOES_NOT_EXIST_PAGE_TITLE = 'Cím:  Szerző:'
+    _DATE_FORMAT = '%Y.%m.%d'
+    _SITE_LANGUAGE = 'Hungarian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self._VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self._SITE_DOMAIN)
+        self.story.setMetadata('language', self._SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanfictionCsodaidokHuAdapter._SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls._VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self._VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&chapter=1')
+
+        element = soup.find('div', id='pagetitle')
+        page_title = ''.join(element(text=True)).encode(_SOURCE_CODE_ENCODING)
+        if page_title == self._STORY_DOES_NOT_EXIST_PAGE_TITLE:
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        author_url = urlparse.urljoin(self.url, element.a['href'])
+
+        story_id = self.story.getMetadata('storyId')
+        element = soup.find('select', {'name': 'chapter'})
+        if element:
+            for option in element('option'):
+                title = option.string
+                url = self._VIEW_CHAPTER_URL_TEMPLATE % (story_id, option['value'])
+                self.chapterUrls.append((title, url))
+
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+
+        for listbox_div in soup('div', {'class': lambda klass: klass and 'listbox' in klass}):
+            a = listbox_div.div.a
+            if not a['href'].startswith('viewstory.php?sid='):
+                continue
+
+            query_data = _get_query_data(a['href'])
+            if query_data['sid'] == story_id:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = ''.join(a(text=True))
+        self.story.setMetadata('title', title)
+        if not self.chapterUrls:
+            self.chapterUrls.append((title, self.url))
+
+        element = a.findNextSibling('a')
+        self.story.setMetadata('author', element.string)
+        query_data = _get_query_data(element['href'])
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', author_url)
+
+        element = element.findNextSibling('span')
+        rating = element.nextSibling.strip(' [')
+
+        if rating.encode(_SOURCE_CODE_ENCODING) != 'Korhatár nélkül':
+            self.story.setMetadata('rating', rating)
+
+        if rating == '18':
+            raise exceptions.AdultCheckRequired(self.url)
+
+        element = element.findNextSiblings('a')[1]
+        self.story.setMetadata('reviews', element.string)
+
+        sections = listbox_div('div', {'class': lambda klass: klass and klass in ['content', 'tail']})
+        for section in sections:
+            for element in section('span', {'class': 'classification'}):
+                key = element.string.encode(_SOURCE_CODE_ENCODING).strip(' :')
+                try:
+                    value = element.nextSibling.string.encode(_SOURCE_CODE_ENCODING).strip()
+                except AttributeError:
+                    value = None
+
+                if key == 'Tartalom':
+                    contents = []
+                    keep_summary_html = self.getConfig('keep_summary_html')
+
+                    for sibling in _yield_next_siblings(element):
+                        if isinstance(sibling, BeautifulSoup.Tag):
+                            if sibling.name == 'span' and sibling.get('class', None) == 'classification':
+                                break
+
+                            if keep_summary_html:
+                                contents.append(self.utf8FromSoup(author_url, sibling))
+                            else:
+                                contents.append(''.join(sibling(text=True)))
+                        else:
+                            contents.append(sibling)
+                    self.story.setMetadata('description', ''.join(contents))
+
+                elif key == 'Kategória':
+                    for sibling in element.findNextSiblings(['a', 'span']):
+                        if sibling.name == 'span':
+                            break
+
+                        self.story.addToList('category', sibling.string)
+
+                elif key == 'Szereplők':
+                    for name in value.split(', '):
+                        self.story.addToList('characters', name)
+
+                elif key == 'Műfaj':
+                    if value != 'Nincs':
+                        self.story.setMetadata('genre', value)
+
+                elif key == 'Figyelmeztetés':
+                    if value != 'Nincs':
+                        for warning in value.split(', '):
+                            self.story.addToList('warnings', warning)
+
+                elif key == 'Kihívás':
+                    if value != 'Nincs':
+                        self.story.setMetadata('challenge', value)
+
+                elif key == 'Sorozat':
+                    if value != 'Nincs':
+                        self.story.setMetadata('series', value)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                elif key == 'Szavak száma':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Feltöltve':
+                    self.story.setMetadata('datePublished', makeDate(value, self._DATE_FORMAT))
+
+                elif key == 'Frissítve':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self._DATE_FORMAT))
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        contents = []
+
+        notes_div = soup.find('div', id='notes')
+        if notes_div:
+            contents.append(self.utf8FromSoup(url, notes_div))
+            story_div = notes_div.findNextSibling('div')
+        else:
+            element = soup.find('div', {'class': 'jumpmenu'})
+            story_div = element.findNextSibling('div')
+
+        contents.append(self.utf8FromSoup(url, story_div.span))
+        return ''.join(contents)

fanficdownloader/adapters/adapter_fictionmaniatv.py

@@ -0,0 +1,178 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+from ..BeautifulSoup import NavigableString
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return FictionManiaTVAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FictionManiaTVAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'fmt'
+    SITE_DOMAIN = 'fictionmania.tv'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/stories/'
+    READ_TEXT_STORY_URL_TEMPLATE = BASE_URL + 'readtextstory.html?storyID=%s'
+    DETAILS_URL_TEMPLATE = BASE_URL + 'details.html?storyID=%s'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+    ALTERNATIVE_DATETIME_FORMAT = '%m/%d/%y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['storyID'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.READ_TEXT_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+        # Always single chapters, probably should use the Anthology feature to
+        # merge chapters of a story
+        self.story.setMetadata('numChapters', 1)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FictionManiaTVAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_TEXT_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL) + '(readtextstory|details)\.html\?storyID=\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        url = self.DETAILS_URL_TEMPLATE % self.story.getMetadata('storyId')
+        soup = self._customized_fetch_url(url)
+
+        keep_summary_html = self.getConfig('keep_summary_html')
+        for row in soup.find('table')('tr'):
+            cells = row('td')
+            key = cells[0].b.string.strip(':')
+            try:
+                value = cells[1].string
+            except AttributeError:
+                value = None
+
+            if key == 'Story Name-Title':
+                self.story.setMetadata('title', value)
+                self.chapterUrls.append((value, self.url))
+
+            elif key == 'File Name':
+                self.story.setMetadata('fileName', value)
+
+            elif key == 'File Size':
+                self.story.setMetadata('fileSize', value)
+
+            elif key == 'Author':
+                element = cells[1].a
+                self.story.setMetadata('author', element.string)
+                query_data = _get_query_data(element['href'])
+                self.story.setMetadata('authorId', query_data['word'])
+                self.story.setMetadata('authorUrl', urlparse.urljoin(url, element['href']))
+
+            elif key == 'Date Added':
+                try:
+                    date = makeDate(value, self.DATETIME_FORMAT)
+                except ValueError:
+                    date = makeDate(value, self.ALTERNATIVE_DATETIME_FORMAT)
+                self.story.setMetadata('datePublished', date)
+
+            elif key == 'Old Name':
+                self.story.setMetadata('oldName', value)
+
+            elif key == 'New Name':
+                self.story.setMetadata('newName', value)
+
+            elif key == 'Other Key Names':
+                for name in value.split(', '):
+                    self.story.addToList('characters', name)
+
+            # I have no clue how the rating system works, if you are reading
+            # transgender fanfiction, you are probably an adult.
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Complete':
+                self.story.setMetadata('status', 'Complete' if value == 'Complete' else 'In-Progress')
+
+            elif key == 'Categories':
+                for element in cells[1]('a'):
+                    self.story.addToList('category', element.string)
+
+            elif key == 'Key Words':
+                for element in cells[1]('a'):
+                    self.story.addToList('keyWords', element.string)
+
+            elif key == 'Main Characters Age':
+                element = cells[1].a
+                self.story.setMetadata('mainCharactersAge', element.string)
+
+            elif key == 'Synopsis':
+                element = cells[1]
+
+                # Replace td with div to avoid possible strange formatting in
+                # the ebook later on
+                element.name = 'div'
+
+                if keep_summary_html:
+                    self.story.setMetadata('description', unicode(element))
+                else:
+                    self.story.setMetadata('description', ''.join(element(text=True)))
+
+            elif key == 'Reads':
+                self.story.setMetadata('readings', value)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        element = soup.find('pre')
+        element.name = 'div'
+
+        # The story's content is contained in a <pre> tag, probably taken 1:1
+        # from the source text file. A simple replacement of all newline
+        # characters with a break line tag should take care of formatting.
+
+        # While wrapping in paragraphs would be possible, it's too much work,
+        # I'd rather display the story 1:1 like it was found in the pre tag.
+        content = unicode(element)
+        content = content.replace('\n', '<br />')
+
+        if self.getConfig('non_breaking_spaces'):
+            content = content.replace(' ', '&nbsp;')
+        return content

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -186,9 +186,9 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             self.setCoverImage(self.url,coverurl)
 
         # fimf has started including extra stuff inside the description div.
-        descdivstr = "%s"%soup.find("div", {"class":"description"})
-        hrstr="<hr />"
-        descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
+        descdivstr = u"%s"%soup.find("div", {"class":"description"})
+        hrstr=u"<hr />"
+        descdivstr = u'<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
         self.setDescription(self.url,descdivstr)
 
         # Can't trust dates from API anymore I'm told.

fanficdownloader/adapters/adapter_literotica.py

@@ -39,19 +39,19 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        
+
         self.story.setMetadata('siteabbrev','litero')
 
         # normalize to first chapter.  Not sure if they ever have more than 2 digits.
         storyid = self.parsedUrl.path.split('/',)[2]
-        if re.match(r'-ch\d\d$',storyid):
+        if re.match(r'-(ch)?\d\d$',storyid):
             storyid = storyid[:-2]+'01'
         self.story.setMetadata('storyId',storyid)
-        
-        self.origurl = url
-        if "//www.i." in self.origurl:
-            ## accept m(mobile)url, but use www.
-            self.origurl = self.origurl.replace("//www.i.","//www.")
+
+        ## accept m(mobile)url, but use www.
+        self.origurl = re.sub("^(www|german|spanish|french|dutch|italian|romanian|portuguese|other)\.i",
+                              "\1",
+                              url)
 
         # normalized story URL.
         self._setURL(url[:url.index('//')+2]+self.getSiteDomain()\
@@ -60,28 +60,44 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = '%m/%d/%y'
-        
+
     @staticmethod
     def getSiteDomain():
-        return 'www.literotica.com'
+        return 'literotica.com'
 
     @classmethod
     def getAcceptDomains(cls):
-        return ['www.literotica.com', 'www.i.literotica.com']
+        return ['www.literotica.com',
+                'www.i.literotica.com',
+                'german.literotica.com',
+                'german.i.literotica.com',
+                'spanish.literotica.com',
+                'spanish.i.literotica.com',
+                'french.literotica.com',
+                'french.i.literotica.com',
+                'dutch.literotica.com',
+                'dutch.i.literotica.com',
+                'italian.literotica.com',
+                'italian.i.literotica.com',
+                'romanian.literotica.com',
+                'romanian.i.literotica.com',
+                'portuguese.literotica.com',
+                'portuguese.i.literotica.com',
+                'other.literotica.com',
+                'other.i.literotica.com']
 
     @classmethod
     def getSiteExampleURLs(self):
-        #return "http://www.literotica.com/s/story-title http://www.literotica.com/stories/showstory.php?id=1234 http://www.i.literotica.com/stories/showstory.php?id=1234"
-        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title"
+        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title http://portuguese.literotica.com/s/story-title http://german.literotica.com/s/story-title"
 
     def getSiteURLPattern(self):
-        return r"https?://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+        return r"https?://(www|german|spanish|french|dutch|italian|romanian|portuguese|other)(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
 
     def extractChapterUrlsAndMetadata(self):
 
         if not (self.is_adult or self.getConfig("is_adult")):
             raise exceptions.AdultCheckRequired(self.url)
-        
+
         url1 = self.origurl
         logger.debug("first page URL: "+url1)
 
@@ -99,7 +115,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
 
         # author
         a = soup1.find("span", "b-story-user-y")
-        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'])
+        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'][0])
         authorurl = a.a['href']
         if authorurl.startswith('//'):
             authorurl = self.parsedUrl.scheme+':'+authorurl
@@ -169,7 +185,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
             # now chapter list.  Assumed oldest to newest.
             self.chapterUrls = []
             row = row.nextSibling
-                
+
             self.story.setMetadata('datePublished',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
             while row['class'] == 'sl':
                 # pages include full URLs.
@@ -183,7 +199,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
 
             row = row.previousSibling
             self.story.setMetadata('dateUpdated',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
-                
+
         else:  # if one post only
             self.chapterUrls = [(soup1.h1.string, url1)]
             self.story.setMetadata('title', soup1.h1.string)
@@ -197,8 +213,15 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('numChapters', len(self.chapterUrls))
 
         self.story.setMetadata('category', soup1.find('div', 'b-breadcrumbs').findAll('a')[1].string)
+        # deliberately not self.setDescription() because it's never HTML.
         self.story.setMetadata('description', soup1.find('meta', {'name': 'description'})['content'])
 
+        # li tags inside div class b-s-story-tag-list
+        taglist = soup1.find('div', {'class':'b-s-story-tag-list'})
+        if taglist:
+            for li in taglist.findAll('a'):
+                self.story.addToList('eroticatags',stripHTML(li))
+
         return
 
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_nocturnallightnet.py

@@ -0,0 +1,177 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return NocturnalLightNetAdapter
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class NocturnalLightNetAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'nln'
+    SITE_DOMAIN = 'nocturnal-light.net'
+    BASE_URL = 'http://' + SITE_DOMAIN + '/fanfiction/'
+    STORY_URL_TEMPLATE = BASE_URL + 'story/%s'
+    AUTHORS_URL_TEMPLATE = BASE_URL + 'authors/%s'
+
+    DATETIME_FORMAT = '%m-%d-%y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        url_tokens = self.parsedUrl.path.split('/')
+        story_id = url_tokens[url_tokens.index('story') + 1]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return NocturnalLightNetAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.STORY_URL_TEMPLATE[:-2]) + r'\d+.*$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if soup.title.string.strip() == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        # "storycontent" is found in a single-chapter story
+        author_anchor = soup.find('div', id=lambda id: id in ('main', 'storycontent')).h1.a
+        self.story.setMetadata('author', author_anchor.string)
+
+        url_tokens = author_anchor['href'].split('/')
+        author_id = url_tokens[url_tokens.index('authors')+1]
+        self.story.setMetadata('authorId', author_id)
+        self.story.setMetadata('authorUrl', self.AUTHORS_URL_TEMPLATE % author_id)
+
+        chapter_anchors = soup('a', href=lambda href: href and href.startswith('/fanfiction/story/'))
+        for chapter_anchor in chapter_anchors:
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((chapter_anchor.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+        for listbox in soup('div', {'class': 'listbox'}):
+            url_tokens = listbox.a['href'].split('/')
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if story_id == url_tokens[url_tokens.index('story')+1]:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = listbox.a.string
+        self.story.setMetadata('title', title)
+
+        # No chapter anchors found in the original story URL, so the story has
+        # only a single chapter.
+        if not chapter_anchors:
+            self.chapterUrls.append((title, self.url))
+
+        for b_tag in listbox('b'):
+            key = b_tag.string.strip(':')
+            try:
+                value = b_tag.nextSibling.string.replace('•', '').strip(': ')
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                for sibling in _yield_next_siblings(b_tag):
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(author_url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+
+                # Pop last break line tag
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Category':
+                for sibling in b_tag.findNextSiblings(['a', 'b']):
+                    if sibling.name == 'b':
+                        break
+
+                    self.story.addToList('category', sibling.string)
+
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Chapters':
+                self.story.setMetadata('numChapters', int(value))
+
+                # Also parse reviews number which lies right after the chapters
+                # section
+                reviews_anchor = b_tag.findNextSibling('a')
+                reviews = reviews_anchor.string.split(' ')[1].strip('()')
+                self.story.setMetadata('reviews', reviews)
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Date Added':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Last Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Read':
+                self.story.setMetadata('readings', value.split()[0])
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        return self.utf8FromSoup(url, soup.find('div', id='storytext'))

fanficdownloader/adapters/adapter_spikeluvercom.py

@@ -3,6 +3,7 @@ import urllib2
 import urlparse
 
 from .. import BeautifulSoup
+from ..htmlcleanup import stripHTML
 
 from base_adapter import BaseSiteAdapter, makeDate
 from .. import exceptions
@@ -12,6 +13,16 @@ def getClass():
     return SpikeluverComAdapter
 
 
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
 class SpikeluverComAdapter(BaseSiteAdapter):
     SITE_ABBREVIATION = 'slc'
     SITE_DOMAIN = 'spikeluver.com'
@@ -23,6 +34,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
     AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=5'
 
     DATETIME_FORMAT = '%m/%d/%Y'
+    STORY_DOES_NOT_EXIST_ERROR_TEXT = 'That story does not exist on this archive.  You may search for it or return to the home page.'
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -34,11 +46,16 @@ class SpikeluverComAdapter(BaseSiteAdapter):
         self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id))
         self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
 
-    def _customized_fetch_url(self, url, exception, parameters=None):
-        try:
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
             data = self._fetchUrl(url, parameters)
-        except urllib2.HTTPError:
-            raise exception(self.url)
 
         return BeautifulSoup.BeautifulSoup(data)
 
@@ -54,7 +71,13 @@ class SpikeluverComAdapter(BaseSiteAdapter):
         return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
 
     def extractChapterUrlsAndMetadata(self):
-        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX, exceptions.StoryDoesNotExist)
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX)
+
+        errortext_div = soup.find('div', {'class': 'errortext'})
+        if errortext_div:
+            error_text = ''.join(errortext_div(text=True)).strip()
+            if error_text == self.STORY_DOES_NOT_EXIST_ERROR_TEXT:
+                raise exceptions.StoryDoesNotExist(self.url)
 
         # No additional login is required, just check for adult
         pagetitle_div = soup.find('div', id='pagetitle')
@@ -63,28 +86,28 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                 raise exceptions.AdultCheckRequired(self.url)
 
         url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
-        soup = self._customized_fetch_url(url, exceptions.StoryDoesNotExist)
+        soup = self._customized_fetch_url(url)
 
         pagetitle_div = soup.find('div', id='pagetitle')
-        self.story.setMetadata('title', pagetitle_div.a.string.strip())
+        self.story.setMetadata('title', stripHTML(pagetitle_div.a))
 
         author_anchor = pagetitle_div.a.findNextSibling('a')
         url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
         components = urlparse.urlparse(url)
         query_data = urlparse.parse_qs(components.query)
 
-        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('author', stripHTML(author_anchor))
         self.story.setMetadata('authorId', query_data['uid'])
         self.story.setMetadata('authorUrl', url)
 
         sort_div = soup.find('div', id='sort')
-        self.story.setMetadata('reviews', sort_div('a')[1].string.strip())
+        self.story.setMetadata('reviews', stripHTML(sort_div('a')[1]))
 
         listbox_tag = soup.find('div', {'class': 'listbox'})
         for span_tag in listbox_tag('span'):
             key = span_tag.string.strip(' :')
             try:
-                value = span_tag.nextSibling.string.strip()
+                value = stripHTML(span_tag.nextSibling)
             # This can happen with some fancy markup in the summary. Just
             # ignore this error and set value to None, the summary parsing
             # takes care of this
@@ -95,11 +118,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                 contents = []
                 keep_summary_html = self.getConfig('keep_summary_html')
 
-                # For some reason span_tag.findNextSiblings() only returns tags
-                # instead both NavigableString objects _and_ Tag objects as
-                # documented, so navigate manually
-                sibling = span_tag.nextSibling
-                while sibling:
+                for sibling in _yield_next_siblings(span_tag):
                     if isinstance(sibling, BeautifulSoup.Tag):
                         # Encountered next label, break. Not as bad as other
                         # e-fiction sites, let's hope this is enough for proper
@@ -113,7 +132,6 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                             contents.append(''.join(sibling(text=True)))
                     else:
                         contents.append(sibling)
-                    sibling = sibling.nextSibling
 
                 # Remove the preceding break line tag and other crud
                 contents.pop()
@@ -128,27 +146,27 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                     if sibling.name == 'br':
                         break
 
-                    self.story.addToList('category', sibling.string.strip())
+                    self.story.addToList('category', stripHTML(sibling))
 
             # Seems to be always "None" for some reason
             elif key == 'Characters':
                 for sibling in span_tag.findNextSiblings(['a', 'br']):
                     if sibling.name == 'br':
                         break
-                    self.story.addToList('characters', sibling.string.strip())
+                    self.story.addToList('characters', stripHTML(sibling))
 
             elif key == 'Genres':
                 for sibling in span_tag.findNextSiblings(['a', 'br']):
                     if sibling.name == 'br':
                         break
 
-                    self.story.addToList('genre', sibling.string.strip())
+                    self.story.addToList('genre', stripHTML(sibling))
 
             elif key == 'Warnings':
                 for sibling in span_tag.findNextSiblings(['a', 'br']):
                     if sibling.name == 'br':
                         break
-                    self.story.addToList('warnings', sibling.string.strip())
+                    self.story.addToList('warnings', stripHTML(sibling))
 
             # Challenges
 
@@ -156,7 +174,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                 a = span_tag.findNextSibling('a')
                 if not a:
                     continue
-                self.story.setMetadata('series', a.string.strip())
+                self.story.setMetadata('series', stripHTML(a))
                 self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href']))
 
             elif key == 'Chapters':
@@ -175,12 +193,15 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
 
         for p_tag in listbox_tag.findNextSiblings('p'):
-            chapter_anchor = p_tag.find('a')
-            title = chapter_anchor.string.strip()
+            chapter_anchor = p_tag.find('a', href=lambda href: href and href.startswith('viewstory.php?sid='))
+            if not chapter_anchor:
+                continue
+
+            title = stripHTML(chapter_anchor)
             url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
             self.chapterUrls.append((title, url))
 
     def getChapterText(self, url):
         url += self.AGE_CONSENT_URL_SUFFIX
-        soup = self._customized_fetch_url(url, exceptions.FailedToDownload)
+        soup = self._customized_fetch_url(url)
         return self.utf8FromSoup(url, soup.find('div', id='story'))

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -70,14 +70,14 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
-        if 'Free Registration' in data \
+        if self.needToLogin \
+                or 'Free Registration' in data \
                 or "Invalid Password!" in data \
                 or "Invalid User Name!" in data \
                 or "Log In" in data \
                 or "Access to unlinked chapters requires" in data:
-            return True
-        else:
-            return False
+            self.needToLogin = True
+        return self.needToLogin
         
     def performLogin(self, url):
         params = {}
@@ -114,11 +114,15 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         url = self.url
         logger.debug("URL: "+url)
 
+        self.needToLogin = False
         try:
             data = self._fetchUrl(url+":i")
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
+            elif e.code == 401:
+                self.needToLogin = True
+                data = ''
             else:
                 raise e
 

fanficdownloader/adapters/adapter_test1.py

@@ -219,15 +219,18 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         elif idstr == '81':
             self.story.addToList('category',u'Pitch Perfect')
             self.story.addToList('characters','Chloe B.')
+        elif idstr == '82':
+            self.story.addToList('characters','Henry (Once Upon a Time)')        
+            self.story.addToList('category',u'Once Upon a Time (TV)')
         elif idstr == '83':
             self.story.addToList('category',u'Rizzoli & Isles')
             self.story.addToList('characters','J. Rizzoli')
             self.story.addToList('category',u'Pitch Perfect')
             self.story.addToList('characters','Chloe B.')
             self.story.addToList('ships','Chloe B. & J. Rizzoli')
-        elif idstr == '82':
-            self.story.addToList('characters','Henry (Once Upon a Time)')        
-            self.story.addToList('category',u'Once Upon a Time (TV)')
+        elif idstr == '90':
+            self.story.setMetadata('characters','Henry (Once Upon a Time)')        
+            self.story.setMetadata('category',u'Once Upon a Time (TV)')
         else:
             self.story.addToList('category','Harry Potter')
             self.story.addToList('category','Furbie')

fanficdownloader/adapters/adapter_tolkienfanfiction.py

@@ -0,0 +1,255 @@
+# -*- coding: utf-8 -*-
+
+"""
+FFDL Adapter for TolkienFanFiction.com.
+
+Chapter URL: http://www.tolkienfanfiction.com/Story_Read_Chapter.php?CHid=1234
+    Metadata
+        Link to Story URL [Index]
+        chapterTitle
+        storyTitle
+Story URL: http://www.tolkienfanfiction.com/Story_Read_Head.php?STid=1034
+    Metadata
+        Links to Chapter URLs
+        storyTitle
+        chapterTitle[s]
+        author
+        authorId
+        authorUrl
+        numChapters
+        wordCount
+        description/summary
+        rating TODO
+        genre TODO
+        Characters
+        Ages (specific) TODO
+Search: http://www.tolkienfanfiction.com/Story_Chapter_Search.php?text=From+Wilderness+to+Cities+White&field=1&type=3&search=Search
+    Strategy
+        Search by exact phrase for styo
+    Metadata
+        dateUpdated
+    Parameters
+        field (field to search)
+            1: title
+            2: description
+            3: chapter text
+        type (any, all or exact phrase)
+            1: any
+            2: all
+            3: exact phrase
+
+"""
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib
+import urllib2
+import urlparse
+import string
+
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+def _is_chapter_url(url):
+    if "Story_Read_Chapter.php" in url:
+        return True
+    else:
+        return False
+
+def _latinize(text):
+    """
+    See http://stackoverflow.com/a/19114706/201318
+    """
+    src = u"áâäÉéêëíóôöúû"
+    tgt = u"aaaEeeeiooouu"
+    src_ord = [ord(char) for char in src]
+    translate_table = dict(zip(src_ord, tgt))
+    return text.translate(translate_table)
+
+def _fix_broken_markup(html):
+    """Replaces invalid comment tags"""
+    if html.startswith("<CENTER>"):
+        logger.error("TolkienFanFiction.com couldn't handle this request: '%s'" % html)
+    html = re.sub("<!-.+?->", "", html)
+    return html
+
+
+class TolkienFanfictionAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["ISO-8859-1",
+                       "Windows-1252"]  # 1252 is a superset of iso-8859-1.
+                                        # Most sites that claim to be
+                                        # iso-8859-1 (and some that claim to be
+                                        # utf8) are really windows-1252.
+
+        self.story.setMetadata('siteabbrev','tolkien')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = '%B %d, %Y'
+
+    @staticmethod
+    def getSiteDomain():
+        return 'tolkienfanfiction.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.tolkienfanfiction.com']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return 'http://www.tolkienfanfiction.com/Story_Read_Head.php?STid=1034 http://www.tolkienfanfiction.com/Story_Read_Chapter.php?CHid=4945'
+
+    def getSiteURLPattern(self):
+        return r"http://www.tolkienfanfiction.com/(Story_Read_Chapter.php\?CH|Story_Read_Head.php\?ST)id=([0-9]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        # if not (self.is_adult or self.getConfig("is_adult")):
+        #     raise exceptions.AdultCheckRequired(self.url)
+
+        if not _is_chapter_url(self.url):
+            self.indexUrl = self.url
+        else:
+            # Get the link to the index page
+            try:
+                chapterHtml = _fix_broken_markup(self._fetchUrl(self.url))
+                chapterSoup = bs.BeautifulSoup(chapterHtml)
+                indexLink = chapterSoup.find("a", text="[Index]").parent
+                self.indexUrl = 'http://' + self.host + '/' + indexLink.get('href')
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(self.url)
+                else:
+                    raise e
+        logger.debug("Determined index page: <%s>" % self.indexUrl)
+
+        storyId = self.indexUrl[self.indexUrl.index('=')+1:]
+        logger.debug("Story ID: %s" % storyId)
+        self.story.setMetadata('storyId', storyId)
+
+        try:
+            indexHtml = _fix_broken_markup(self._fetchUrl(self.indexUrl))
+            soup = bs.BeautifulSoup(indexHtml)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # chapterUrls
+        for pfLink in soup.findAll("a", text='[PF] '):
+            chapterLink = pfLink.parent.findNext("a")
+            chapterTitle = chapterLink.string
+            if self.getConfig('strip_chapter_numeral'):
+                chapterTitle = re.sub("^\d+:", "", chapterTitle)
+            chapterUrl = 'http://' + self.host + '/' + chapterLink['href']
+            self.chapterUrls.append((chapterTitle, chapterUrl))
+            numChapters = len(self.chapterUrls)
+            self.story.setMetadata('numChapters', numChapters)
+        logger.debug('Number of Chapters: %s' % numChapters)
+
+        # title
+        title = soup.find("table", "headertitle").find("tr").contents[1].string
+        logger.debug("Title: '%s'" % title)
+        self.story.setMetadata('title', title)
+
+        # author
+        authorLink = soup.find("a", {"href":lambda x: x.startswith("Author_Profile.php")})
+        authorName = authorLink.find("b").string
+        authorHref = authorLink['href']
+        authorUrl = 'http:' + self.host + '/' + authorHref
+        authorId = authorHref[authorHref.index('=')+1:]
+        self.story.setMetadata('author', authorName)
+        self.story.setMetadata('authorId', authorId)
+        self.story.setMetadata('authorUrl', authorUrl)
+        logger.debug("Author: %s [%s] @ <%s>" % (authorId, authorName, authorUrl))
+
+        # numWords
+        numWordsMatch = re.search("Word Count: (\d+)<BR>", indexHtml)
+        if numWordsMatch:
+            numWords = numWordsMatch.group(1)
+            logger.debug('Number of words: %s' % numWords)
+            self.story.setMetadata('numWords', numWords)
+
+        # description
+        description = soup.find("b", text="Description:").parent.nextSibling.nextSibling
+        self.story.setMetadata('description', description)
+        logger.debug("Summary: '%s'" % description)
+
+        # characters
+        characters = soup.find("b", text="Characters").parent.nextSibling.nextSibling.nextSibling
+        for character in characters.split(", "):
+            self.story.addToList('characters', character)
+        logger.debug("Characters: %s" % self.story.getMetadata('characters'))
+
+        logger.debug('Title as `str`: ' + str(title))
+        # For publication date we need to search
+        try:
+            queryString = urllib.urlencode((
+                ('type', 3),
+                ('field', 1),
+                # need translate here for the weird accented letters
+                ('text', _latinize(title)),
+                ('search', 'Search'),
+            ))
+            searchUrl = 'http://%s/Story_Chapter_Search.php?%s' % (self.host, queryString)
+            logger.debug("Search URL: <%s>" % searchUrl)
+            searchHtml = _fix_broken_markup(self._fetchUrl(searchUrl))
+            searchSoup = bs.BeautifulSoup(searchHtml)
+            date = searchSoup.find(text="Updated:").nextSibling.string
+            logger.debug("Last Updated: '%s'" % date)
+            self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # Set the URL to the Index URL
+        self._setURL(self.indexUrl)
+
+    def getChapterText(self, url):
+
+        logger.debug('Downloading chapter <%s>' % url)
+
+        time.sleep(0.5)
+        htmldata = _fix_broken_markup(self._fetchUrl(url))
+        soup = bs.BeautifulSoup(htmldata)
+
+        #strip comments from soup
+        [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # Strip redundant headings
+        [font.parent.extract() for font in soup.findAll("font", {"size": "4"})]
+
+        # get story text
+        textDiv = soup.find("div", "text")
+        storytext = self.utf8FromSoup(url, textDiv)
+
+        return storytext
+
+
+def getClass():
+    return TolkienFanfictionAdapter

fanficdownloader/adapters/adapter_voracity2eficcom.py

@@ -12,6 +12,16 @@ def getClass():
     return Voracity2EficComAdapter
 
 
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
 class Voracity2EficComAdapter(BaseSiteAdapter):
     SITE_ABBREVIATION = 'voe'
     SITE_DOMAIN = 'voracity2.e-fic.com'
@@ -65,11 +75,16 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
 
         self.is_logged_in = True
 
-    def _customized_fetch_url(self, url, exception, parameters=None):
-        try:
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
             data = self._fetchUrl(url, parameters)
-        except urllib2.HTTPError:
-            raise exception(self.url)
 
         return BeautifulSoup.BeautifulSoup(data)
 
@@ -85,7 +100,7 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
         return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
 
     def extractChapterUrlsAndMetadata(self):
-        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX, urllib2.URLError)
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX)
 
         # Check if the story is for "Registered Users Only", i.e. has adult
         # content. Based on the "is_adult" attributes either login or raise an
@@ -104,7 +119,7 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
                 raise exceptions.FailedToDownload(error_text)
 
         url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
-        soup = self._customized_fetch_url(url, urllib2.URLError)
+        soup = self._customized_fetch_url(url)
 
         # If logged in and the skin doesn't match the required skin throw an
         # error
@@ -142,11 +157,7 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
                 contents = []
                 keep_summary_html = self.getConfig('keep_summary_html')
 
-                # For some reason span_tag.findNextSiblings() only returns tags
-                # instead both NavigableString objects _and_ Tag objects as
-                # documented, so navigate manually
-                sibling = b_tag.nextSibling
-                while sibling:
+                for sibling in _yield_next_siblings(b_tag):
                     if isinstance(sibling, BeautifulSoup.Tag):
                         # Encountered next label, break. This method is the
                         # safest and most reliable I could think of. Blame
@@ -162,7 +173,6 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
                             contents.append(''.join(sibling(text=True)))
                     else:
                         contents.append(sibling)
-                    sibling = sibling.nextSibling
 
                 # Remove the preceding break line tag and other crud
                 contents.pop()
@@ -218,5 +228,5 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
         url += self.AGE_CONSENT_URL_SUFFIX
-        soup = self._customized_fetch_url(url, urllib2.URLError)
+        soup = self._customized_fetch_url(url)
         return self.utf8FromSoup(url, soup.find('div', id='story'))

fanficdownloader/adapters/adapter_wraithbaitcom.py

@@ -125,6 +125,10 @@ class WraithBaitComAdapter(BaseSiteAdapter):
         rating=pt.text.split('[')[1].split(']')[0]
         self.story.setMetadata('rating', rating)
 
+        st = soup.find('div', {'class' : 'storytitle'})
+        a = st.findAll('a', href=re.compile(r'reviews.php\?type=ST&item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
+        self.story.setMetadata('reviews',stripHTML(a))
+        
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # include author on chapters if multiple authors.

fanficdownloader/adapters/base_adapter.py

@@ -328,7 +328,8 @@ class BaseSiteAdapter(Configurable):
 
     def setCoverImage(self,storyurl,imgurl):
         if self.getConfig('include_images'):
-            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True)
+            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True,
+                                 coverexclusion=self.getConfig('cover_exclusion_regexp'))
 
     # This gives us a unicode object, not just a string containing bytes.
     # (I gave soup a unicode string, you'd think it could give it back...)

fanficdownloader/configurable.py

@@ -53,46 +53,52 @@ class Configuration(ConfigParser.SafeConfigParser):
             self.addConfigSection(sitewithout+":"+fileform)
         self.addConfigSection("overrides")
         
-        self.validEntries = [
+        self.listTypeEntries = [
             'category',
             'genre',
-            'language',
             'characters',
             'ships',
+            'warnings',
+            'extratags',
+            'author',
+            'authorId',
+            'authorUrl',
+            'lastupdate',
+            ]
+        
+        self.validEntries = self.listTypeEntries + [
             'series',
             'seriesUrl',
+            'language',
             'status',
             'datePublished',
             'dateUpdated',
             'dateCreated',
             'rating',
-            'warnings',
             'numChapters',
             'numWords',
             'site',
             'storyId',
-            'authorId',
-            'extratags',
             'title',
             'storyUrl',
             'description',
-            'author',
-            'authorUrl',
             'formatname',
             'formatext',
             'siteabbrev',
             'version',
             # internal stuff.
-            'langcode',
-            'output_css',
             'authorHTML',
             'seriesHTML',
-            'lastupdate'
+            'langcode',
+            'output_css',
             ]
 
     def addConfigSection(self,section):
         self.sectionslist.insert(0,section)
 
+    def isListType(self,key):
+        return key in self.listTypeEntries or self.hasConfig("include_in_"+key)
+        
     def isValidMetaEntry(self, key):
         return key in self.getValidMetaList()
 
@@ -162,6 +168,9 @@ class Configurable(object):
     def __init__(self, configuration):
         self.configuration = configuration
 
+    def isListType(self,key):
+        return self.configuration.isListType(key)
+
     def isValidMetaEntry(self, key):
         return self.configuration.isValidMetaEntry(key)
 

fanficdownloader/epubutils.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging

fanficdownloader/story.py

@@ -28,6 +28,9 @@ import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 from configurable import Configurable
 
+SPACE_REPLACE=u'\s'
+SPLIT_META=u'\,'
+
 # Create convert_image method depending on which graphics lib we can
 # load.  Preferred: calibre, PIL, none
 
@@ -230,19 +233,19 @@ class InExMatch:
     def  __init__(self,line):
         if "=~" in line:
             (self.keys,self.match) = line.split("=~")
-            self.match = self.match.replace('\s',' ')
+            self.match = self.match.replace(SPACE_REPLACE,' ')
             self.regex = re.compile(self.match)
         elif "!~" in line:
             (self.keys,self.match) = line.split("!~")
-            self.match = self.match.replace('\s',' ')
+            self.match = self.match.replace(SPACE_REPLACE,' ')
             self.regex = re.compile(self.match)
             self.negate = True
         elif "==" in line:
             (self.keys,self.match) = line.split("==")
-            self.match = self.match.replace('\s',' ')
+            self.match = self.match.replace(SPACE_REPLACE,' ')
         elif "!=" in line:
             (self.keys,self.match) = line.split("!=")
-            self.match = self.match.replace('\s',' ')
+            self.match = self.match.replace(SPACE_REPLACE,' ')
             self.negate = True
         self.keys = map( lambda x: x.strip(), self.keys.split(",") )
 
@@ -301,8 +304,7 @@ class Story(Configurable):
         self.logfile=None # cheesy way to carry log file forward across update.
 
         ## Look for config parameter, split and add each to metadata field.
-        for (config,metadata) in [("extratags","extratags"),
-                                  ("extracategories","category"),
+        for (config,metadata) in [("extracategories","category"),
                                   ("extragenres","genre"),
                                   ("extracharacters","characters"),
                                   ("extraships","ships"),
@@ -321,19 +323,29 @@ class Story(Configurable):
                 iel = []
                 self.in_ex_cludes[ie] = self.set_in_ex_clude(ies)
 
+    def join_list(self, key, vallist):
+        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist))
+                
     def setMetadata(self, key, value, condremoveentities=True):
-        ## still keeps < < and &
-        if condremoveentities:
-            self.metadata[key]=conditionalRemoveEntities(value)
+
+        # keep as list type, but set as only value.
+        if self.isList(key):
+            self.addToList(key,value,condremoveentities=condremoveentities,clear=True)
         else:
-            self.metadata[key]=value
+            ## still keeps < < and &
+            if condremoveentities:
+                self.metadata[key]=conditionalRemoveEntities(value)
+            else:
+                self.metadata[key]=value
+                
         if key == "language":
             try:
                 # getMetadata not just self.metadata[] to do replace_metadata.
-                self.metadata['langcode'] = langs[self.getMetadata(key)]
+                self.setMetadata('langcode',langs[self.getMetadata(key)])
             except:
-                self.metadata['langcode'] = 'en'
-        if key == 'dateUpdated':
+                self.setMetadata('langcode','en')
+                
+        if key == 'dateUpdated' and value:
             # Last Update tags for Bill.
             self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
@@ -397,6 +409,7 @@ class Story(Configurable):
     ## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp]
     def setReplace(self,replace):
         for line in replace.splitlines():
+            # print("replacement line:%s"%line)
             (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None)
             if "&&" in line:
                 (line,conditional) = line.split("&&")
@@ -416,15 +429,20 @@ class Story(Configurable):
                 # A way to explicitly include spaces in the
                 # replacement string.  The .ini parser eats any
                 # trailing spaces.
-                replacement=replacement.replace('\s',' ')
+                replacement=replacement.replace(SPACE_REPLACE,' ')
                 self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
 
-    def doReplacements(self,value,key):
+    def doReplacements(self,value,key,return_list=False,seen_list=[]):
         value = self.do_in_ex_clude('include_metadata_pre',value,key)
         value = self.do_in_ex_clude('exclude_metadata_pre',value,key)
-        # if key=='characters' and value=='Bilbo':
-        #     value = ''
-        for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
+
+        retlist = [value]
+        for replaceline in self.replacements:
+            if replaceline in seen_list: # recursion on pattern, bail
+                # print("bailing on %s"%replaceline)
+                continue
+            #print("replacement tuple:%s"%replaceline)
+            (metakeys,regexp,replacement,condkey,condregexp) = replaceline
             if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
                     and regexp.search(value):
@@ -434,10 +452,34 @@ class Story(Configurable):
                     doreplace = condval != None and condregexp.search(condval)
 
                 if doreplace:
-                    value = regexp.sub(replacement,value)
-        value = self.do_in_ex_clude('include_metadata_post',value,key)
-        value = self.do_in_ex_clude('exclude_metadata_post',value,key)
-        return value
+                    # split into more than one list entry if
+                    # SPLIT_META present in replacement string.  Split
+                    # first, then regex sub, then recurse call replace
+                    # on each.  Break out of loop, each split element
+                    # handled individually by recursion call.
+                    if SPLIT_META in replacement:
+                        retlist = []
+                        for splitrepl in replacement.split(SPLIT_META):
+                            retlist.extend(self.doReplacements(regexp.sub(splitrepl,value),
+                                                               key,
+                                                               return_list=True,
+                                                               seen_list=seen_list+[replaceline]))
+                        break
+                    else:
+                        # print("replacement,value:%s,%s->%s"%(replacement,value,regexp.sub(replacement,value)))
+                        value = regexp.sub(replacement,value)
+                        retlist = [value]
+                    
+        for val in retlist:
+            retlist = map(partial(self.do_in_ex_clude,'include_metadata_post',key=key),retlist)
+            retlist = map(partial(self.do_in_ex_clude,'exclude_metadata_post',key=key),retlist)
+        # value = self.do_in_ex_clude('include_metadata_post',value,key)
+        # value = self.do_in_ex_clude('exclude_metadata_post',value,key)
+
+        if return_list:
+            return retlist
+        else:
+            return self.join_list(key,retlist)
 
     def getMetadataRaw(self,key):
         if self.isValidMetaEntry(key) and self.metadata.has_key(key):
@@ -451,8 +493,9 @@ class Story(Configurable):
             return value
 
         if self.isList(key):
-            join_string = self.getConfig("join_string_"+key,u", ").replace('\s',' ')
-            value = join_string.join(self.getList(key, removeallentities, doreplacements=True))
+            # join_string = self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ')
+            # value = join_string.join(self.getList(key, removeallentities, doreplacements=True))
+            value = self.join_list(key,self.getList(key, removeallentities, doreplacements=True))
             if doreplacements:
                 value = self.doReplacements(value,key+"_LIST")
             return value
@@ -482,7 +525,8 @@ class Story(Configurable):
                        doreplacements=True,
                        keeplists=False):
         '''
-        All single value *and* list value metadata as strings (unless keeplists=True, then keep lists).
+        All single value *and* list value metadata as strings (unless
+        keeplists=True, then keep lists).
         '''
         allmetadata = {}
 
@@ -502,16 +546,16 @@ class Story(Configurable):
                     auth=removeAllEntities(auth)
 
                 htmllist.append(linkhtml%('author',aurl,auth))
-            join_string = self.getConfig("join_string_authorHTML",u", ").replace('\s',' ')
-            self.setMetadata('authorHTML',join_string.join(htmllist))
+            # join_string = self.getConfig("join_string_authorHTML",u", ").replace(SPACE_REPLACE,' ')
+            self.setMetadata('authorHTML',self.join_list("join_string_authorHTML",htmllist))
         else:
             self.setMetadata('authorHTML',linkhtml%('author',self.getMetadata('authorUrl', removeallentities, doreplacements),
                                                     self.getMetadata('author', removeallentities, doreplacements)))
 
-        if self.getMetadataRaw('seriesUrl') != None:
+        if self.getMetadataRaw('seriesUrl'):
             self.setMetadata('seriesHTML',linkhtml%('series',self.getMetadata('seriesUrl', removeallentities, doreplacements),
                                                     self.getMetadata('series', removeallentities, doreplacements)))
-        elif self.getMetadataRaw('series') != None:
+        elif self.getMetadataRaw('series'):
             self.setMetadata('seriesHTML',self.getMetadataRaw('series'))
 
         # logger.debug("make_linkhtml_entries:%s"%self.getConfig('make_linkhtml_entries'))
@@ -534,8 +578,8 @@ class Story(Configurable):
                     v=removeAllEntities(v)
 
                 htmllist.append(linkhtml%(k,url,v))
-            join_string = self.getConfig("join_string_"+k+"HTML",u", ").replace('\s',' ')
-            self.setMetadata(k+'HTML',join_string.join(htmllist))
+            # join_string = self.getConfig("join_string_"+k+"HTML",u", ").replace(SPACE_REPLACE,' ')
+            self.setMetadata(k+'HTML',self.join_list("join_string_"+k+"HTML",htmllist))
 
         for k in self.getValidMetaList():
             if self.isList(k) and keeplists:
@@ -550,11 +594,12 @@ class Story(Configurable):
         for v in l:
             self.addToList(listname,v.strip())
 
-    def addToList(self,listname,value):
+    def addToList(self,listname,value,condremoveentities=True,clear=False):
         if value==None:
             return
-        value = conditionalRemoveEntities(value)
-        if not self.isList(listname) or not listname in self.metadata:
+        if condremoveentities:
+            value = conditionalRemoveEntities(value)
+        if clear or not self.isList(listname) or not listname in self.metadata:
             # Calling addToList to a non-list meta will overwrite it.
             self.metadata[listname]=[]
         # prevent duplicates.
@@ -566,7 +611,7 @@ class Story(Configurable):
 
     def isList(self,listname):
         'Everything set with an include_in_* is considered a list.'
-        return self.hasConfig("include_in_"+listname) or \
+        return self.isListType(listname) or \
             ( self.isValidMetaEntry(listname) and self.metadata.has_key(listname) \
                   and isinstance(self.metadata[listname],list) )
 
@@ -595,16 +640,20 @@ class Story(Configurable):
 
         if retlist:
             if doreplacements:
-                retlist = filter( lambda x : x!=None and x!='' ,
-                                  map(partial(self.doReplacements,key=listname),retlist) )
+                newretlist = []
+                for val in retlist:
+                    newretlist.extend(self.doReplacements(val,listname,return_list=True))
+                retlist = newretlist
+                
             if removeallentities:
-                retlist = filter( lambda x : x!=None and x!='' ,
-                                  map(removeAllEntities,retlist) )
+                retlist = map(removeAllEntities,retlist)
+                
+            retlist = filter( lambda x : x!=None and x!='' ,retlist)
 
         # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
         # use replace_metadata to change separator first if needed.
         # ships=>[ ]*(/|&|&)[ ]*=>/
-        if listname == 'ships' and self.getConfig('sort_ships'):
+        if listname == 'ships' and self.getConfig('sort_ships') and retlist:
             retlist = [ '/'.join(sorted(x.split('/'))) for x in retlist ]
 
         if retlist:
@@ -723,6 +772,10 @@ class Story(Configurable):
                      '','',''))
                 #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
 
+        # apply coverexclusion to explicit covers, too.  Primarily for ffnet imageu.
+        if cover and coverexclusion and re.search(coverexclusion,imgurl):
+            return
+
         prefix='ffdl'
         if imgurl not in self.imgurls:
             parsedUrl = urlparse.urlparse(imgurl)
@@ -754,7 +807,7 @@ class Story(Configurable):
                 return "failedtoload"
 
             # explicit cover, make the first image.
-            if cover and not self.getConfig('never_make_cover'):
+            if cover:
                 if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']:
                     # remove existing cover, if there is one.
                     del self.imgurls[0]
@@ -773,7 +826,7 @@ class Story(Configurable):
                 if self.cover == None and \
                         self.getConfig('make_firstimage_cover') and \
                         not self.getConfig('never_make_cover') and \
-                        (not coverexclusion or not re.search(coverexclusion,imgurl)):
+                        not (coverexclusion and re.search(coverexclusion,imgurl)):
                     newsrc = "images/cover.%s"%ext
                     self.cover=newsrc
                     self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})

index.html

@@ -55,11 +55,17 @@
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
 	  <h3>Changes:</h3>
+	  <p>
+            Now supporting over 100 different sites!  Thanks, cryzed, for pushing us over the top.
+	  </p>
 	  <p>
 	    <ul>
-              <li>New site: spikeluver.com -- Thanks, cryzed!</li>
-              <li>New site: bloodshedverse.com -- Thanks, cryzed!</li>
-              <li>Updates for site: voracity2.e-fic.com -- Thanks, cryzed!</li>
+              <li>New German language site bdsm-geschichten.net, thanks to doe5716.</li>
+              <li>New site tolkienfanfiction.com, thanks to doe5716.</li>
+              <li>Add other languages for literotica.com, thanks to doe5716.</li>
+              <li>Fixes for bloodshedverse & spikelover having html tags in title, etc.</li>
+              <li>Known issue: Password protected FimFiction.net stories aren't working.  FimF changed API access.</li>
+              <li>Known issue: Specific metadata 'eroticatags' for literotica.com doesn't work on all stories.</li>
 	    </ul>
 	  </p>
 	  <p>
@@ -70,7 +76,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-5-02.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://2-0-00.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

makezip.py

@@ -1,9 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-# epubmerge.py 1.0
-
-# Copyright 2011, Jim Miller
+# Copyright 2014, Jim Miller
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

plugin-defaults.ini

@@ -232,6 +232,9 @@ connect_timeout:60.0
 ## Make sure to keep at least one space at the start of each line and
 ## to escape % to %%, if used.
 ## template => regexp to match => GC Setting to use.
+## To use this, make sure you go to the Generate Cover tab in FFDL
+## config and check 'Allow generate_cover_settings from personal.ini
+## to override'
 #generate_cover_settings:
 # ${category} => Buffy:? [tT]he Vampire Slayer => BuffyCover
 # ${category} => Star Trek => StarTrekCover
@@ -599,6 +602,12 @@ extracategories:The Sentinel
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[bdsm-geschichten.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [bloodshedverse.com]
 ## website encoding(s) In theory, each website reports the character
 ## encoding they use for each page.  In practice, some sites report it
@@ -607,7 +616,7 @@ extracategories:The Sentinel
 ## explicitly set the encoding and order if you need to.  The special
 ## value 'auto' will call chardet and use the encoding it reports if
 ## it has +90% confidence.  'auto' is not reliable.
-website_encodings:ISO-8859-1,auto
+website_encodings:Windows-1252,ISO-8859-1,auto
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -781,6 +790,38 @@ extracategories:Harry Potter
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
+[fanfiction.csodaidok.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-2,auto
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,challenge
+reviews_label:Reviews
+challenge_label:Challenge
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+[fanfic.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfiction.mugglenet.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -839,6 +880,45 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[fictionmania.tv]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## items to include in the log page Empty metadata entries, or those
+## that haven't changed since the last update, will *not* appear, even
+## if in the list.  You can include extra text or HTML that will be
+## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
+## summary,<br />,...
+## Don't include numChapters since all stories are a single "chapter", there's
+## no way to reliably find the next chapter
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,series,title,author,description,category,genre,rating,warnings
+
+## items to include in the title page
+## Empty metadata entries will *not* appear, even if in the list.
+## You can include extra text or HTML that will be included as-is in
+## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
+## All current formats already include title and author.
+## Don't include numChapters since all stories are a single "chapter", there's
+## no way to reliably find the next chapter
+titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numWords,site,description
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:fileName,fileSize,oldName,newName,keyWords,mainCharactersAge,readings
+
+## Turns all space characters into "&nbsp" HTML entities to forcefully preserve
+## formatting with spaces. Enabling this will blow up the filesize quite a bit
+## and is probably not a good idea, unless you absolutely need the story
+## formatting.
+## Specific to fictionmania.tv
+non_breaking_spaces:false
+
 [fictionpad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -864,36 +944,6 @@ dislikes_label:Dislikes
 #username:YourName
 #password:yourpassword
 
-[storiesonline.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Clear FanFiction from defaults, site is original fiction.
-extratags:
-
-extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
-#extra_titlepage_entries:size,universeHTML,codes,notice
-
-size_label:Size
-universe_label:Universe
-universeUrl_label:Universe URL
-universeHTML_label:Universe
-codes_label:Codes
-notice_label:Notice
-
-## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
-## make entryHTML.
-make_linkhtml_entries:universe
-
-## storiesonline.net stories can be in a series or a universe, but not
-## both.  By default, universe will be populated in 'series' with
-## index=0
-universe_as_series: true
-
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -946,6 +996,11 @@ extracategories:Star Trek
 extracharacters:Kirk,Spock
 extraships:Kirk/Spock
 
+[literotica.com]
+extra_valid_entries:eroticatags
+eroticatags_label:Erotica Tags
+#extra_titlepage_entries: eroticatags
+
 [lumos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -995,6 +1050,17 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[nocturnal-light.net]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:readings,reviews
+readings_label:Readings
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
 [occlumency.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1099,6 +1165,36 @@ extracategories:Buffy the Vampire Slayer
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
 
+[storiesonline.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Clear FanFiction from defaults, site is original fiction.
+extratags:
+
+extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
+#extra_titlepage_entries:size,universeHTML,codes,notice
+
+size_label:Size
+universe_label:Universe
+universeUrl_label:Universe URL
+universeHTML_label:Universe
+codes_label:Codes
+notice_label:Notice
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:universe
+
+## storiesonline.net stories can be in a series or a universe, but not
+## both.  By default, universe will be populated in 'series' with
+## index=0
+universe_as_series: true
+
 [svufiction.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1172,6 +1268,10 @@ extracategories:Harry Potter
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1
 
+[tolkienfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [trekiverse.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1287,8 +1387,11 @@ type_label:Type of Couple
 [www.fanfiction.net]
 user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
-## Change this to false to use them anyway.
-never_make_cover: true
+## Set this to true to never use them.
+#never_make_cover: false
+
+## fanfiction.net shows the user's 
+cover_exclusion_regexp:/imageu/
 
 ## fanfiction.net is blocking people more aggressively.  If you
 ## download fewer stories less often you can likely get by with
@@ -1718,6 +1821,9 @@ extracharacters:Wolverine,Rogue
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
 
+extra_valid_entries:reviews
+reviews_label:Reviews
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For