Update CLI download zip.

doesn't support HTTPS fixed
www. fixed
timer removed

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-5-06
+version: 2-0-02
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 8, 25)
+    version             = (2, 0, 2)
     minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/common_utils.py

@@ -8,12 +8,19 @@ __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import os
-from PyQt4 import QtGui
-from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
-                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
-                      QTextEdit,
-                      QListWidget, QAbstractItemView)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+    
 from calibre.constants import iswindows
 from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
 from calibre.gui2.actions import menu_action_unique_name

calibre-plugin/config.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging
@@ -13,10 +13,31 @@ logger = logging.getLogger(__name__)
 import traceback, copy, threading
 from collections import OrderedDict
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
-                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
-                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
-                      QDialogButtonBox, QGroupBox )
+try:
+    from PyQt5.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                          QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
+                          QDialogButtonBox, QGroupBox )
+except ImportError as e:
+    from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                          QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
+                          QDialogButtonBox, QGroupBox )
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()
 
 from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog
@@ -60,7 +81,7 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
 
-from calibre.gui2.complete import MultiCompleteLineEdit
+from calibre.gui2.complete2 import EditWithComplete  #MultiCompleteLineEdit
 
 class RejectURLList:
     def __init__(self,prefs):
@@ -220,6 +241,7 @@ class ConfigWidget(QWidget):
         prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
         prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
+        prefs['reject_always'] = self.basic_tab.reject_always.isChecked()
 
         if self.readinglist_tab:
             # lists
@@ -243,7 +265,7 @@ class ConfigWidget(QWidget):
         prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
         gc_site_settings = {}
         for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
             if val != 'none':
                 gc_site_settings[site] = val
                 #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
@@ -275,12 +297,12 @@ class ConfigWidget(QWidget):
 
         # Custom Columns tab
         # error column
-        prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())
+        prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
 
         # cust cols tab
         colsmap = {}
         for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
             if val != 'none':
                 colsmap[col] = val
                 #print("colsmap[%s]:%s"%(col,colsmap[col]))
@@ -482,6 +504,11 @@ class BasicTab(QWidget):
         self.reject_reasons.clicked.connect(self.show_reject_reasons)
         self.l.addWidget(self.reject_reasons)
 
+        self.reject_always = QCheckBox(_('Reject Without Confirmation?'),self)
+        self.reject_always.setToolTip(_("Always reject URLs on the Reject List without stopping and asking."))
+        self.reject_always.setChecked(prefs['reject_always'])
+        self.l.addWidget(self.reject_always)
+
         topl.addWidget(defs_gb)
 
         horz = QHBoxLayout()
@@ -649,7 +676,7 @@ class ReadingListTab(QWidget):
         label = QLabel(_('"Send to Device" Reading Lists'))
         label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
-        self.send_lists_box = MultiCompleteLineEdit(self)
+        self.send_lists_box = EditWithComplete(self)
         self.send_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.send_lists_box.update_items_cache(reading_lists)
         self.send_lists_box.setText(prefs['send_lists'])
@@ -665,7 +692,7 @@ class ReadingListTab(QWidget):
         label = QLabel(_('"To Read" Reading Lists'))
         label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
-        self.read_lists_box = MultiCompleteLineEdit(self)
+        self.read_lists_box = EditWithComplete(self)
         self.read_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.read_lists_box.update_items_cache(reading_lists)
         self.read_lists_box.setText(prefs['read_lists'])
@@ -727,17 +754,17 @@ class GenerateCoverTab(QWidget):
             horz.addWidget(label)
             dropdown = QComboBox(self)
             dropdown.setToolTip(s)
-            dropdown.addItem('',QVariant('none'))
+            dropdown.addItem('','none')
             for setting in gc_settings:
-                dropdown.addItem(setting,QVariant(setting))
+                dropdown.addItem(setting,setting)
             if site == _("Default"):
                 self.gc_dropdowns["Default"] = dropdown
                 if 'Default' in prefs['gc_site_settings']:
-                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings']['Default'])))
+                    dropdown.setCurrentIndex(dropdown.findData(prefs['gc_site_settings']['Default']))
             else:
                 self.gc_dropdowns[site] = dropdown
             if site in prefs['gc_site_settings']:
-                dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings'][site])))
+                dropdown.setCurrentIndex(dropdown.findData(prefs['gc_site_settings'][site]))
 
             horz.addWidget(dropdown)
             self.sl.addLayout(horz)
@@ -966,12 +993,12 @@ class CustomColumnsTab(QWidget):
                 label.setToolTip(_("Update this %s column(%s) with...")%(key,column['datatype']))
                 horz.addWidget(label)
                 dropdown = QComboBox(self)
-                dropdown.addItem('',QVariant('none'))
+                dropdown.addItem('','none')
                 for md in permitted_values[column['datatype']]:
-                    dropdown.addItem(titleLabels[md],QVariant(md))
+                    dropdown.addItem(titleLabels[md],md)
                 self.custcol_dropdowns[key] = dropdown
                 if key in prefs['custom_cols']:
-                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
+                    dropdown.setCurrentIndex(dropdown.findData(prefs['custom_cols'][key]))
                 if column['datatype'] == 'enumeration':
                     dropdown.setToolTip(_("Metadata values valid for this type of column.")+"\n"+_("Values that aren't valid for this enumeration column will be ignored."))
                 else:
@@ -1007,11 +1034,11 @@ class CustomColumnsTab(QWidget):
         horz.addWidget(label)
         self.errorcol = QComboBox(self)
         self.errorcol.setToolTip(tooltip)
-        self.errorcol.addItem('',QVariant('none'))
+        self.errorcol.addItem('','none')
         for key, column in custom_columns.iteritems():
             if column['datatype'] in ('text','comments'):
-                self.errorcol.addItem(column['name'],QVariant(key))
-        self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
+                self.errorcol.addItem(column['name'],key)
+        self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
         horz.addWidget(self.errorcol)
         self.l.addLayout(horz)
 

calibre-plugin/dialogs.py

@@ -19,12 +19,36 @@ logger = logging.getLogger(__name__)
 import urllib
 import email
 
-from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
-                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
-                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal,
-                      QGroupBox, QFrame)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame)
+
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
@@ -146,19 +170,6 @@ class RejectUrlEntry:
             
         return retval
 
-# This is a more than slightly kludgey way to get
-# EditWithComplete to *not* alpha-order the reasons, but leave
-# them in the order entered.  If
-# calibre.gui2.complete2.CompleteModel.set_items ever changes,
-# this function will need to also.
-def complete_model_set_items_kludge(self, items):
-    items = [unicode(x.strip()) for x in items]
-    items = [x for x in items if x]
-    items = tuple(items)
-    self.all_items = self.current_items = items
-    self.current_prefix = ''
-    self.reset()
-    
 class NotGoingToDownload(Exception):
     def __init__(self,error,icon='dialog_error.png'):
         self.error=error
@@ -196,9 +207,9 @@ class DroppableQTextEdit(QTextEdit):
                             urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                 else:
                     urllist.extend(get_urls_from_text("%s"%msg))
-        
         if urllist:
             self.append("\n".join(urllist))
+            return None
         return QTextEdit.dropEvent(self,event)
         
     def canInsertFromMimeData(self, source):
@@ -559,7 +570,7 @@ class LoopProgressDialog(QProgressDialog):
                  status_prefix=_("Fetched metadata for")):
         QProgressDialog.__init__(self,
                                  init_label,
-                                 QString(), 0, len(book_list), gui)
+                                 _('Cancel'), 0, len(book_list), gui)
         self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
         self.book_list = book_list
@@ -829,11 +840,11 @@ class StoryListTableWidget(QTableWidget):
             icon = get_icon(book['icon'])
 
         status_cell = IconWidgetItem(None,icon,val)
-        status_cell.setData(Qt.UserRole, QVariant(val))
+        status_cell.setData(Qt.UserRole, val)
         self.setItem(row, 0, status_cell)
         
         title_cell = ReadOnlyTableWidgetItem(book['title'])
-        title_cell.setData(Qt.UserRole, QVariant(row))
+        title_cell.setData(Qt.UserRole, row)
         self.setItem(row, 1, title_cell)
         
         self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
@@ -848,7 +859,7 @@ class StoryListTableWidget(QTableWidget):
         books = []
         #print("=========================\nbooks:%s"%self.books)
         for row in range(self.rowCount()):
-            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
+            rnum = convert_qvariant(self.item(row, 1).data(Qt.UserRole))
             book = self.books[rnum]
             books.append(book)
         return books
@@ -914,15 +925,12 @@ class RejectListTableWidget(QTableWidget):
     def populate_table_row(self, row, rej):
 
         url_cell = ReadOnlyTableWidgetItem(rej.url)
-        url_cell.setData(Qt.UserRole, QVariant(rej.book_id))
+        url_cell.setData(Qt.UserRole, rej.book_id)
         self.setItem(row, 0, url_cell)
         self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
         self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
         
-        note_cell = EditWithComplete(self)
-        note_cell.lineEdit().mcompleter.model().set_items = \
-            partial(complete_model_set_items_kludge,
-                    note_cell.lineEdit().mcompleter.model())
+        note_cell = EditWithComplete(self,sort_func=lambda x:1)
         
         items = [rej.note]+self.rejectreasons
         note_cell.update_items_cache(items)
@@ -992,10 +1000,7 @@ class RejectListDialog(SizePersistedDialog):
         button_layout.addItem(spacerItem1)
 
         if show_all_reasons:
-            self.reason_edit = EditWithComplete(self)
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
             
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)
@@ -1037,7 +1042,7 @@ class RejectListDialog(SizePersistedDialog):
         rejectrows = []
         for row in range(self.rejects_table.rowCount()):
             url = unicode(self.rejects_table.item(row, 0).text()).strip()
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id =convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
             title = unicode(self.rejects_table.item(row, 1).text()).strip()
             auth = unicode(self.rejects_table.item(row, 2).text()).strip()
             note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
@@ -1047,7 +1052,7 @@ class RejectListDialog(SizePersistedDialog):
     def get_reject_list_ids(self):
         rejectrows = []
         for row in range(self.rejects_table.rowCount()):
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id = convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
             if book_id:
                 rejectrows.append(book_id)
         return rejectrows
@@ -1089,11 +1094,7 @@ class EditTextDialog(QDialog):
             self.textedit.setToolTip(tooltip)
 
         if rejectreasons or reasonslabel:
-            self.reason_edit = EditWithComplete(self)
-
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
             
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)

calibre-plugin/ffdl_plugin.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging
@@ -19,10 +19,12 @@ import urllib
 import email
 import traceback
 
-from PyQt4.Qt import (QApplication, QMenu, QToolButton, QTimer)
-
-from PyQt4.Qt import QPixmap, Qt
-from PyQt4.QtCore import QBuffer
+try:
+    from PyQt5.Qt import (QApplication, QMenu, QTimer)
+    from PyQt5.QtCore import QBuffer
+except ImportError as e:
+    from PyQt4.Qt import (QApplication, QMenu, QTimer)
+    from PyQt4.QtCore import QBuffer
 
 from calibre.constants import numeric_version as calibre_version
 
@@ -207,7 +209,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 #print("text/plain:%s"%event.mimeData().data(mimetype))
                 urllist.extend(get_urls_from_text(event.mimeData().data(mimetype)))
 
-        #print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
+        # print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
         if urllist or dropped_ids:
             QTimer.singleShot(1, partial(self.do_drop,
                                          dropped_ids=dropped_ids,
@@ -385,6 +387,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
     def get_urls_from_page_menu(self):
 
+        urltxt = ""
         if prefs['urlsfromclip']:
             try:
                 urltxt = self.get_urls_clip(storyurls=False)[0]
@@ -712,6 +715,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # No need to do anything with perfs here, but we could.
         prefs
 
+    def make_id_searchstr(self,url):
+        # older idents can be uri vs url and have | instead of : after
+        # http, plus many sites are now switching to https.
+        return 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url))
+
     def prep_downloads(self, options, books, merge=False, extrapayload=None):
         '''Fetch metadata for stories from servers, launch BG job when done.'''
 
@@ -764,7 +772,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if not merge: # skip reject list when merging.
             if rejecturllist.check(url):
                 rejnote = rejecturllist.get_full_note(url)
-                if question_dialog(self.gui, _('Reject URL?'),'''
+                if prefs['reject_always'] or question_dialog(self.gui, _('Reject URL?'),'''
                           <h3>%s</h3>
                           <p>%s</p>
                           <p>"<b>%s</b>"</p>
@@ -857,7 +865,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         series = story.getMetadata('series')
         if not merge and series and prefs['checkforseriesurlid']:
             # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
+            searchstr = self.make_id_searchstr(story.getMetadata('seriesUrl'))
             identicalbooks = db.search_getting_ids(searchstr, None)
             # print("searchstr:%s"%searchstr)
             # print("identicalbooks:%s"%identicalbooks)
@@ -934,7 +942,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 logger.debug("from URL(%s)"%url)
     
                 # try to find by identifier url or uri first.
-                searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\:','https?(\:|\|)',url)
+                searchstr = self.make_id_searchstr(url)
                 identicalbooks = db.search_getting_ids(searchstr, None)
                 # print("searchstr:%s"%searchstr)
                 # print("identicalbooks:%s"%identicalbooks)
@@ -1797,6 +1805,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['url'] = ''
         book['site'] = ''
         book['added'] = False
+        book['pubdate'] = None
         return book
     
     def convert_urls_to_books(self, urls):
@@ -1962,9 +1971,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         
             # fill from first of each if not already present:
             for k in ('pubdate', 'timestamp', 'updatedate'):
-                if k not in b: # not in this book?  Skip it.
+                if k not in b or not b[k]: # not in this book?  Skip it.
                     continue
-                if k not in book: # first is good enough for publisher.
+                if k not in book or not book[k]: # first is good enough for publisher.
                     book[k]=b[k]
 
                 # Do these even on first to get the all_metadata settings.
@@ -2013,8 +2022,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             book['comments'] = existingbook['comments']
         else:
             book['title'] = deftitle = book_list[0]['title']
-            book['comments'] = _("Anthology containing:")+"\n" + \
-                "\n".join([ _("%s by %s")%(b['title'],', '.join(b['author'])) for b in book_list ])
+            if len(book['author']) > 1:
+                book['comments'] = _("Anthology containing:")+"\n" + \
+                    "\n".join([ _("%s by %s")%(b['title'],', '.join(b['author'])) for b in book_list ])
+            else:
+                book['comments'] = _("Anthology containing:")+"\n" + \
+                    "\n".join([ b['title'] for b in book_list ])
             # book['all_metadata']['description']
         
             # if all same series, use series for name.  But only if all and not previous named

calibre-plugin/jobs.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 

calibre-plugin/prefs.py

@@ -25,6 +25,7 @@ default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
 Dup from another site'''
+default_prefs['reject_always'] = False
 
 default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False

defaults.ini

@@ -626,6 +626,18 @@ extracategories:The Sentinel
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[bdsm-geschichten.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## This site offers no index page so we can either guess the chapter URLs
+## by dec/incrementing numbers ('guess') or walk all the chapters in the metadata
+## parsing state ('parse'). Since guessing can lead to errors for non-standard
+## story URLs, the default is to parse
+#find_chapters:guess 
+
 [bloodshedverse.com]
 ## website encoding(s) In theory, each website reports the character
 ## encoding they use for each page.  In practice, some sites report it
@@ -634,7 +646,7 @@ extracategories:The Sentinel
 ## explicitly set the encoding and order if you need to.  The special
 ## value 'auto' will call chardet and use the encoding it reports if
 ## it has +90% confidence.  'auto' is not reliable.
-website_encodings:ISO-8859-1,auto
+website_encodings:Windows-1252,ISO-8859-1,auto
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -999,7 +1011,7 @@ extraships:Kirk/Spock
 [literotica.com]
 extra_valid_entries:eroticatags
 eroticatags_label:Erotica Tags
-#extra_titlepage_entries: eroticatags
+extra_titlepage_entries: eroticatags
 
 [lumos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
@@ -1230,6 +1242,12 @@ extraships:Harry Potter/Draco Malfoy
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds
 
+[themaplebookshelf.com]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [themasque.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1268,6 +1286,10 @@ extracategories:Harry Potter
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1
 
+[tolkienfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [trekiverse.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1383,8 +1405,11 @@ type_label:Type of Couple
 [www.fanfiction.net]
 user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
-## Change this to false to use them anyway.
-never_make_cover: true
+## Set this to true to never use them.
+#never_make_cover: false
+
+## fanfiction.net shows the user's 
+cover_exclusion_regexp:/imageu/
 
 ## fanfiction.net is blocking people more aggressively.  If you
 ## download fewer stories less often you can likely get by with
@@ -1808,6 +1833,9 @@ extracharacters:Wolverine,Rogue
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
 
+extra_valid_entries:reviews
+reviews_label:Reviews
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

downloader.py

@@ -227,9 +227,9 @@ def main(argv,
            except:
                options.update = False
                pass
-       
-       ## Check for include_images and absence of PIL, give warning.
-       if adapter.getConfig('include_images'):
+
+       ## Check for include_images without no_image_processing. In absence of PIL, give warning.
+       if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
            try:
                from calibre.utils.magick import Image
                logging.debug("Using calibre.utils.magick")

fanficdownloader/adapters/__init__.py

@@ -130,11 +130,14 @@ import adapter_nocturnallightnet
 import adapter_fanfichu
 import adapter_fanfictioncsodaidokhu
 import adapter_fictionmaniatv
+import adapter_bdsmgeschichten
+import adapter_tolkienfanfiction
+import adapter_themaplebookshelf
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
 ## to pick out the adapter.
-    
+
 ## List of registered site adapters.
 __class_list = []
 __domain_map = {}
@@ -202,7 +205,7 @@ def getConfigSectionFor(url):
     (cls,fixedurl) = getClassFor(url)
     if cls:
         return cls.getConfigSection()
-    
+
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
 
@@ -234,9 +237,9 @@ def getClassFor(url):
 
     if cls:
         fixedurl = cls.stripURLParameters(fixedurl)
-    
+
     return (cls,fixedurl)
-    
+
 def getClassFromList(domain):
     try:
         return __domain_map[domain]

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -165,11 +165,11 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r"^/works/\d+$"))
+        a = soup.find('a', href=re.compile(r"/works/\d+$"))
         self.story.setMetadata('title',stripHTML(a))
 		
         # Find authorid and URL from... author url.
-        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        alist = soup.findAll('a', href=re.compile(r"/users/\w+/pseuds/\w+"))
         if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
             self.story.setMetadata('author','Anonymous')
             self.story.setMetadata('authorUrl','http://archiveofourown.org/')

fanficdownloader/adapters/adapter_bdsmgeschichten.py

@@ -0,0 +1,346 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+def _translate_date_german_english(date):
+    fullmon = {"Januar":"01",
+               "Februar":"02",
+               u"März":"03",
+               "April":"04",
+               "Mai":"05",
+               "Juni":"06",
+               "Juli":"07",
+               "August":"08",
+               "September":"09",
+               "Oktober":"10",
+               "November":"11",
+               "Dezember":"12"}
+    for (name,num) in fullmon.items():
+        date = date.replace(name,num)
+    return date
+
+_REGEX_TRAILING_DIGIT = re.compile("(\d+)$")
+_REGEX_DASH_TO_END = re.compile("-[^-]+$")
+_REGEX_CHAPTER_TITLE = re.compile(ur"""
+    \s*
+    [\u2013-]?
+    \s*
+    ([\dIVX-]+)?
+    \.?
+    \s*
+    [\[\(]?
+    \s*
+    (Teil|Kapitel|Tag)?
+    \s*
+    ([\dIVX-]+)?
+    \s*
+    [\]\)]?
+    \s*
+    $
+""", re.VERBOSE)
+_INITIAL_STEP = 5
+
+class BdsmGeschichtenAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8", "Windows-1252"]
+
+        self.story.setMetadata('siteabbrev','bdsmgesch')
+
+        # Replace possible chapter numbering
+        chapterMatch = _REGEX_TRAILING_DIGIT.search(url)
+        if chapterMatch is None:
+            self.maxChapter = 1
+        else:
+            self.maxChapter = int(chapterMatch.group(1))
+        # url = re.sub(_REGEX_TRAILING_DIGIT, "1", url)
+
+        # set storyId
+        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId'))
+
+        # normalize URL
+        self._setURL('http://%s/%s' % (self.getSiteDomain(), self.story.getMetadata('storyId')))
+
+        self.dateformat = '%d. %m %Y - %H:%M'
+
+    @staticmethod
+    def getSiteDomain():
+        return 'bdsm-geschichten.net'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.bdsm-geschichten.net', 'www.bdsm-geschichten.net']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://www.bdsm-geschichten.net/title-of-story-1 http://bdsm-geschichten.net/title-of-story-1"
+
+    def getSiteURLPattern(self):
+        return r"http://(www\.)?bdsm-geschichten.net/(?P<storyId>[a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+
+        try:
+            data1 = self._fetchUrl(self.url)
+            soup = bs.BeautifulSoup(data1)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #strip comments from soup
+        [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # Cache the soups so we won't have to redownload in getChapterText later
+        self.soupsCache = {}
+        self.soupsCache[self.url] = soup
+
+        # author
+        authorDiv = soup.find("div", "author-pane-line author-name")
+        authorId = authorDiv.string.strip()
+        self.story.setMetadata('authorId', authorId)
+        self.story.setMetadata('author', authorId)
+        # TODO not really true need to be loggedin for this to work or fetch userid
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+authorId)
+
+        # TODO better metadata
+        date = soup.find("div", {"class": "submitted"}).string.strip()
+        date = re.sub(" &#151;.*", "", date)
+        date = _translate_date_german_english(date)
+        self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+        title1 = soup.find("h1", {'class': 'title'}).string
+
+
+        for tagLink in soup.find("ul", "taxonomy").findAll("a"):
+            self.story.addToList('category', tagLink.string)
+
+        ## Retrieve chapter soups
+        if self.getConfig('find_chapters') == 'guess':
+            self.chapterUrls = []
+            self._find_chapters_by_guessing(title1)
+        else:
+            self._find_chapters_by_parsing(soup)
+
+        firstChapterUrl = self.chapterUrls[0][1]
+        if firstChapterUrl in self.soupsCache:
+            firstChapterSoup = self.soupsCache[firstChapterUrl]
+            h1 = firstChapterSoup.find("h1").text
+        else:
+            h1 = soup.find("h1").text
+
+        h1 = re.sub(_REGEX_CHAPTER_TITLE, "", h1)
+        self.story.setMetadata('title', h1)
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+        return
+
+    def _find_chapters_by_parsing(self, soup):
+
+        # store original soup
+        origSoup = soup
+
+        #
+        # find first chapter
+        #
+        firstLink = None
+        firstLinkDiv = soup.find("div", "field-field-erster-teil")
+        if firstLinkDiv is not None:
+            firstLink = "http://%s%s" % (self.getSiteDomain(), firstLinkDiv.findNext("a")['href'])
+            logger.debug("Found first chapter right away <%s>" % firstLink)
+            try:
+                soup = bs.BeautifulSoup(self._fetchUrl(firstLink))
+                self.soupsCache[firstLink] = soup
+                self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(self.url)
+                else:
+                    raise exceptions.StoryDoesNotExist(firstLink)
+        else:
+            logger.debug("DIDN'T find first chapter right away")
+            # parse previous Link until first
+            while True:
+                prevLink = None
+                prevLinkDiv = soup.find("div", "field-field-vorheriger-teil")
+                if prevLinkDiv is not None:
+                    prevLink = prevLinkDiv.find("a")
+                if prevLink is None:
+                    prevLink = soup.find("a", text=re.compile("&lt;&lt;&lt;")) # <<<
+                if prevLink is None:
+                    logger.debug("Couldn't find prev part")
+                    break
+                else:
+                    logger.debug("Previous Chapter <%s>" % prevLink)
+                    if type(prevLink) != bs.Tag or prevLink.name != "a":
+                        prevLink = prevLink.findParent("a")
+                    if prevLink is None or '#' in prevLink['href']:
+                        logger.debug("Couldn't find prev part (false positive) <%s>" % prevLink)
+                        break
+                    prevLink = prevLink['href']
+                try:
+                    soup = bs.BeautifulSoup(self._fetchUrl(prevLink))
+                    self.soupsCache[prevLink] = soup
+                    prevTtitle = soup.find("h1", {'class': 'title'}).string
+                    self.chapterUrls.insert(0, (prevTtitle, prevLink))
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(nextLink)
+                    else:
+                        raise e
+                firstLink = prevLink
+
+        # if first chapter couldn't be determined, assume the URL originally
+        # passed is the first chapter
+        if firstLink is None:
+            logger.debug("Couldn't set first chapter")
+            firstLink = self.url
+            self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
+
+        # set first URL
+        logger.debug("Set first link: %s" % firstLink)
+        self._setURL(firstLink)
+        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(firstLink).group('storyId'))
+
+        #
+        # Parse next chapters
+        #
+        while True:
+            nextLink = None
+            nextLinkDiv = soup.find("div", "field-field-naechster-teil")
+            if nextLinkDiv is not None:
+                nextLink = nextLinkDiv.find("a")
+            if nextLink is None:
+                nextLink = soup.find("a", text=re.compile("&gt;&gt;&gt;"))
+            if nextLink is None:
+                nextLink = soup.find("a", text=re.compile("Fortsetzung"))
+
+            if nextLink is None:
+                logger.debug("Couldn't find next part")
+                break
+            else:
+                if type(nextLink) != bs.Tag or nextLink.name != "a":
+                    nextLink = nextLink.findParent("a")
+                if nextLink is None or '#' in nextLink['href']:
+                    logger.debug("Couldn't find next part (false positive) <%s>" % nextLink)
+                    break
+                nextLink = nextLink['href']
+
+            if not nextLink.startswith('http:'):
+                nextLink = 'http://' + self.getSiteDomain() + nextLink
+
+            for loadedChapter in self.chapterUrls:
+                if loadedChapter[0] == nextLink:
+                    logger.debug("ERROR: Repeating chapter <%s> Try to fix it" % nextLink)
+                    nextLinkMatch = _REGEX_TRAILING_DIGIT.match(nextLink)
+                    if nextLinkMatch is not None:
+                        curChap = nextLinkMatch.group(1)
+                        nextLink = re.sub(_REGEX_TRAILING_DIGIT, str(int(curChap) + 1), nextLink)
+                    else:
+                        break
+            try:
+                data = self._fetchUrl(nextLink)
+                soup = bs.BeautifulSoup(data)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(nextLink)
+                else:
+                    raise e
+            title2 = soup.find("h1", {'class': 'title'}).string
+            self.chapterUrls.append((title2, nextLink))
+            logger.debug("Grabbing next chapter URL " + nextLink)
+            self.soupsCache[nextLink] = soup
+            # [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+        logger.debug("Chapters: %s" % self.chapterUrls)
+
+
+    def _find_chapters_by_guessing(self, title1):
+        step = _INITIAL_STEP
+        curMax = self.maxChapter + step
+        lastHit = True
+        while True:
+            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(curMax), self.url) 
+            if nextChapterUrl == self.url:
+                logger.debug("Unable to guess next chapter because URL doesn't end in numbers")
+                break;
+            try:
+                logger.debug("Trying chapter URL " + nextChapterUrl)
+                data = self._fetchUrl(nextChapterUrl)
+                hit = True
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    hit = False
+                else:
+                    raise e
+            if hit:
+                logger.debug("Found chapter URL " + nextChapterUrl)
+                self.maxChapter = curMax
+                self.soupsCache[nextChapterUrl] = bs.BeautifulSoup(data)
+                if not lastHit:
+                    break
+                lastHit = curMax
+                curMax += step
+            else:
+                lastHit = False
+                curMax -= 1
+            logger.debug(curMax)
+
+        for i in xrange(1, self.maxChapter):
+            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(i), self.url)
+            nextChapterTitle = re.sub("1", str(i), title1)
+            self.chapterUrls.append((nextChapterTitle, nextChapterUrl))
+
+    def getChapterText(self, url):
+
+        if url in self.soupsCache:
+            logger.debug('Getting chapter <%s> from cache' % url)
+            soup = self.soupsCache[url]
+        else:
+            logger.debug('Downloading chapter <%s>' % url)
+            data1 = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data1)
+            #strip comments from soup
+            [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        storyDiv1 = bs.Tag(soup, "div")
+        for para in soup.find("div", "full-node").find('div', 'content').findAll("p"):
+            storyDiv1.append(para)
+        storyDiv1.append('<br />')
+        storytext = self.utf8FromSoup(url,storyDiv1)
+
+        return storytext
+
+
+def getClass():
+    return BdsmGeschichtenAdapter

fanficdownloader/adapters/adapter_bloodshedversecom.py

@@ -4,6 +4,7 @@ import urllib2
 import urlparse
 
 from .. import BeautifulSoup
+from ..htmlcleanup import stripHTML
 
 from base_adapter import BaseSiteAdapter, makeDate
 from .. import exceptions
@@ -74,11 +75,11 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
         # Since no 404 error code we have to raise the exception ourselves.
         # A title that is just 'by' indicates that there is no author name
         # and no story title available.
-        if soup.title.string.strip() == 'by':
+        if stripHTML(soup.title) == 'by':
             raise exceptions.StoryDoesNotExist(self.url)
 
         for option in soup.find('select', {'name': 'chapter'}):
-            title = option.string.strip()
+            title = stripHTML(option)
             url = self.READ_URL_TEMPLATE % option['value']
             self.chapterUrls.append((title, url))
 
@@ -101,15 +102,15 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
             raise exceptions.FailedToDownload(self.url)
 
         title_anchor = list_box.find('a', {'class': 'fictitle'})
-        self.story.setMetadata('title', title_anchor.string.strip())
+        self.story.setMetadata('title', stripHTML(title_anchor))
 
         author_anchor = title_anchor.findNextSibling('a')
-        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('author', stripHTML(author_anchor))
         self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
         self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
 
         list_review = list_box.find('div', {'class': 'list_review'})
-        reviews = list_review.a.string.strip().split(' ', 1)[0]
+        reviews = stripHTML(list_review.a).split(' ', 1)[0]
         self.story.setMetadata('reviews', reviews)
 
         summary_div = list_box.find('div', {'class': 'list_summary'})
@@ -122,7 +123,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
 
         # I'm assuming this to be the category, not sure what else it could be
         first_listinfo = list_box.find('div', {'class': 'list_info'})
-        self.story.addToList('category', first_listinfo.a.string.strip())
+        self.story.addToList('category', stripHTML(first_listinfo.a))
 
         for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
             for b_tag in list_info('b'):

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -74,7 +74,8 @@ class EFPFanFicNet(BaseSiteAdapter):
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
-        if 'Fai il login e leggi la storia!' in data:
+        if( 'Fai il login e leggi la storia!' in data or
+            'Questa storia presenta contenuti non adatti ai minori' in data ):
             return True
         else:
             return False

fanficdownloader/adapters/adapter_literotica.py

@@ -39,64 +39,97 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        
+
         self.story.setMetadata('siteabbrev','litero')
 
         # normalize to first chapter.  Not sure if they ever have more than 2 digits.
-        storyid = self.parsedUrl.path.split('/',)[2]
-        if re.match(r'-ch\d\d$',storyid):
-            storyid = storyid[:-2]+'01'
-        self.story.setMetadata('storyId',storyid)
-        
-        self.origurl = url
-        if "//www.i." in self.origurl:
-            ## accept m(mobile)url, but use www.
-            self.origurl = self.origurl.replace("//www.i.","//www.")
+        storyId = self.parsedUrl.path.split('/',)[2]
+        # replace later chapters with first chapter but don't remove numbers
+        # from the URL that disambiguate stories with the same title.
+        storyId = re.sub("-ch-?\d\d", "", storyId)
+        self.story.setMetadata('storyId', storyId)
 
-        # normalized story URL.
-        self._setURL(url[:url.index('//')+2]+self.getSiteDomain()\
-                         +"/s/"+self.story.getMetadata('storyId'))
+        ## accept m(mobile)url, but use www.
+        url = re.sub("^(www|german|spanish|french|dutch|italian|romanian|portuguese|other)\.i",
+                              "\1",
+                              url)
+
+        ## strip ?page=...
+        url = re.sub("\?page=.*$", "", url)
+
+        ## set url
+        self._setURL(url)
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = '%m/%d/%y'
-        
+
     @staticmethod
     def getSiteDomain():
-        return 'www.literotica.com'
+        return 'literotica.com'
 
     @classmethod
     def getAcceptDomains(cls):
-        return ['www.literotica.com', 'www.i.literotica.com']
+        return ['www.literotica.com',
+                'www.i.literotica.com',
+                'german.literotica.com',
+                'german.i.literotica.com',
+                'spanish.literotica.com',
+                'spanish.i.literotica.com',
+                'french.literotica.com',
+                'french.i.literotica.com',
+                'dutch.literotica.com',
+                'dutch.i.literotica.com',
+                'italian.literotica.com',
+                'italian.i.literotica.com',
+                'romanian.literotica.com',
+                'romanian.i.literotica.com',
+                'portuguese.literotica.com',
+                'portuguese.i.literotica.com',
+                'other.literotica.com',
+                'other.i.literotica.com']
 
     @classmethod
     def getSiteExampleURLs(self):
-        #return "http://www.literotica.com/s/story-title http://www.literotica.com/stories/showstory.php?id=1234 http://www.i.literotica.com/stories/showstory.php?id=1234"
-        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title"
+        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title http://portuguese.literotica.com/s/story-title http://german.literotica.com/s/story-title"
 
     def getSiteURLPattern(self):
-        return r"https?://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+        return r"https?://(www|german|spanish|french|dutch|italian|romanian|portuguese|other)(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
 
     def extractChapterUrlsAndMetadata(self):
+        """
+        NOTE: Some stories can have versions, 
+              e.g. /my-story-ch-05-version-10
+        NOTE: If two stories share the same title, a running index is added,
+              e.g.: /my-story-ch-02-1
+        Strategy:
+            * Go to author's page, search for the current story link,
+            * If it's in a tr.root-story => One-part story
+                * , get metadata and be done
+            * If it's in a tr.sl => Chapter in series
+                * Search up from there until we find a tr.ser-ttl (this is the
+                story)
+                * Gather metadata
+                * Search down from there for all tr.sl until the next
+                tr.ser-ttl, foreach
+                    * Chapter link is there
+        """
 
         if not (self.is_adult or self.getConfig("is_adult")):
             raise exceptions.AdultCheckRequired(self.url)
-        
-        url1 = self.origurl
-        logger.debug("first page URL: "+url1)
 
+        logger.debug("Chapter/Story URL: <%s> " % self.url)
         try:
-            data1 = self._fetchUrl(url1)
+            data1 = self._fetchUrl(self.url)
             soup1 = bs.BeautifulSoup(data1)
+            #strip comments from soup
+            [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
         except urllib2.HTTPError, e:
             if e.code == 404:
-                raise exceptions.StoryDoesNotExist(url1)
+                raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
 
-        #strip comments from soup
-        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
-
         # author
         a = soup1.find("span", "b-story-user-y")
         self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'][0])
@@ -110,105 +143,77 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         try:
             dataAuth = self._fetchUrl(authorurl)
             soupAuth = bs.BeautifulSoup(dataAuth)
+            #strip comments from soup
+            [comment.extract() for comment in soupAuth.findAll(text=lambda text:isinstance(text, bs.Comment))]
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(authorurl)
             else:
                 raise e
 
+        ## Find link to url in author's page
         ## site has started using //domain.name/asdf urls remove https?: from front
-        storyLink = soupAuth.find('a', href=url1[url1.index(':')+1:])
+        storyLink = soupAuth.find('a', href=self.url[self.url.index(':')+1:])
 
         if storyLink is not None:
-            # pull the published date from the author page
-            # default values from single link.  Updated below if multiple chapter.
-            date = storyLink.parent.parent.findAll('td')[-1].text
+            urlTr = storyLink.parent.parent
+            if urlTr['class'] == "sl":
+                isSingleStory = False
+            else:
+                isSingleStory = True
+        else:
+            raise exceptions.FailedToDownload("Couldn't find story <%s> on author's page <%s>" % (url, authorurl))
+
+        if isSingleStory:
+            self.story.setMetadata('title', storyLink.text)
+            self.story.setMetadata('description', urlTr.findAll("td")[1].text)
+            self.story.addToList('eroticatags', urlTr.findAll("td")[2].text)
+            date = urlTr.findAll('td')[-1].text
             self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
             self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
-
-        # find num of pages
-        # find a "3 Pages:" string on the page and parse it
-        pgs = soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0]
-
-        # If there are multiple pages, find and request the last page
-        if "1" != pgs:
-            logger.debug("last page number: "+pgs)
-            try:
-                data2 = self._fetchUrl(url1, {'page': pgs})
-                soup2 = bs.BeautifulSoup(data2)
-                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    # TODO: Probably should reformat this
-                    raise exceptions.StoryDoesNotExist(url1, {'page': pgs})
-                else:
-                    raise e
+            self.chapterUrls = [(storyLink.text, self.url)]
         else:
-            #If we're already on the last page, copy the soup
-            soup2 = soup1
+            seriesTr = urlTr.previousSibling
+            while seriesTr['class'] != 'ser-ttl':
+                seriesTr = seriesTr.previousSibling
+            m = re.match("^(?P<title>.*?):\s(?P<numChapters>\d+)\sPart\sSeries$", seriesTr.find("strong").text)
+            self.story.setMetadata('title', m.group('title'))
+            self.story.setMetadata('numChapters', int(m.group('numChapters')))
 
-        # parse out the list of chapters
-        chaps = soup2.find('div', id='b-series')
-        if chaps:  # may be one post only
-            #self.chapterUrls = [(ch.a.text, ch.a['href']) for ch in chaps.findAll('li')]
-
-            # if there are chapters, lets pull them and title from the
-            # author page because *this* chapter is omitted from the
-            # list on the last page.
-            row = storyLink.parent.parent.previousSibling
-            while row['class'] != 'ser-ttl':
-                row = row.previousSibling
-
-            seriesTitle = stripHTML(row)
-            if seriesTitle:
-                # this regex is deliberately greedy. We want to get the biggest match before a ':'
-                self.story.setMetadata('title', re.match('(.*):[^:]*$', seriesTitle).group(1))
-            else:
-                self.story.setMetadata('title', soup1.h1.string)
-
-            # now chapter list.  Assumed oldest to newest.
+            ## Walk the chapters
+            chapterTr = seriesTr.nextSibling
             self.chapterUrls = []
-            row = row.nextSibling
-                
-            self.story.setMetadata('datePublished',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
-            while row['class'] == 'sl':
-                # pages include full URLs.
-                chapurl = row.a['href']
-                if chapurl.startswith('//'):
-                    chapurl = self.parsedUrl.scheme+':'+chapurl
-                self.chapterUrls.append((row.a.string,chapurl))
-                if not row.nextSibling:
-                    break
-                row = row.nextSibling
+            dates = []
+            descriptions = []
+            while chapterTr is not None and chapterTr['class'] == 'sl':
+                descriptions.append(chapterTr.findAll("td")[1].text)
+                chapterLink = chapterTr.find("td", "fc").find("a")
+                self.chapterUrls.append((chapterLink.text, "http:" + chapterLink["href"]))
+                self.story.addToList('eroticatags', chapterTr.findAll("td")[2].text)
+                dates.append(makeDate(chapterTr.findAll('td')[-1].text, self.dateformat))
+                chapterTr = chapterTr.nextSibling
 
-            row = row.previousSibling
-            self.story.setMetadata('dateUpdated',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
-                
-        else:  # if one post only
-            self.chapterUrls = [(soup1.h1.string, url1)]
-            self.story.setMetadata('title', soup1.h1.string)
+            ## Set description to joint chapter descriptions
+            self.story.setMetadata('description', " / ".join(descriptions))
 
-        # normalize on first chapter URL.
-        self._setURL(self.chapterUrls[0][1])
+            ## Set the oldest date as publication date, the newest as update date
+            dates.sort()
+            self.story.setMetadata('datePublished', dates[0])
+            self.story.setMetadata('dateUpdated', dates[-1])
 
-        # reset storyId to first chapter.
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+            # normalize on first chapter URL.
+            self._setURL(self.chapterUrls[0][1])
 
-        self.story.setMetadata('numChapters', len(self.chapterUrls))
-
-        self.story.setMetadata('category', soup1.find('div', 'b-breadcrumbs').findAll('a')[1].string)
-        # deliberately not self.setDescription() because it's never HTML.
-        self.story.setMetadata('description', soup1.find('meta', {'name': 'description'})['content'])
-        
-        # li tags inside div class b-s-story-tag-list
-        for li in soup1.find('div', {'class':'b-s-story-tag-list'}).findAll('a'):
-            self.story.addToList('eroticatags',stripHTML(li))
+        # set storyId to 'title-author' to avoid duplicates
+        # self.story.setMetadata('storyId', 
+        #     re.sub("[^a-z0-9]", "", self.story.getMetadata('title').lower())
+        #     + "-"
+        #     + re.sub("[^a-z0-9]", "", self.story.getMetadata('author').lower()))
 
         return
 
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5)
+        logger.debug('Getting chapter text from <%s>' % url)
         data1 = self._fetchUrl(url)
         soup1 = bs.BeautifulSoup(data1)
 

fanficdownloader/adapters/adapter_spikeluvercom.py

@@ -3,6 +3,7 @@ import urllib2
 import urlparse
 
 from .. import BeautifulSoup
+from ..htmlcleanup import stripHTML
 
 from base_adapter import BaseSiteAdapter, makeDate
 from .. import exceptions
@@ -88,25 +89,25 @@ class SpikeluverComAdapter(BaseSiteAdapter):
         soup = self._customized_fetch_url(url)
 
         pagetitle_div = soup.find('div', id='pagetitle')
-        self.story.setMetadata('title', pagetitle_div.a.string.strip())
+        self.story.setMetadata('title', stripHTML(pagetitle_div.a))
 
         author_anchor = pagetitle_div.a.findNextSibling('a')
         url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
         components = urlparse.urlparse(url)
         query_data = urlparse.parse_qs(components.query)
 
-        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('author', stripHTML(author_anchor))
         self.story.setMetadata('authorId', query_data['uid'])
         self.story.setMetadata('authorUrl', url)
 
         sort_div = soup.find('div', id='sort')
-        self.story.setMetadata('reviews', sort_div('a')[1].string.strip())
+        self.story.setMetadata('reviews', stripHTML(sort_div('a')[1]))
 
         listbox_tag = soup.find('div', {'class': 'listbox'})
         for span_tag in listbox_tag('span'):
             key = span_tag.string.strip(' :')
             try:
-                value = span_tag.nextSibling.string.strip()
+                value = stripHTML(span_tag.nextSibling)
             # This can happen with some fancy markup in the summary. Just
             # ignore this error and set value to None, the summary parsing
             # takes care of this
@@ -145,27 +146,27 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                     if sibling.name == 'br':
                         break
 
-                    self.story.addToList('category', sibling.string.strip())
+                    self.story.addToList('category', stripHTML(sibling))
 
             # Seems to be always "None" for some reason
             elif key == 'Characters':
                 for sibling in span_tag.findNextSiblings(['a', 'br']):
                     if sibling.name == 'br':
                         break
-                    self.story.addToList('characters', sibling.string.strip())
+                    self.story.addToList('characters', stripHTML(sibling))
 
             elif key == 'Genres':
                 for sibling in span_tag.findNextSiblings(['a', 'br']):
                     if sibling.name == 'br':
                         break
 
-                    self.story.addToList('genre', sibling.string.strip())
+                    self.story.addToList('genre', stripHTML(sibling))
 
             elif key == 'Warnings':
                 for sibling in span_tag.findNextSiblings(['a', 'br']):
                     if sibling.name == 'br':
                         break
-                    self.story.addToList('warnings', sibling.string.strip())
+                    self.story.addToList('warnings', stripHTML(sibling))
 
             # Challenges
 
@@ -173,7 +174,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                 a = span_tag.findNextSibling('a')
                 if not a:
                     continue
-                self.story.setMetadata('series', a.string.strip())
+                self.story.setMetadata('series', stripHTML(a))
                 self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href']))
 
             elif key == 'Chapters':
@@ -196,7 +197,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
             if not chapter_anchor:
                 continue
 
-            title = chapter_anchor.string.strip()
+            title = stripHTML(chapter_anchor)
             url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
             self.chapterUrls.append((title, url))
 

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -70,14 +70,14 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
-        if 'Free Registration' in data \
+        if self.needToLogin \
+                or 'Free Registration' in data \
                 or "Invalid Password!" in data \
                 or "Invalid User Name!" in data \
                 or "Log In" in data \
                 or "Access to unlinked chapters requires" in data:
-            return True
-        else:
-            return False
+            self.needToLogin = True
+        return self.needToLogin
         
     def performLogin(self, url):
         params = {}
@@ -114,11 +114,15 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         url = self.url
         logger.debug("URL: "+url)
 
+        self.needToLogin = False
         try:
             data = self._fetchUrl(url+":i")
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
+            elif e.code == 401:
+                self.needToLogin = True
+                data = ''
             else:
                 raise e
 

fanficdownloader/adapters/adapter_themaplebookshelf.py

@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+class TheMapleBookshelfComSiteAdapter(BaseSiteAdapter):
+    """
+    Use Printable version which is easier to parse and has everything in one
+    page and cache between extractChapterUrlsAndMetadata and getChapterText
+    """
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev','maplebook')
+        self.decode = ["Windows-1252", "utf8"]
+        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId'))
+        self._setURL("http://%s/Literati/viewstory.php?sid=%s" % (self.getSiteDomain(), self.story.getMetadata('storyId')))
+        self.dateformat = "%b %d, %Y"
+
+    @staticmethod
+    def getSiteDomain():
+        return 'themaplebookshelf.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://www.themaplebookshelf.com/Literati/viewstory.php?sid=227 http://themaplebookshelf.com/Literati/viewstory.php?sid=227&chapter=2"
+
+    def getSiteURLPattern(self):
+        return r"http://themaplebookshelf.com/Literati/viewstory.php\?sid=(?P<storyId>\d+)"
+
+    def extractChapterUrlsAndMetadata(self):
+        logger.debug(self.url)
+        self._setURL(self.url + "&action=printable")
+        try:
+            html = self._fetchUrl(self.url)
+            soup = bs.BeautifulSoup(html)
+            # #strip comments from soup
+            # [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+        ## title + author
+        pagetitleDiv = soup.find("div", {"id": "pagetitle"})
+        self.story.setMetadata('title', pagetitleDiv.find("a").text)
+        authorLink = pagetitleDiv.findAll("a")[1]
+        self.story.setMetadata('author', authorLink.text)
+        self.story.setMetadata('authorUrl', "http://" + self.getSiteDomain() + "/" + authorLink['href'])
+        self.story.setMetadata('authorId', re.search("\d+", authorLink['href']).group(0))
+
+        ## Description
+        description = ""
+        summaryEnd = soup.find("div", "content").find("span", "label").nextSibling
+        while summaryEnd is not None:
+            description += stripHTML(summaryEnd)
+            summaryEnd = summaryEnd.nextSibling
+            if type(summaryEnd) != bs.NavigableString and summaryEnd.name == 'br':
+                break
+        self.story.setMetadata('description', description)
+
+        ## General Metadata
+        for kSpan in soup.findAll("span", "label"):
+            k = kSpan.text.strip().replace(':', '')
+            vSpan = kSpan.nextSibling
+            if k == 'Summary:' or not vSpan or not vSpan.string:
+                continue
+            v = vSpan.string.strip()
+            if v == 'None':
+                continue
+            logger.debug("%s '%s'" %(k, v))
+            if k == 'Genre':
+                for genre in v.split(", "):
+                    self.story.addToList('genre', genre)
+            elif k == 'Chapters':
+                self.story.setMetadata('numChapters', int(v))
+            elif k == 'Word count':
+                self.story.setMetadata('numWords', v)
+            elif k == 'Published':
+                self.story.setMetadata('datePublished', makeDate(v, self.dateformat))
+            elif k == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(v, self.dateformat))
+            # TODO: Series, Warnings
+
+        ## Chapter URLs (fragment identifiers in the document, so we don' need to fetch so much)
+        for chapterNumB in soup.findAll("b", text=re.compile("^\d+\.$")):
+            self.chapterUrls.append((
+                chapterNumB.parent.parent.find("a").text,
+                self.url + chapterNumB.parent.parent.find("a")["href"]
+                ))
+
+        ## fix all local image 'src' to absolute
+        for img in soup.findAll("img", {"src": re.compile("^(?!http)")}):
+            img['src'] = re.sub("viewstory.php\?.*", "", self.url) + img['src']
+
+        self.html = soup
+
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from <%s>' % url)
+        anchor = url.replace(self.url, "")
+        anchor = anchor.replace("#", "")
+        chapterDiv = self.html.find("a", {"name": anchor}).parent.findNext("div", "chapter")
+        return self.utf8FromSoup(self.url, chapterDiv)
+
+def getClass():
+    return TheMapleBookshelfComSiteAdapter

fanficdownloader/adapters/adapter_tolkienfanfiction.py

@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+
+"""
+FFDL Adapter for TolkienFanFiction.com.
+
+Chapter URL: http://www.tolkienfanfiction.com/Story_Read_Chapter.php?CHid=1234
+    Metadata
+        Link to Story URL [Index]
+        chapterTitle
+        storyTitle
+Story URL: http://www.tolkienfanfiction.com/Story_Read_Head.php?STid=1034
+    Metadata
+        Links to Chapter URLs
+        storyTitle
+        chapterTitle[s]
+        author
+        authorId
+        authorUrl
+        numChapters
+        wordCount
+        description/summary
+        rating TODO
+        genre TODO
+        Characters
+        Ages (specific) TODO
+Search: http://www.tolkienfanfiction.com/Story_Chapter_Search.php?text=From+Wilderness+to+Cities+White&field=1&type=3&search=Search
+    Strategy
+        Search by exact phrase for styo
+    Metadata
+        dateUpdated
+    Parameters
+        field (field to search)
+            1: title
+            2: description
+            3: chapter text
+        type (any, all or exact phrase)
+            1: any
+            2: all
+            3: exact phrase
+
+"""
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib
+import urllib2
+import urlparse
+import string
+
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+def _is_story_url(url):
+    return "Story_Read_Head.php" in url
+
+def _latinize(text):
+    """
+    See http://stackoverflow.com/a/19114706/201318
+    """
+    src = u"áâäÉéêëíóôöúû"
+    tgt = u"aaaEeeeiooouu"
+    src_ord = [ord(char) for char in src]
+    translate_table = dict(zip(src_ord, tgt))
+    return text.translate(translate_table)
+
+def _fix_broken_markup(html):
+    """Replaces invalid comment tags"""
+    if html.startswith("<CENTER>"):
+        logger.error("TolkienFanFiction.com couldn't handle this request: '%s'" % html)
+    html = re.sub("<!-.+?->", "", html)
+    return html
+
+
+class TolkienFanfictionAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["ISO-8859-1", "Windows-1252"] 
+
+        self.story.setMetadata('siteabbrev','tolkien')
+
+        self.dateformat = '%B %d, %Y'
+
+        self._normalizeURL(url)
+
+    def _normalizeURL(self, url):
+        if _is_story_url(url):
+            self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId'))
+            self._setURL('http://' + self.getSiteDomain() + '/Story_Read_Head.php?STid=' + self.story.getMetadata('storyId'))
+
+    @staticmethod
+    def getSiteDomain():
+        return 'tolkienfanfiction.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['tolkienfanfiction.com', 'www.tolkienfanfiction.com']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return 'http://www.tolkienfanfiction.com/Story_Read_Head.php?STid=1034 http://www.tolkienfanfiction.com/Story_Read_Chapter.php?CHid=4945'
+
+    def getSiteURLPattern(self):
+        return r"http://(?:www.)?tolkienfanfiction.com/(?:Story_Read_Chapter\.php\?CH|Story_Read_Head\.php\?ST)id=(?P<storyId>[0-9]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not _is_story_url(self.url):
+            # Get the link to the index page
+            try:
+                chapterHtml = _fix_broken_markup(self._fetchUrl(self.url))
+                chapterSoup = bs.BeautifulSoup(chapterHtml)
+                indexLink = chapterSoup.find("a", text="[Index]").parent
+                self._normalizeURL('http://' + self.getSiteDomain() + '/' + indexLink.get('href'))
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(self.url)
+                else:
+                    raise e
+        logger.debug("Determined index page: <%s>" % self.url)
+
+        try:
+            indexHtml = _fix_broken_markup(self._fetchUrl(self.url))
+            soup = bs.BeautifulSoup(indexHtml)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # chapterUrls
+        for pfLink in soup.findAll("a", text='[PF] '):
+            chapterLink = pfLink.parent.findNext("a")
+            chapterTitle = chapterLink.string
+            if self.getConfig('strip_chapter_numeral'):
+                chapterTitle = re.sub("^\d+:", "", chapterTitle)
+            chapterUrl = 'http://' + self.host + '/' + chapterLink['href']
+            self.chapterUrls.append((chapterTitle, chapterUrl))
+            numChapters = len(self.chapterUrls)
+            self.story.setMetadata('numChapters', numChapters)
+        logger.debug('Number of Chapters: %s' % numChapters)
+
+        # title
+        title = soup.find("table", "headertitle").find("tr").contents[1].string
+        logger.debug("Title: '%s'" % title)
+        self.story.setMetadata('title', title)
+
+        # author
+        authorLink = soup.find("a", {"href":lambda x: x.startswith("Author_Profile.php")})
+        authorName = authorLink.find("b").string
+        authorHref = authorLink['href']
+        authorUrl = 'http:' + self.host + '/' + authorHref
+        authorId = authorHref[authorHref.index('=')+1:]
+        self.story.setMetadata('author', authorName)
+        self.story.setMetadata('authorId', authorId)
+        self.story.setMetadata('authorUrl', authorUrl)
+        logger.debug("Author: %s [%s] @ <%s>" % (authorId, authorName, authorUrl))
+
+        # numWords
+        numWordsMatch = re.search("Word Count: (\d+)<BR>", indexHtml)
+        if numWordsMatch:
+            numWords = numWordsMatch.group(1)
+            logger.debug('Number of words: %s' % numWords)
+            self.story.setMetadata('numWords', numWords)
+
+        # description
+        description = soup.find("b", text="Description:").parent.nextSibling.nextSibling
+        self.story.setDescription(description)
+        logger.debug("Summary: '%s'" % description)
+
+        # characters
+        characters = soup.find("b", text="Characters").parent.nextSibling.nextSibling.nextSibling
+        for character in characters.split(", "):
+            self.story.addToList('characters', character)
+        logger.debug("Characters: %s" % self.story.getMetadata('characters'))
+
+        logger.debug('Title as `str`: ' + str(title))
+        # For publication date we need to search
+        try:
+            queryString = urllib.urlencode((
+                ('type', 3),
+                ('field', 1),
+                # need translate here for the weird accented letters
+                ('text', _latinize(title)),
+                ('search', 'Search'),
+            ))
+            searchUrl = 'http://%s/Story_Chapter_Search.php?%s' % (self.host, queryString)
+            logger.debug("Search URL: <%s>" % searchUrl)
+            searchHtml = _fix_broken_markup(self._fetchUrl(searchUrl))
+            searchSoup = bs.BeautifulSoup(searchHtml)
+            date = searchSoup.find(text="Updated:").nextSibling.string
+            logger.debug("Last Updated: '%s'" % date)
+            self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+    def getChapterText(self, url):
+
+        logger.debug('Downloading chapter <%s>' % url)
+
+        time.sleep(0.5)
+        htmldata = _fix_broken_markup(self._fetchUrl(url))
+        soup = bs.BeautifulSoup(htmldata)
+
+        #strip comments from soup
+        [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # Strip redundant headings
+        [font.parent.extract() for font in soup.findAll("font", {"size": "4"})]
+
+        # get story text
+        textDiv = soup.find("div", "text")
+        return self.utf8FromSoup(url, textDiv)
+
+def getClass():
+    return TolkienFanfictionAdapter

fanficdownloader/adapters/adapter_wraithbaitcom.py

@@ -125,6 +125,10 @@ class WraithBaitComAdapter(BaseSiteAdapter):
         rating=pt.text.split('[')[1].split(']')[0]
         self.story.setMetadata('rating', rating)
 
+        st = soup.find('div', {'class' : 'storytitle'})
+        a = st.findAll('a', href=re.compile(r'reviews.php\?type=ST&item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
+        self.story.setMetadata('reviews',stripHTML(a))
+        
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # include author on chapters if multiple authors.

fanficdownloader/adapters/base_adapter.py

@@ -297,9 +297,10 @@ class BaseSiteAdapter(Configurable):
     @classmethod
     def getSiteExampleURLs(self):
         """
+        Return a string of space separated example URLs.
         Needs to be overriden in each adapter class.  It's the adapter
         writer's responsibility to make sure the example(s) pass the
-        URL validate.
+        validateURL method.
         """
         return 'no such example'
     
@@ -328,7 +329,8 @@ class BaseSiteAdapter(Configurable):
 
     def setCoverImage(self,storyurl,imgurl):
         if self.getConfig('include_images'):
-            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True)
+            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True,
+                                 coverexclusion=self.getConfig('cover_exclusion_regexp'))
 
     # This gives us a unicode object, not just a string containing bytes.
     # (I gave soup a unicode string, you'd think it could give it back...)

fanficdownloader/epubutils.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging

fanficdownloader/story.py

@@ -324,7 +324,7 @@ class Story(Configurable):
                 self.in_ex_cludes[ie] = self.set_in_ex_clude(ies)
 
     def join_list(self, key, vallist):
-        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(vallist)
+        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist))
                 
     def setMetadata(self, key, value, condremoveentities=True):
 
@@ -345,7 +345,7 @@ class Story(Configurable):
             except:
                 self.setMetadata('langcode','en')
                 
-        if key == 'dateUpdated':
+        if key == 'dateUpdated' and value:
             # Last Update tags for Bill.
             self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
@@ -409,6 +409,7 @@ class Story(Configurable):
     ## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp]
     def setReplace(self,replace):
         for line in replace.splitlines():
+            # print("replacement line:%s"%line)
             (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None)
             if "&&" in line:
                 (line,conditional) = line.split("&&")
@@ -440,6 +441,7 @@ class Story(Configurable):
             if replaceline in seen_list: # recursion on pattern, bail
                 # print("bailing on %s"%replaceline)
                 continue
+            #print("replacement tuple:%s"%replaceline)
             (metakeys,regexp,replacement,condkey,condregexp) = replaceline
             if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
@@ -464,7 +466,9 @@ class Story(Configurable):
                                                                seen_list=seen_list+[replaceline]))
                         break
                     else:
-                        retlist = [regexp.sub(replacement,value)]
+                        # print("replacement,value:%s,%s->%s"%(replacement,value,regexp.sub(replacement,value)))
+                        value = regexp.sub(replacement,value)
+                        retlist = [value]
                     
         for val in retlist:
             retlist = map(partial(self.do_in_ex_clude,'include_metadata_post',key=key),retlist)
@@ -649,7 +653,7 @@ class Story(Configurable):
         # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
         # use replace_metadata to change separator first if needed.
         # ships=>[ ]*(/|&|&)[ ]*=>/
-        if listname == 'ships' and self.getConfig('sort_ships'):
+        if listname == 'ships' and self.getConfig('sort_ships') and retlist:
             retlist = [ '/'.join(sorted(x.split('/'))) for x in retlist ]
 
         if retlist:
@@ -768,6 +772,10 @@ class Story(Configurable):
                      '','',''))
                 #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
 
+        # apply coverexclusion to explicit covers, too.  Primarily for ffnet imageu.
+        if cover and coverexclusion and re.search(coverexclusion,imgurl):
+            return
+
         prefix='ffdl'
         if imgurl not in self.imgurls:
             parsedUrl = urlparse.urlparse(imgurl)
@@ -799,7 +807,7 @@ class Story(Configurable):
                 return "failedtoload"
 
             # explicit cover, make the first image.
-            if cover and not self.getConfig('never_make_cover'):
+            if cover:
                 if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']:
                     # remove existing cover, if there is one.
                     del self.imgurls[0]
@@ -818,7 +826,7 @@ class Story(Configurable):
                 if self.cover == None and \
                         self.getConfig('make_firstimage_cover') and \
                         not self.getConfig('never_make_cover') and \
-                        (not coverexclusion or not re.search(coverexclusion,imgurl)):
+                        not (coverexclusion and re.search(coverexclusion,imgurl)):
                     newsrc = "images/cover.%s"%ext
                     self.cover=newsrc
                     self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})

index.html

@@ -60,10 +60,9 @@
 	  </p>
 	  <p>
 	    <ul>
-              <li>New site: fictionmania.tv -- Thanks, cryzed!</li>
-              <li>Fix for site recognizer to handle with/without www. using https URLs.</li>
-              <li>Fix for some utf8 descriptions on fimf.</li>
-              <li>Site specific metadata 'eroticatags' for literotica.com.</li>
+              <li>Fix for AO3 authors all coming out as Anonymous.</li>
+              <li>Known issue: Password protected FimFiction.net stories aren't working.  FimF changed API access.</li>
+              <li>Known issue: Specific metadata 'eroticatags' for literotica.com doesn't work on all stories.</li>
 	    </ul>
 	  </p>
 	  <p>
@@ -74,7 +73,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-5-05.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://2-0-01.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

makezip.py

@@ -1,9 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-# epubmerge.py 1.0
-
-# Copyright 2011, Jim Miller
+# Copyright 2014, Jim Miller
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

plugin-defaults.ini

@@ -602,6 +602,18 @@ extracategories:The Sentinel
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[bdsm-geschichten.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## This site offers no index page so we can either guess the chapter URLs
+## by dec/incrementing numbers ('guess') or walk all the chapters in the metadata
+## parsing state ('parse'). Since guessing can lead to errors for non-standard
+## story URLs, the default is to parse
+#find_chapters:guess 
+
 [bloodshedverse.com]
 ## website encoding(s) In theory, each website reports the character
 ## encoding they use for each page.  In practice, some sites report it
@@ -610,7 +622,7 @@ extracategories:The Sentinel
 ## explicitly set the encoding and order if you need to.  The special
 ## value 'auto' will call chardet and use the encoding it reports if
 ## it has +90% confidence.  'auto' is not reliable.
-website_encodings:ISO-8859-1,auto
+website_encodings:Windows-1252,ISO-8859-1,auto
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -993,7 +1005,7 @@ extraships:Kirk/Spock
 [literotica.com]
 extra_valid_entries:eroticatags
 eroticatags_label:Erotica Tags
-#extra_titlepage_entries: eroticatags
+extra_titlepage_entries: eroticatags
 
 [lumos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
@@ -1224,6 +1236,12 @@ extraships:Harry Potter/Draco Malfoy
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds
 
+[themaplebookshelf.com]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [themasque.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1262,6 +1280,10 @@ extracategories:Harry Potter
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1
 
+[tolkienfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [trekiverse.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1377,8 +1399,11 @@ type_label:Type of Couple
 [www.fanfiction.net]
 user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
-## Change this to false to use them anyway.
-never_make_cover: true
+## Set this to true to never use them.
+#never_make_cover: false
+
+## fanfiction.net shows the user's 
+cover_exclusion_regexp:/imageu/
 
 ## fanfiction.net is blocking people more aggressively.  If you
 ## download fewer stories less often you can likely get by with
@@ -1808,6 +1833,9 @@ extracharacters:Wolverine,Rogue
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
 
+extra_valid_entries:reviews
+reviews_label:Reviews
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For