Update CLI download zip.

Bump versions.
Changes for literotica handling of outer wrapping <p> tag.
2026-09-16 11:47:12 +08:00 · 2014-11-18 16:27:52 -06:00 · 2014-11-18 16:25:53 -06:00 · 2014-11-18 16:22:30 -06:00 · 2014-11-17 18:08:00 -06:00 · 2014-11-17 09:59:55 -06:00
147 changed files with 14515 additions and 3698 deletions
@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-84
+version: 2-0-10
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -42,8 +42,8 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
    description         = _('UI plugin to download FanFiction stories from various sites.')
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (1, 8, 00)
-    minimum_calibre_version = (0, 8, 57)
+    version             = (2, 0, 10)
+    minimum_calibre_version = (1, 48, 0)

    #: This field defines the GUI plugin class that contains all the code
    #: that actually does something. Its format is module_path:class_name
@@ -8,12 +8,19 @@ __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'

 import os
-from PyQt4 import QtGui
-from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
-                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
-                      QTextEdit,
-                      QListWidget, QAbstractItemView)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+    
 from calibre.constants import iswindows
 from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
 from calibre.gui2.actions import menu_action_unique_name
@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'

 import logging
@@ -13,10 +13,31 @@ logger = logging.getLogger(__name__)
 import traceback, copy, threading
 from collections import OrderedDict

-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
-                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
-                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
-                      QDialogButtonBox, QGroupBox )
+try:
+    from PyQt5.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                          QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
+                          QDialogButtonBox, QGroupBox )
+except ImportError as e:
+    from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                          QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
+                          QDialogButtonBox, QGroupBox )
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()

 from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog
@@ -55,12 +76,12 @@ from calibre_plugins.fanfictiondownloader_plugin.dialogs \
            EditTextDialog, RejectUrlEntry)
    
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
-    import (getConfigSections, getNormalStoryURL)
+    import getConfigSections

 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
    import ( KeyboardConfigDialog, PrefsViewerDialog )

-from calibre.gui2.complete import MultiCompleteLineEdit
+from calibre.gui2.complete2 import EditWithComplete  #MultiCompleteLineEdit

 class RejectURLList:
    def __init__(self,prefs):
@@ -209,6 +230,7 @@ class ConfigWidget(QWidget):
        prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
        prefs['mark'] = self.basic_tab.mark.isChecked()
        prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
+        prefs['autoconvert'] = self.basic_tab.autoconvert.isChecked()
        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
        prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
@@ -219,6 +241,7 @@ class ConfigWidget(QWidget):
        prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
        prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
        prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
+        prefs['reject_always'] = self.basic_tab.reject_always.isChecked()

        if self.readinglist_tab:
            # lists
@@ -242,12 +265,13 @@ class ConfigWidget(QWidget):
        prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
        gc_site_settings = {}
        for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
            if val != 'none':
                gc_site_settings[site] = val
                #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
        prefs['gc_site_settings'] = gc_site_settings
        prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
+        prefs['gc_polish_cover'] = self.generatecover_tab.gc_polish_cover.isChecked()

        # Count Pages tab
        countpagesstats = []
@@ -273,12 +297,12 @@ class ConfigWidget(QWidget):

        # Custom Columns tab
        # error column
-        prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())
+        prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))

        # cust cols tab
        colsmap = {}
        for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
            if val != 'none':
                colsmap[col] = val
                #print("colsmap[%s]:%s"%(col,colsmap[col]))
@@ -422,6 +446,11 @@ class BasicTab(QWidget):
        self.showmarked.setChecked(prefs['showmarked'])
        self.l.addWidget(self.showmarked)

+        self.autoconvert = QCheckBox(_("Automatically Convert new/update books?"),self)
+        self.autoconvert.setToolTip(_("Automatically call calibre's Convert for new/update books.\nConverts to the current output format as chosen in calibre's\nPreferences->Behavior settings."))
+        self.autoconvert.setChecked(prefs['autoconvert'])
+        self.l.addWidget(self.autoconvert)
+
        gui_gb = groupbox = QGroupBox(_("GUI Options"))
        self.l = QVBoxLayout()
        groupbox.setLayout(self.l)
@@ -475,6 +504,11 @@ class BasicTab(QWidget):
        self.reject_reasons.clicked.connect(self.show_reject_reasons)
        self.l.addWidget(self.reject_reasons)

+        self.reject_always = QCheckBox(_('Reject Without Confirmation?'),self)
+        self.reject_always.setToolTip(_("Always reject URLs on the Reject List without stopping and asking."))
+        self.reject_always.setChecked(prefs['reject_always'])
+        self.l.addWidget(self.reject_always)
+
        topl.addWidget(defs_gb)

        horz = QHBoxLayout()
@@ -642,7 +676,7 @@ class ReadingListTab(QWidget):
        label = QLabel(_('"Send to Device" Reading Lists'))
        label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
        horz.addWidget(label)        
-        self.send_lists_box = MultiCompleteLineEdit(self)
+        self.send_lists_box = EditWithComplete(self)
        self.send_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
        self.send_lists_box.update_items_cache(reading_lists)
        self.send_lists_box.setText(prefs['send_lists'])
@@ -658,7 +692,7 @@ class ReadingListTab(QWidget):
        label = QLabel(_('"To Read" Reading Lists'))
        label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
        horz.addWidget(label)        
-        self.read_lists_box = MultiCompleteLineEdit(self)
+        self.read_lists_box = EditWithComplete(self)
        self.read_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
        self.read_lists_box.update_items_cache(reading_lists)
        self.read_lists_box.setText(prefs['read_lists'])
@@ -720,12 +754,17 @@ class GenerateCoverTab(QWidget):
            horz.addWidget(label)
            dropdown = QComboBox(self)
            dropdown.setToolTip(s)
-            dropdown.addItem('',QVariant('none'))
+            dropdown.addItem('','none')
            for setting in gc_settings:
-                dropdown.addItem(setting,QVariant(setting))
-            self.gc_dropdowns[site] = dropdown
+                dropdown.addItem(setting,setting)
+            if site == _("Default"):
+                self.gc_dropdowns["Default"] = dropdown
+                if 'Default' in prefs['gc_site_settings']:
+                    dropdown.setCurrentIndex(dropdown.findData(prefs['gc_site_settings']['Default']))
+            else:
+                self.gc_dropdowns[site] = dropdown
            if site in prefs['gc_site_settings']:
-                dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings'][site])))
+                dropdown.setCurrentIndex(dropdown.findData(prefs['gc_site_settings'][site]))

            horz.addWidget(dropdown)
            self.sl.addLayout(horz)
@@ -742,6 +781,11 @@ class GenerateCoverTab(QWidget):
        self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
        self.l.addWidget(self.allow_gc_from_ini)
            
+        self.gc_polish_cover = QCheckBox(_("Use calibre's Polish feature to inject/update the cover"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated cover into the ebook, EPUB only."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        self.l.addWidget(self.gc_polish_cover)
+            
 class CountPagesTab(QWidget):

    def __init__(self, parent_dialog, plugin_action):
@@ -949,12 +993,12 @@ class CustomColumnsTab(QWidget):
                label.setToolTip(_("Update this %s column(%s) with...")%(key,column['datatype']))
                horz.addWidget(label)
                dropdown = QComboBox(self)
-                dropdown.addItem('',QVariant('none'))
+                dropdown.addItem('','none')
                for md in permitted_values[column['datatype']]:
-                    dropdown.addItem(titleLabels[md],QVariant(md))
+                    dropdown.addItem(titleLabels[md],md)
                self.custcol_dropdowns[key] = dropdown
                if key in prefs['custom_cols']:
-                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
+                    dropdown.setCurrentIndex(dropdown.findData(prefs['custom_cols'][key]))
                if column['datatype'] == 'enumeration':
                    dropdown.setToolTip(_("Metadata values valid for this type of column.")+"\n"+_("Values that aren't valid for this enumeration column will be ignored."))
                else:
@@ -990,11 +1034,11 @@ class CustomColumnsTab(QWidget):
        horz.addWidget(label)
        self.errorcol = QComboBox(self)
        self.errorcol.setToolTip(tooltip)
-        self.errorcol.addItem('',QVariant('none'))
+        self.errorcol.addItem('','none')
        for key, column in custom_columns.iteritems():
            if column['datatype'] in ('text','comments'):
-                self.errorcol.addItem(column['name'],QVariant(key))
-        self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
+                self.errorcol.addItem(column['name'],key)
+        self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
        horz.addWidget(self.errorcol)
        self.l.addLayout(horz)

@@ -19,12 +19,36 @@ logger = logging.getLogger(__name__)
 import urllib
 import email

-from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
-                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
-                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal,
-                      QGroupBox, QFrame)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame)
+
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()

 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
@@ -136,7 +160,8 @@ class RejectUrlEntry:
    def fullnote(self):
        retval = ""
        if self.title and self.auth:
-            retval = retval + _("%s by %s")%(self.title,self.auth)
+            # don't translate--ends up being saved and confuses regex above.
+            retval = retval + "%s by %s"%(self.title,self.auth)
            if self.note:
                retval = retval + " - "
                
@@ -145,19 +170,6 @@ class RejectUrlEntry:
            
        return retval

-# This is a more than slightly kludgey way to get
-# EditWithComplete to *not* alpha-order the reasons, but leave
-# them in the order entered.  If
-# calibre.gui2.complete2.CompleteModel.set_items ever changes,
-# this function will need to also.
-def complete_model_set_items_kludge(self, items):
-    items = [unicode(x.strip()) for x in items]
-    items = [x for x in items if x]
-    items = tuple(items)
-    self.all_items = self.current_items = items
-    self.current_prefix = ''
-    self.reset()
-    
 class NotGoingToDownload(Exception):
    def __init__(self,error,icon='dialog_error.png'):
        self.error=error
@@ -195,9 +207,9 @@ class DroppableQTextEdit(QTextEdit):
                            urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                else:
                    urllist.extend(get_urls_from_text("%s"%msg))
-        
        if urllist:
            self.append("\n".join(urllist))
+            return None
        return QTextEdit.dropEvent(self,event)
        
    def canInsertFromMimeData(self, source):
@@ -508,7 +520,7 @@ class UserPassDialog(QDialog):
        self.l = QGridLayout()
        self.setLayout(self.l)

-        if exception.passwdonly:
+        if exception and exception.passwdonly:
            self.setWindowTitle(_('Password'))
            self.l.addWidget(QLabel(_("Author requires a password for this story(%s).")%exception.url),0,0,1,2)
            # user isn't used, but it's easier to still have it for
@@ -558,7 +570,7 @@ class LoopProgressDialog(QProgressDialog):
                 status_prefix=_("Fetched metadata for")):
        QProgressDialog.__init__(self,
                                 init_label,
-                                 QString(), 0, len(book_list), gui)
+                                 _('Cancel'), 0, len(book_list), gui)
        self.setWindowTitle(win_title)
        self.setMinimumWidth(500)
        self.book_list = book_list
@@ -828,11 +840,11 @@ class StoryListTableWidget(QTableWidget):
            icon = get_icon(book['icon'])

        status_cell = IconWidgetItem(None,icon,val)
-        status_cell.setData(Qt.UserRole, QVariant(val))
+        status_cell.setData(Qt.UserRole, val)
        self.setItem(row, 0, status_cell)
        
        title_cell = ReadOnlyTableWidgetItem(book['title'])
-        title_cell.setData(Qt.UserRole, QVariant(row))
+        title_cell.setData(Qt.UserRole, row)
        self.setItem(row, 1, title_cell)
        
        self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
@@ -847,7 +859,7 @@ class StoryListTableWidget(QTableWidget):
        books = []
        #print("=========================\nbooks:%s"%self.books)
        for row in range(self.rowCount()):
-            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
+            rnum = convert_qvariant(self.item(row, 1).data(Qt.UserRole))
            book = self.books[rnum]
            books.append(book)
        return books
@@ -913,15 +925,12 @@ class RejectListTableWidget(QTableWidget):
    def populate_table_row(self, row, rej):

        url_cell = ReadOnlyTableWidgetItem(rej.url)
-        url_cell.setData(Qt.UserRole, QVariant(rej.book_id))
+        url_cell.setData(Qt.UserRole, rej.book_id)
        self.setItem(row, 0, url_cell)
        self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
        self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
        
-        note_cell = EditWithComplete(self)
-        note_cell.lineEdit().mcompleter.model().set_items = \
-            partial(complete_model_set_items_kludge,
-                    note_cell.lineEdit().mcompleter.model())
+        note_cell = EditWithComplete(self,sort_func=lambda x:1)
        
        items = [rej.note]+self.rejectreasons
        note_cell.update_items_cache(items)
@@ -991,10 +1000,7 @@ class RejectListDialog(SizePersistedDialog):
        button_layout.addItem(spacerItem1)

        if show_all_reasons:
-            self.reason_edit = EditWithComplete(self)
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
            
            items = ['']+rejectreasons
            self.reason_edit.update_items_cache(items)
@@ -1036,7 +1042,7 @@ class RejectListDialog(SizePersistedDialog):
        rejectrows = []
        for row in range(self.rejects_table.rowCount()):
            url = unicode(self.rejects_table.item(row, 0).text()).strip()
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id =convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
            title = unicode(self.rejects_table.item(row, 1).text()).strip()
            auth = unicode(self.rejects_table.item(row, 2).text()).strip()
            note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
@@ -1046,7 +1052,7 @@ class RejectListDialog(SizePersistedDialog):
    def get_reject_list_ids(self):
        rejectrows = []
        for row in range(self.rejects_table.rowCount()):
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id = convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
            if book_id:
                rejectrows.append(book_id)
        return rejectrows
@@ -1088,11 +1094,7 @@ class EditTextDialog(QDialog):
            self.textedit.setToolTip(tooltip)

        if rejectreasons or reasonslabel:
-            self.reason_edit = EditWithComplete(self)
-
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
            
            items = ['']+rejectreasons
            self.reason_edit.update_items_cache(items)
@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'

 import logging
@@ -13,16 +13,18 @@ logger = logging.getLogger(__name__)
 import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
-from datetime import datetime
+from datetime import datetime, time
 from string import Template
 import urllib
 import email
 import traceback

-from PyQt4.Qt import (QApplication, QMenu, QToolButton, QTimer)
-
-from PyQt4.Qt import QPixmap, Qt
-from PyQt4.QtCore import QBuffer
+try:
+    from PyQt5.Qt import (QApplication, QMenu, QTimer)
+    from PyQt5.QtCore import QBuffer
+except ImportError as e:
+    from PyQt4.Qt import (QApplication, QMenu, QTimer)
+    from PyQt4.QtCore import QBuffer

 from calibre.constants import numeric_version as calibre_version

@@ -32,6 +34,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.utils.config import prefs as calibre_prefs
 from calibre.utils.date import local_tz
 from calibre.library.comments import sanitize_comments_html
 from calibre.constants import config_dir as calibre_config_dir
@@ -125,9 +128,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

        # Assign our menu to this action
        self.menu = QMenu(self.gui)
-        self.old_actions_unique_map = {}
-        # menu_actions is just to keep a live reference to the menu
-        # items to prevent GC removing it.
+        # menu_actions is to keep a live reference to the menu items
+        # to prevent GC removing it and so rebuild_menus has a list
        self.menu_actions = []
        self.qaction.setMenu(self.menu)
        self.menus_lock = threading.RLock()
@@ -206,7 +208,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                #print("text/plain:%s"%event.mimeData().data(mimetype))
                urllist.extend(get_urls_from_text(event.mimeData().data(mimetype)))

-        #print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
+        # print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
        if urllist or dropped_ids:
            QTimer.singleShot(1, partial(self.do_drop,
                                         dropped_ids=dropped_ids,
@@ -235,8 +237,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            #self.qaction.setText("FFDL")
            do_user_config = self.interface_action_base_plugin.do_user_config
            self.menu.clear()
-            self.actions_unique_map = {}
+            
+            for action in self.menu_actions:
+                self.gui.keyboard.unregister_shortcut(action.calibre_shortcut_unique_name)
+                # starting in calibre 2.10.0, actions are registers at
+                # the top gui level for OSX' benefit.
+                if calibre_version >= (2,10,0):
+                    self.gui.removeAction(action)
            self.menu_actions = []
+            
            self.add_action = self.create_menu_item_ex(self.menu, _('&Add New from URL(s)'), image='plus.png',
                                                       unique_name='Add New FanFiction Book(s) from URL(s)',
                                                       shortcut_name=_('Add New FanFiction Book(s) from URL(s)'),
@@ -322,11 +331,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                             shortcut_name=_('About FanFictionDownLoader'), 
                                                             triggered=self.about)
            
-            # Before we finalize, make sure we delete any actions for menus that are no longer displayed
-            for menu_id, unique_name in self.old_actions_unique_map.iteritems():
-                if menu_id not in self.actions_unique_map:
-                    self.gui.keyboard.unregister_shortcut(unique_name)
-            self.old_actions_unique_map = self.actions_unique_map
            self.gui.keyboard.finalize()

    def about(self):
@@ -349,7 +353,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        #print("create_menu_item_ex before %s"%menu_text)
        ac = create_menu_action_unique(self, parent_menu, menu_text, image, tooltip,
                                       shortcut, triggered, is_checked, shortcut_name, unique_name)
-        self.actions_unique_map[ac.calibre_shortcut_unique_name] = ac.calibre_shortcut_unique_name
        self.menu_actions.append(ac)
        #print("create_menu_item_ex after %s"%menu_text)
        return ac
@@ -384,6 +387,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

    def get_urls_from_page_menu(self):

+        urltxt = ""
        if prefs['urlsfromclip']:
            try:
                urltxt = self.get_urls_clip(storyurls=False)[0]
@@ -416,7 +420,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        
    def list_story_urls(self):
        '''Get list of URLs from existing books.'''
-        if self.gui.current_view().selectionModel().selectedRows() == 0 :
+        if not self.gui.current_view().selectionModel().selectedRows() :
            self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
                                             3000)
            return
@@ -660,13 +664,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        #print("update_dialog()")
        
        db = self.gui.current_db
-        book_list = map( self.make_book_id_only, id_list )
+        books = map( self.make_book_id_only, id_list )

-        for j, book in enumerate(book_list):
+        for j, book in enumerate(books):
            book['listorder'] = j
-            
+
        LoopProgressDialog(self.gui,
-                           book_list,
+                           books,
                           partial(self.populate_book_from_calibre_id, db=self.gui.current_db),
                           self.update_dialog_finish,
                           init_label=_("Collecting stories for update..."),
@@ -711,6 +715,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        # No need to do anything with perfs here, but we could.
        prefs

+    def make_id_searchstr(self,url):
+        # older idents can be uri vs url and have | instead of : after
+        # http, plus many sites are now switching to https.
+        return 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url))
+
    def prep_downloads(self, options, books, merge=False, extrapayload=None):
        '''Fetch metadata for stories from servers, launch BG job when done.'''

@@ -718,8 +727,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            url_list = split_text_to_urls(books)
            books = self.convert_urls_to_books(url_list)

+        ## for tweak_fg_sleep
+        options['ffnetcount']=len(filter(lambda x : x['site']=='www.fanfiction.net', books))
+
        options['version'] = self.version
        logger.debug(self.version)
+        options['personal.ini'] = get_ffdl_personalini()
        
        #print("prep_downloads:%s"%books)

@@ -760,7 +773,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if not merge: # skip reject list when merging.
            if rejecturllist.check(url):
                rejnote = rejecturllist.get_full_note(url)
-                if question_dialog(self.gui, _('Reject URL?'),'''
+                if prefs['reject_always'] or question_dialog(self.gui, _('Reject URL?'),'''
                          <h3>%s</h3>
                          <p>%s</p>
                          <p>"<b>%s</b>"</p>
@@ -813,14 +826,34 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        
        skip_date_update = False
        
-        options['personal.ini'] = get_ffdl_personalini()
        adapter = get_ffdl_adapter(url,fileform)
+        ## save and share cookiejar and pagecache between all
+        ## downloads.
+        if 'pagecache' not in options:
+            options['pagecache'] = adapter.get_empty_pagecache()
+        adapter.set_pagecache(options['pagecache'])
+        if 'cookiejar' not in options:
+            options['cookiejar'] = adapter.get_empty_cookiejar()
+        adapter.set_cookiejar(options['cookiejar'])
+            
+        # reduce foreground sleep time for ffnet when few books.
+        if 'ffnetcount' in options and \
+                adapter.getConfig('tweak_fg_sleep') and \
+                adapter.getSiteDomain() == 'www.fanfiction.net':
+            minslp = float(adapter.getConfig('min_fg_sleep'))
+            maxslp = float(adapter.getConfig('max_fg_sleep'))
+            dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+            m = (maxslp-minslp) / (dwnlds-1)
+            b = minslp - m
+            slp = min(maxslp,m*float(options['ffnetcount'])+b)
+            #print("m:%s b:%s = %s"%(m,b,slp))
+            adapter.set_sleep(slp)

        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other
        for x in range(0,2):
            try:
-                adapter.getStoryMetadataOnly()
+                adapter.getStoryMetadataOnly(get_cover=False)
            except exceptions.FailedToLogin, f:
                logger.warn("Login Failed, Need Username/Password.")
                userpass = UserPassDialog(self.gui,url,f)
@@ -836,12 +869,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                    adapter.is_adult=True

        # let other exceptions percolate up.
-        story = adapter.getStoryMetadataOnly()
+        story = adapter.getStoryMetadataOnly(get_cover=False)

        series = story.getMetadata('series')
        if not merge and series and prefs['checkforseriesurlid']:
            # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = 'identifiers:"~ur(i|l):=%s"'%story.getMetadata('seriesUrl').replace(":","|")
+            searchstr = self.make_id_searchstr(story.getMetadata('seriesUrl'))
            identicalbooks = db.search_getting_ids(searchstr, None)
            # print("searchstr:%s"%searchstr)
            # print("identicalbooks:%s"%identicalbooks)
@@ -873,7 +906,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

        # all_metadata duplicates some data, but also includes extra_entries, etc.
        book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-        
+
        book['title'] = story.getMetadata("title", removeallentities=True)
        book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
        book['publisher'] = story.getMetadata("site")
@@ -918,18 +951,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                logger.debug("from URL(%s)"%url)
    
                # try to find by identifier url or uri first.
-                searchstr = 'identifiers:"~ur(i|l):=%s"'%url.replace(":","|")
+                searchstr = self.make_id_searchstr(url)
                identicalbooks = db.search_getting_ids(searchstr, None)
+                # print("searchstr:%s"%searchstr)
+                # print("identicalbooks:%s"%identicalbooks)
                if len(identicalbooks) < 1:
                    # find dups
                    authlist = story.getList("author", removeallentities=True)
-                    if len(authlist) > 100 and calibre_version < (0, 8, 61):
-                        ## should be fixed from 0.8.61 on.  In the
-                        ## meantime, if it matches the title *and* first
-                        ## 100 authors, I'm prepared to assume it's a
-                        ## match.
-                        logger.debug("reduce author list to 100 only when calibre < 0.8.61")
-                        authlist = authlist[:100]
                    mi = MetaInformation(story.getMetadata("title", removeallentities=True),
                                         authlist)
                    identicalbooks = db.find_identical_books(mi)
@@ -943,7 +971,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                    raise NotGoingToDownload(_("Skipping duplicate story."),"list_remove.png")
    
                if len(identicalbooks) > 1:
-                    raise NotGoingToDownload("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite.","minusminus.png")
+                    raise NotGoingToDownload(_("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png")
    
                ## changed: add new book when CALIBREONLY if none found.
                if collision == CALIBREONLY and not identicalbooks:
@@ -959,7 +987,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                if book_id and mi: # book_id and mi only set if matched by title/author.
                    liburl = self.get_story_url(db,book_id)
                    if book['url'] != liburl and prefs['checkforurlchange'] and \
-                        not (book['url'].replace('https','http') == liburl and 'fanfiction.net' in liburl):
+                        not (book['url'].replace('https','http') == liburl): # several sites have been changing to
+                                                                             # https now.  Don't flag when that's the only change.
                        # special case for ffnet urls change to https.
                        if not question_dialog(self.gui, _('Change Story URL?'),'''
                                                  <h3>%s</h3>
@@ -1014,7 +1043,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        (epuburl,chaptercount) = \
                            get_dcsource_chaptercount(StringIO(db.format(book_id,'EPUB',
                                                                         index_is_id=True)))
-                        urlchaptercount = int(story.getMetadata('numChapters'))
+                        urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
                        if chaptercount == urlchaptercount:
                            if collision == UPDATE:
                                raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
@@ -1029,10 +1058,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                if collision == OVERWRITE and \
                        db.has_format(book_id,formmapping[fileform],index_is_id=True):
                    # check make sure incoming is newer.
-                    lastupdated=story.getMetadataRaw('dateUpdated').date()
-                    fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8]).date()
-                    if fileupdated > lastupdated:
+                    lastupdated=story.getMetadataRaw('dateUpdated')
+                    fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8])
+
+                    # updated doesn't have time (or is midnight), use dates only.
+                    # updated does have time, use full timestamps.
+                    if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
+                            (lastupdated.time() != time.min and fileupdated > lastupdated):
                        raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png')
+                    
+                    
        
                # For update, provide a tmp file copy of the existing epub so
                # it can't change underneath us.  Now also overwrite for logpage preserve.
@@ -1062,7 +1097,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                          dir=options['tdir'])
            logger.debug("title:"+book['title'])
            logger.debug("outfile:"+tmp.name)
-            book['outfile'] = tmp.name            
+            book['outfile'] = tmp.name
+            
+            # cookiejar = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
+            #                                     suffix='.cookiejar',
+            #                                     dir=options['tdir'])
+            # adapter.save_cookiejar(cookiejar.name)
+            # book['cookiejar'] = cookiejar.name
+            # pagecache = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
+            #                                     suffix='.pagecache',
+            #                                     dir=options['tdir'])
+            # adapter.save_pagecache(pagecache.name)
+            # book['pagecache'] = pagecache.name
                
        return
        
@@ -1119,7 +1165,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                      _('FFDL log'), _('FFDL download ended'), msg,
                                      show_copy_button=False)
            return
-            
+
+        cookiejarfile = PersistentTemporaryFile(suffix='.cookiejar',
+                                                dir=options['tdir'])
+        options['cookiejar'].save(cookiejarfile.name,
+                                  ignore_discard=True,
+                                  ignore_expires=True)
+        options['cookiejarfile']=cookiejarfile.name
+        del options['cookiejar'] ## can't be pickled.
+        
        func = 'arbitrary_n'
        cpus = self.gui.job_manager.server.pool_size
        args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', 'do_download_worker',
@@ -1130,6 +1184,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                func, args=args,
                description=desc)
        
+        self.gui.jobs_pointer.start()
        self.gui.status_bar.show_message(_('Starting %d FanFictionDownLoads')%len(book_list),3000)

    def update_books_loop(self,book,db=None,
@@ -1224,6 +1279,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            cp_plugin = self.gui.iactions['Count Pages']
            cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])

+        if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
+            self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
+            self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
+
    def download_list_completed(self, job, options={},merge=False):
        if job.failed:
            self.gui.job_exception(job, dialog_title='Failed to Download Stories')
@@ -1327,7 +1386,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        bad_list = sorted(bad_list,key=lambda x : x['listorder'])
        
        self.gui.status_bar.show_message(_('Merging %s books.')%total_good)
-
        
        existingbook = None
        if 'mergebook' in options:
@@ -1385,7 +1443,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if prefs['mark'] or (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
            self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.')%len(book_list))
-            label = custom_columns[prefs['errorcol']]['label']
+            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
+                label = custom_columns[prefs['errorcol']]['label']
+            else:
+                label = None
            LoopProgressDialog(self.gui,
                               book_list,
                               partial(self.update_error_column_loop, db=self.gui.current_db, label=label),
@@ -1394,12 +1455,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                               win_title=_("Update calibre for BAD FanFiction stories"),
                               status_prefix=_("Updated"))

-    def update_error_column_loop(self,book,db=None,label='errorcol'):
-        if book['calibre_id']:
-            custom_columns = self.gui.library_view.model().custom_columns
-            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
-                logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
-                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+    def update_error_column_loop(self,book,db=None,label=None):
+        if book['calibre_id'] and label:
+            logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)

    def add_book_or_update_format(self,book,options,prefs,mi=None):
        db = self.gui.current_db
@@ -1428,8 +1487,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
            for fmt in fmts:
                if fmt != formmapping[options['fileform']]:
-                    logger.debug("remove f:"+fmt)
+                    logger.debug("deleteotherforms remove f:"+fmt)
                    db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+        elif prefs['autoconvert']:
+            ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+            ## the format is already there.
+            fmt = calibre_prefs['output_format']
+            # delete if there, but not if the format we just made.
+            if fmt != formmapping[options['fileform']] and \
+                    db.has_format(book_id,fmt,index_is_id=True):
+                logger.debug("autoconvert remove f:"+fmt)
+                db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                

        return book_id

@@ -1537,8 +1606,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )

                    if meta not in book['all_metadata']:
-                        logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
-                        continue
+                        # if double quoted, use as a literal value.
+                        if meta[0] == '"' and meta[-1] == '"':
+                            val = meta[1:-1]
+                            logger.debug("No metadata value for %s, setting custom column(%s) literally to %s."%(meta,custcol,val))
+                        else:
+                            logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
+                            continue
+                    else:
+                        val = book['all_metadata'][meta]    
                    
                    if custcol not in custom_columns:
                        continue
@@ -1546,22 +1622,31 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        coldef = custom_columns[custcol]
                        label = coldef['label']

-                    if flag == 'r' or book['added']: # flag 'n' isn't actually needed--*always* set if configured and new book.
+                    if flag == 'r' or (flag == 'n' and book['added']):
                        if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
                            if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
                                # re-split list, strip commas, convert to floats, sum up.
-                                val = sum([ float(x.replace(",","")) for x in book['all_metadata'][meta].split(", ") ])
+                                val = sum([ float(x.replace(",","")) for x in val.split(", ") ])
                            else:
-                                val = unicode(book['all_metadata'][meta]).replace(",","")
+                                val = unicode(val).replace(",","")
                        else:
-                            val = book['all_metadata'][meta]
+                            val = val
                        if val != '':
+                            if coldef['datatype'] == 'bool':
+                                if val.lower() in ('t','true','1','yes','y'):
+                                    val = True
+                                elif val.lower() in ('f','false','0','no','n'):
+                                    val = False
+                                else:
+                                    val = None # for tri-state 'booleans'. Yes/No/Null
+                            #print("setting 'r' or 'added':%s"%val)
                            db.set_custom(book_id, val, label=label, commit=False)

                    if flag == 'a':
                        vallist = []
                        try:
                            existing=db.get_custom(book_id,label=label,index_is_id=True)
+                            #print("existing:%s"%existing)
                            if isinstance(existing,list):
                                vallist = existing
                            elif existing:
@@ -1569,8 +1654,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        except:
                            pass

-                        if book['all_metadata'][meta]:
-                            vallist = [book['all_metadata'][meta]]
+                        #print("vallist:%s"%vallist)
+                        if val:
+                            vallist.append(val)
                            
                        db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)

@@ -1579,28 +1665,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if 'authorUrl' in book['all_metadata']:
            authurls = book['all_metadata']['authorUrl'].split(", ")
            authorlist = [ a.replace('&',';') for a in book['author'] ]
-            if hasattr(db, 'new_api'): # new_api starts in calibre 1.0.0
-                authorids = db.new_api.get_item_ids('authors',authorlist)
-                authordata = db.new_api.author_data(authorids.values())
-                # print("\n\nauthorids:%s"%authorids)
-                # print("authordata:%s"%authordata)
+            authorids = db.new_api.get_item_ids('authors',authorlist)
+            authordata = db.new_api.author_data(authorids.values())
+            # print("\n\nauthorids:%s"%authorids)
+            # print("authordata:%s"%authordata)

-                author_id_to_link_map = dict()
-                for i, author in enumerate(authorlist):
-                    author_id_to_link_map[authorids[author]] = authurls[i]
+            author_id_to_link_map = dict()
+            for i, author in enumerate(authorlist):
+                author_id_to_link_map[authorids[author]] = authurls[i]

-                # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
-                db.new_api.set_link_for_authors(author_id_to_link_map)
-            else:
-                # keep for pre-calibre 1.0.0
-                for i, auth in enumerate(authorlist):
-                    #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                    autid=db.get_author_id(auth)
-                    db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
+            # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
+            db.new_api.set_link_for_authors(author_id_to_link_map)
+            
        db.commit()

        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
+
+            #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
            
            # force a refresh if generating cover so complex composite
            # custom columns are current and correct
@@ -1632,15 +1713,39 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                
            if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
                setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
+                logger.debug("Generate Cover Setting from site(%s)"%setting_name)

            if not setting_name and 'Default' in prefs['gc_site_settings']:
                setting_name =  prefs['gc_site_settings']['Default']
+                logger.debug("Generate Cover Setting from Default(%s)"%setting_name)
                
            if setting_name:
                logger.debug("Running Generate Cover with settings %s."%setting_name)
                realmi = db.get_metadata(book_id, index_is_id=True)
                gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)

+                if prefs['gc_polish_cover'] and \
+                        options['fileform'] == "epub":
+                    # set cover inside epub from calibre's polish feature
+                    from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+                    from calibre.utils.logging import Log
+                    from collections import namedtuple
+
+                    # Couldn't find a better way to get the cover path.
+                    cover_path = os.path.join(db.library_path, db.path(book_id, index_is_id=True), 'cover.jpg')
+                    data = {'cover':cover_path}
+                    #print("cover_path:%s"%cover_path)
+                    opts = ALL_OPTS.copy()
+                    opts.update(data)
+                    O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+                    opts = O(**opts)
+                
+                    log = Log(level=Log.DEBUG)
+                    outfile = db.format_abspath(book_id, formmapping[options['fileform']], index_is_id=True)
+                    #print("polish cover outfile:%s"%outfile)
+                    polish({outfile:outfile}, opts, log, logger.info)
+                
+
    def get_clean_reading_lists(self,lists):
        if lists == None or lists.strip() == "" :
            return []
@@ -1729,7 +1834,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        book['end'] = None
        book['comment'] = '' # note this is a comment on the d/l or update.
        book['url'] = ''
+        book['site'] = ''
        book['added'] = False
+        book['pubdate'] = None
        return book
    
    def convert_urls_to_books(self, urls):
@@ -1805,13 +1912,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            book['status'] = _('Not Found')
        else:
            # get normalized url or None.
-            book['url'] = self.is_good_downloader_url(url)
-            if book['url'] == None:
+            urlsitetuple = adapters.getNormalStoryURLSite(url)
+            if urlsitetuple == None:
                book['url'] = url
                book['comment'] = _("URL is not a valid story URL.")
                book['good'] = False
                book['icon']='dialog_error.png'
                book['status'] = _('Bad URL')
+            else:
+                (book['url'],book['site'])=urlsitetuple
    
    def get_story_url(self, db, book_id=None, path=None):
        if book_id == None:
@@ -1893,9 +2002,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        
            # fill from first of each if not already present:
            for k in ('pubdate', 'timestamp', 'updatedate'):
-                if k not in b: # not in this book?  Skip it.
+                if k not in b or not b[k]: # not in this book?  Skip it.
                    continue
-                if k not in book: # first is good enough for publisher.
+                if k not in book or not book[k]: # first is good enough for publisher.
                    book[k]=b[k]

                # Do these even on first to get the all_metadata settings.
@@ -1916,11 +2025,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            for (k,v) in b['all_metadata'].iteritems():
                #print("merge_meta_books v:%s k:%s"%(v,k))
                if k in ('numChapters','numWords'):
-                    if k not in book['all_metadata']:
-                        book['all_metadata'][k] = b['all_metadata'][k]
-                    else:
-                        # lot of work for a simple add.
-                        book['all_metadata'][k] = unicode(int(book['all_metadata'][k].replace(',',''))+int(b['all_metadata'][k].replace(',','')))
+                    if k in b['all_metadata'] and b['all_metadata'][k]:
+                        if k not in book['all_metadata']:
+                            book['all_metadata'][k] = b['all_metadata'][k]
+                        else:
+                            # lot of work for a simple add.
+                            book['all_metadata'][k] = unicode(int(book['all_metadata'][k].replace(',',''))+int(b['all_metadata'][k].replace(',','')))
                elif k in ('dateUpdated','datePublished','dateCreated',
                           'series','status','title'):
                    pass # handled above, below or skip these for now, not going to do anything with them.
@@ -1935,14 +2045,20 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        # cust cols can convert back to numbers and
                        # add.
                        book['anthology_meta_list'][k]=True
-    
+
+        print("book['url']:%s"%book['url'])
+        configuration = get_ffdl_config(book['url'],fileform)
        if existingbook:
            book['title'] = deftitle = existingbook['title']
            book['comments'] = existingbook['comments']
        else:
            book['title'] = deftitle = book_list[0]['title']
-            book['comments'] = _("Anthology containing:")+"\n" + \
-                "\n".join([ _("%s by %s")%(b['title'],', '.join(b['author'])) for b in book_list ])
+            if len(book['author']) > 1:
+                book['comments'] = _("Anthology containing:")+"\n" + \
+                    "\n".join([ _("%s by %s")%(b['title'],', '.join(b['author'])) for b in book_list ])
+            else:
+                book['comments'] = _("Anthology containing:")+"\n" + \
+                    "\n".join([ b['title'] for b in book_list ])
            # book['all_metadata']['description']
        
            # if all same series, use series for name.  But only if all and not previous named
@@ -1954,7 +2070,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        book['title'] = deftitle
                        break
            
-            configuration = get_ffdl_config(book['url'],fileform)
            logger.debug("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
            if configuration.getConfig('anthology_title_pattern'):
                tmplt = Template(configuration.getConfig('anthology_title_pattern'))
@@ -1970,7 +2085,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        for v in ['Completed','In-Progress']:
            if v in book['tags']:
                book['tags'].remove(v)
-        book['tags'].append('Anthology')
+        book['tags'].extend(configuration.getConfigList('anthology_tags'))
        book['all_metadata']['anthology'] = "true"
        
        return book
@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'

@@ -19,6 +19,11 @@ from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
 from calibre.constants import numeric_version as calibre_version

+# for smarten punc
+from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+from calibre.utils.logging import Log
+from collections import namedtuple
+
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
@@ -58,10 +63,6 @@ def do_download_worker(book_list, options,
                              done=None,
                              args=args)
            job._book = book
-            # job._book_id = book_id
-            # job._title = title
-            # job._modified_date = modified_date
-            # job._existing_isbn = existing_isbn
            server.add_job(job)
        else:
            # was already bad before the subprocess ever started.
@@ -69,7 +70,7 @@ def do_download_worker(book_list, options,
    
    # This server is an arbitrary_n job, so there is a notifier available.
    # Set the % complete to a small number to avoid the 'unavailable' indicator
-    notification(0.01, 'Downloading FanFiction Stories')
+    notification(0.01, _('Downloading FanFiction Stories'))

    # dequeue the job results as they arrive, saving the results
    count = 0
@@ -81,12 +82,9 @@ def do_download_worker(book_list, options,
        if not job.is_finished:
            continue
        # A job really finished. Get the information.
-        output_book = job.result
-        #print("output_book:%s"%output_book)
        book_list.remove(job._book)
        book_list.append(job.result)
        book_id = job._book['calibre_id']
-        #title = job._title
        count = count + 1
        notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
        # Add this job's output to the current log
@@ -94,15 +92,10 @@ def do_download_worker(book_list, options,
        logger.info(job.details)

        if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            logger.info("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
-            logger.info("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\n"+_("Successful:")+"\n%s\n"%("\n".join([book['url'] for book in 
+                                                      filter(lambda x: x['good'], book_list) ] ) ) )
+            logger.info("\n"+_("Unsuccessful:")+"\n%s\n"%("\n".join([book['url'] for book in 
+                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
            break

    server.close()
@@ -112,11 +105,10 @@ def do_download_worker(book_list, options,

 def do_download_for_worker(book,options,notification=lambda x,y:x):
    '''
-    Child job, to extract isbn from formats for this specific book,
-    when run as a worker job
+    Child job, to download story when run as a worker job
    '''
    try:
-        book['comment'] = 'Download started...'
+        book['comment'] = _('Download started...')

        configuration = get_ffdl_config(book['url'],
                                        options['fileform'],
@@ -125,8 +117,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
        if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
            configuration.set("overrides","never_make_cover","true")

-        # images only for epub, even if the user mistakenly turned it
-        # on else where.
+        # images only for epub, html, even if the user mistakenly
+        # turned it on else where.
        if options['fileform'] not in ("epub","html"):
            configuration.set("overrides","include_images","false")
        
@@ -136,6 +128,10 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
        adapter.password = book['password']
        adapter.setChaptersRange(book['begin'],book['end'])
        
+        adapter.load_cookiejar(options['cookiejarfile'])
+        logger.debug("cookiejar:%s"%adapter.cookiejar)
+        adapter.set_pagecache(options['pagecache'])
+        
        story = adapter.getStoryMetadataOnly()
        if 'calibre_series' in book:
            adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
@@ -182,7 +178,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):

            # update now handled by pre-populating the old images and
            # chapters in the adapter rather than merging epubs.
-            urlchaptercount = int(story.getMetadata('numChapters'))
+            urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
            (url,
             chaptercount,
             adapter.oldchapters,
@@ -194,13 +190,13 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
            # dup handling from ffdl_plugin needed for anthology updates.
            if options['collision'] == UPDATE:
                if chaptercount == urlchaptercount:
-                    book['comment']="Already contains %d chapters.  Reuse as is."%chaptercount
+                    book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
                    book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                    return book

            # dup handling from ffdl_plugin needed for anthology updates.
            if chaptercount > urlchaptercount:
-                raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
+                raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')

            if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
                    and adapter.getConfig("do_update_hook"):
@@ -211,16 +207,12 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):

            writer.writeStory(outfilename=outfile, forceOverwrite=True)
            
-            book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
+            book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
        
        if options['smarten_punctuation'] and options['fileform'] == "epub" \
                and calibre_version >= (0, 9, 39):
            # do smarten_punctuation from calibre's polish feature
-            from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
-            from calibre.utils.logging import Log
-            from collections import namedtuple
-        
            data = {'smarten_punctuation':True}
            opts = ALL_OPTS.copy()
            opts.update(data)
@@ -25,6 +25,7 @@ default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
 Dup from another site'''
+default_prefs['reject_always'] = False

 default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False
@@ -34,6 +35,7 @@ default_prefs['suppressauthorsort'] = False
 default_prefs['suppresstitlesort'] = False
 default_prefs['mark'] = False
 default_prefs['showmarked'] = False
+default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'
@@ -56,6 +58,7 @@ default_prefs['addtolistsonread'] = False
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
+default_prefs['gc_polish_cover'] = False

 default_prefs['countpagesstats'] = []

@@ -23,6 +23,7 @@ import getpass
 import string
 import ConfigParser
 from subprocess import call
+import pprint

 import logging
 if sys.version_info >= (2, 7):
@@ -226,9 +227,9 @@ def main(argv,
           except:
               options.update = False
               pass
-       
-       ## Check for include_images and absence of PIL, give warning.
-       if adapter.getConfig('include_images'):
+
+       ## Check for include_images without no_image_processing. In absence of PIL, give warning.
+       if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
           try:
               from calibre.utils.magick import Image
               logging.debug("Using calibre.utils.magick")
@@ -270,31 +271,29 @@ def main(argv,
           elif chaptercount == 0:
               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
           else:
-               if not options.metaonly:
+               # update now handled by pre-populating the old
+               # images and chapters in the adapter rather than
+               # merging epubs.
+               (url,
+                chaptercount,
+                adapter.oldchapters,
+                adapter.oldimgs,
+                adapter.oldcover,
+                adapter.calibrebookmark,
+                adapter.logfile) = get_update_data(output_filename)

-                   # update now handled by pre-populating the old
-                   # images and chapters in the adapter rather than
-                   # merging epubs.
-                   (url,
-                    chaptercount,
-                    adapter.oldchapters,
-                    adapter.oldimgs,
-                    adapter.oldcover,
-                    adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(output_filename)
+               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)

-                   print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+               if not (options.update and chaptercount == urlchaptercount) \
+                       and adapter.getConfig("do_update_hook"):
+                   chaptercount = adapter.hookForUpdates(chaptercount)

-                   if not (options.update and chaptercount == urlchaptercount) \
-                           and adapter.getConfig("do_update_hook"):
-                       chaptercount = adapter.hookForUpdates(chaptercount)
-
-                   writeStory(configuration,adapter,"epub")
+               writeStory(configuration,adapter,"epub")
                   
       else:
           # regular download
           if options.metaonly:
-               print adapter.getStoryMetadataOnly()
+               pprint.pprint(adapter.getStoryMetadataOnly().getAllMetadata())
           
           output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
       
@@ -0,0 +1,57 @@
+# coding: utf-8
+
+import re
+import codecs
+
+stack = []
+
+def get_end_tag(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'.*<([^\ >]+).*', r'</\1>', tag)
+    return u''
+
+def get_tag_name(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'</*([^\ >]+).*', r'\1', tag)
+    return u''
+
+def push(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        stack.append(tag)
+
+def pop():
+    if len(stack) > 0:
+        return stack.pop()
+    return u''
+
+def pop_end_tag():
+    return unicode(get_end_tag(pop()))
+
+def spool_end():
+    html = u''
+    for tag in reversed(stack):
+        html += get_end_tag(tag)
+    return html
+
+def spool_start():
+    html = u''
+    for item in stack:
+        html += item
+    return html
+
+def has_elements():
+    return len(stack) > 0
+
+def get_last():
+    # t = pop()
+    # push(t)
+    # return t
+    if len(stack) > 0:
+        return stack[len(stack)-1]
+    return u''
+
+def flush():
+    del stack[:]
+
+def get_stack():
+    return stack
@@ -39,7 +39,7 @@ import adapter_mediaminerorg
 import adapter_potionsandsnitchesnet
 import adapter_tenhawkpresentscom
 import adapter_adastrafanficcom
-import adapter_thewriterscoffeeshopcom
+import adapter_twcslibrarynet
 import adapter_tthfanficorg
 import adapter_twilightednet
 import adapter_twiwritenet
@@ -121,11 +121,29 @@ import adapter_simplyundeniablecom
 import adapter_scarheadnet
 import adapter_fictionpadcom
 import adapter_storiesonlinenet
+import adapter_trekiverseorg
+import adapter_literotica
+import adapter_voracity2eficcom
+import adapter_spikeluvercom
+import adapter_bloodshedversecom
+import adapter_nocturnallightnet
+import adapter_fanfichu
+import adapter_fanfictioncsodaidokhu
+import adapter_fictionmaniatv
+import adapter_bdsmgeschichten
+import adapter_tolkienfanfiction
+import adapter_themaplebookshelf
+import adapter_fannation
+import adapter_sheppardweircom
+import adapter_samandjacknet
+import adapter_csiforensicscom
+import adapter_lotrfanfictioncom
+import adapter_fhsarchivecom

 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
 ## to pick out the adapter.
-    
+
 ## List of registered site adapters.
 __class_list = []
 __domain_map = {}
@@ -144,6 +162,13 @@ for x in imports():
            __domain_map[site]=cls

 def getNormalStoryURL(url):
+    r = getNormalStoryURLSite(url)
+    if r:
+        return r[0]
+    else:
+        return None
+
+def getNormalStoryURLSite(url):
    if not getNormalStoryURL.__dummyconfig:
        getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
    # pulling up an adapter is pretty low over-head.  If
@@ -151,10 +176,11 @@ def getNormalStoryURL(url):
    try:
        adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
        url = adapter.url
+        site = adapter.getSiteDomain()
        del adapter
-        return url
+        return (url,site)
    except:
-        return None;
+        return None

 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None
@@ -185,18 +211,20 @@ def getConfigSectionFor(url):
    (cls,fixedurl) = getClassFor(url)
    if cls:
        return cls.getConfigSection()
-    
+
    # No adapter found.
    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )

 def getClassFor(url):
    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
+    if fixedurl.startswith("//"):
+        fixedurl = "http:%s"%url
    if not fixedurl.startswith("http"):
        fixedurl = "http://%s"%url
    ## remove any trailing '#' locations.
    fixedurl = re.sub(r"#.*$","",fixedurl)
-    
+
    parsedUrl = up.urlparse(fixedurl)
    domain = parsedUrl.netloc.lower()
    if( domain != parsedUrl.netloc ):
@@ -207,17 +235,17 @@ def getClassFor(url):
        domain = domain.replace("www.","")
        #logger.debug("trying site:without www: "+domain)
        cls = getClassFromList(domain)
-        fixedurl = fixedurl.replace("http://www.","http://")
+        fixedurl = re.sub(r"^http(s?)://www\.",r"http\1://",fixedurl)
    if not cls:
        #logger.debug("trying site:www."+domain)
        cls = getClassFromList("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
+        fixedurl = re.sub(r"^http(s?)://",r"http\1://www.",fixedurl)

    if cls:
        fixedurl = cls.stripURLParameters(fixedurl)
-    
+
    return (cls,fixedurl)
-    
+
 def getClassFromList(domain):
    try:
        return __domain_map[domain]
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -53,12 +54,19 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
        return 'www.adastrafanfic.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
    def extractChapterUrlsAndMetadata(self):

        if self.is_adult or self.getConfig("is_adult"):
@@ -77,14 +77,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'archiveofourown.org'

-    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/works/123456 http://"+cls.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+cls.getSiteDomain()+"/works/123456/chapters/78901"

    def getSiteURLPattern(self):
        # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
        # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
+        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
        
    ## Login
    def needToLoginCheck(self, data):
@@ -123,6 +123,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        else:
            return True

+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
    ## Getting the chapter list and the meta data, plus 'is adult' checking.
    def extractChapterUrlsAndMetadata(self):

@@ -145,51 +152,67 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
            
        except urllib2.HTTPError, e:
            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.meta)
+                raise exceptions.StoryDoesNotExist(self.url)
            else:
                raise e
-                
+
+        if "Sorry, we couldn&#x27;t find the work you were looking for." in data:
+            raise exceptions.StoryDoesNotExist(self.url)
+            
        if self.needToLoginCheck(data):
            # need to log in for this one.
            self.performLogin(url,data)
-            data = self._fetchUrl(url)
-            meta = self._fetchUrl(metaurl)
+            data = self._fetchUrl(url,usecache=False)
+            meta = self._fetchUrl(metaurl,usecache=False)
            
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)
+        for tag in soup.findAll('div',id='admin-banner'):
+            tag.extract()
        metasoup = bs.BeautifulSoup(meta)
+        for tag in metasoup.findAll('div',id='admin-banner'):
+            tag.extract()

        # Now go hunting for all the meta data and the chapter list.
        
        ## Title
-        a = soup.find('a', href=re.compile(r"^/works/\d+$"))
+        a = soup.find('a', href=re.compile(r"/works/\d+$"))
        self.story.setMetadata('title',stripHTML(a))
 		
        # Find authorid and URL from... author url.
-        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        alist = soup.findAll('a', href=re.compile(r"/users/\w+/pseuds/\w+"))
        if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
            self.story.setMetadata('author','Anonymous')
            self.story.setMetadata('authorUrl','http://archiveofourown.org/')
            self.story.setMetadata('authorId','0')
        else:
            for a in alist:
-                self.story.addToList('authorId',a['href'].split('/')[2])
-                self.story.addToList('authorUrl','http://'+self.host+a['href'])
+                self.story.addToList('authorId',a['href'].split('/')[-1])
+                self.story.addToList('authorUrl',a['href'])
                self.story.addToList('author',a.text)

+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on AO3 it's
+        # possible for authors to insert new chapters out-of-order or
+        # change the dates of earlier ones by editing them--That WILL
+        # break epub update.
        # Find the chapters:
        chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
        self.story.setMetadata('numChapters',len(chapters))
        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        for x in range(0,len(chapters)):
-            # just in case there's tags, like <i> in chapter titles.
-            chapter=chapters[x]
-            if len(chapters)==1:
-                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
-            else:
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapters[0]['href']+addurl))
+        else:
+            for index, chapter in enumerate(chapters):
+                # strip just in case there's tags, like <i> in chapter titles.
                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
-
-
+                # (2013-09-21)
+                date = stripHTML(chapter.findNext('span'))[1:-1]
+                chapterDate = makeDate(date,self.dateformat)
+                if newestChapter == None or chapterDate > newestChapter:
+                    newestChapter = chapterDate
+                    self.newestChapterNum = index

        a = metasoup.find('blockquote',{'class':'userstuff'})
        if a != None:
@@ -299,6 +322,12 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                    self.setSeries(series_name, series_index)
                    self.story.setMetadata('seriesUrl',series_url)

+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
    # grab the text for an individual chapter.
    def getChapterText(self, url):
        logger.debug('Getting chapter text from: %s' % url)
@@ -306,25 +335,31 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
        data = self._fetchUrl(url)
        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-		
-        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-        if headnotes != None:
-            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+
+        exclude_notes=self.getConfigList('exclude_notes')
+
+        if 'authorheadnotes' not in exclude_notes:
+            headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
            if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(headnotes)
+                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+                if headnotes != None:
+                    chapter.append("<b>Author's Note:</b>")
+                    chapter.append(headnotes)
        
-        chapsumm = soup.find('div', {'id' : "summary"})
-        if chapsumm != None:
-            chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
-            chapter.append(chapsumm)
-        chapnotes = soup.find('div', {'id' : "notes"})
-        if chapnotes != None:
-            chapnotes = chapnotes.find('blockquote')
+        if 'chaptersummary' not in exclude_notes:
+            chapsumm = soup.find('div', {'id' : "summary"})
+            if chapsumm != None:
+                chapsumm = chapsumm.find('blockquote')
+                chapter.append("<b>Summary for the Chapter:</b>")
+                chapter.append(chapsumm)
+                
+        if 'chapterheadnotes' not in exclude_notes:
+            chapnotes = soup.find('div', {'id' : "notes"})
            if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapnotes)
+                chapnotes = chapnotes.find('blockquote')
+                if chapnotes != None:
+                    chapter.append("<b>Notes for the Chapter:</b>")
+                    chapter.append(chapnotes)
 		
        text = soup.find('div', {'class' : "userstuff module"})
        chtext = text.find('h3', {'class' : "landmark heading"})
@@ -332,17 +367,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
            chtext.extract()
        chapter.append(text)
 		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-        if chapfoot != None:
-            chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
-            chapter.append(chapfoot)
+        if 'chapterfootnotes' not in exclude_notes:
+            chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+            if chapfoot != None:
+                chapfoot = chapfoot.find('blockquote')
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapfoot)
 		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        if footnotes != None:
-            footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
-            chapter.append(footnotes)
+        if 'authorfootnotes' not in exclude_notes:
+            footnotes = soup.find('div', {'id' : "work_endnotes"})
+            if footnotes != None:
+                footnotes = footnotes.find('blockquote')
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(footnotes)
 			
        if None == soup:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
@@ -71,7 +71,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
        return ['archive.skyehawke.com','www.skyehawke.com']

    @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
        return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"

    def getSiteURLPattern(self):
@@ -91,8 +91,6 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
            else:
                raise e

-        data = self._fetchUrl(url)
-            
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)
        # print data
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -67,8 +68,8 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
        return 'ashwinder.sycophanthex.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -64,8 +65,8 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
        return 'asr3.slashzone.org'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
@@ -0,0 +1,346 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+def _translate_date_german_english(date):
+    fullmon = {"Januar":"01",
+               "Februar":"02",
+               u"März":"03",
+               "April":"04",
+               "Mai":"05",
+               "Juni":"06",
+               "Juli":"07",
+               "August":"08",
+               "September":"09",
+               "Oktober":"10",
+               "November":"11",
+               "Dezember":"12"}
+    for (name,num) in fullmon.items():
+        date = date.replace(name,num)
+    return date
+
+_REGEX_TRAILING_DIGIT = re.compile("(\d+)$")
+_REGEX_DASH_TO_END = re.compile("-[^-]+$")
+_REGEX_CHAPTER_TITLE = re.compile(ur"""
+    \s*
+    [\u2013-]?
+    \s*
+    ([\dIVX-]+)?
+    \.?
+    \s*
+    [\[\(]?
+    \s*
+    (Teil|Kapitel|Tag)?
+    \s*
+    ([\dIVX-]+)?
+    \s*
+    [\]\)]?
+    \s*
+    $
+""", re.VERBOSE)
+_INITIAL_STEP = 5
+
+class BdsmGeschichtenAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8", "Windows-1252"]
+
+        self.story.setMetadata('siteabbrev','bdsmgesch')
+
+        # Replace possible chapter numbering
+        chapterMatch = _REGEX_TRAILING_DIGIT.search(url)
+        if chapterMatch is None:
+            self.maxChapter = 1
+        else:
+            self.maxChapter = int(chapterMatch.group(1))
+        # url = re.sub(_REGEX_TRAILING_DIGIT, "1", url)
+
+        # set storyId
+        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId'))
+
+        # normalize URL
+        self._setURL('http://%s/%s' % (self.getSiteDomain(), self.story.getMetadata('storyId')))
+
+        self.dateformat = '%d. %m %Y - %H:%M'
+
+    @staticmethod
+    def getSiteDomain():
+        return 'bdsm-geschichten.net'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.bdsm-geschichten.net', 'www.bdsm-geschichten.net']
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://www.bdsm-geschichten.net/title-of-story-1 http://bdsm-geschichten.net/title-of-story-1"
+
+    def getSiteURLPattern(self):
+        return r"http://(www\.)?bdsm-geschichten.net/(?P<storyId>[a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+
+        try:
+            data1 = self._fetchUrl(self.url)
+            soup = bs.BeautifulSoup(data1)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #strip comments from soup
+        [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # Cache the soups so we won't have to redownload in getChapterText later
+        self.soupsCache = {}
+        self.soupsCache[self.url] = soup
+
+        # author
+        authorDiv = soup.find("div", "author-pane-line author-name")
+        authorId = authorDiv.string.strip()
+        self.story.setMetadata('authorId', authorId)
+        self.story.setMetadata('author', authorId)
+        # TODO not really true need to be loggedin for this to work or fetch userid
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+authorId)
+
+        # TODO better metadata
+        date = soup.find("div", {"class": "submitted"}).string.strip()
+        date = re.sub(" &#151;.*", "", date)
+        date = _translate_date_german_english(date)
+        self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+        title1 = soup.find("h1", {'class': 'title'}).string
+
+
+        for tagLink in soup.find("ul", "taxonomy").findAll("a"):
+            self.story.addToList('category', tagLink.string)
+
+        ## Retrieve chapter soups
+        if self.getConfig('find_chapters') == 'guess':
+            self.chapterUrls = []
+            self._find_chapters_by_guessing(title1)
+        else:
+            self._find_chapters_by_parsing(soup)
+
+        firstChapterUrl = self.chapterUrls[0][1]
+        if firstChapterUrl in self.soupsCache:
+            firstChapterSoup = self.soupsCache[firstChapterUrl]
+            h1 = firstChapterSoup.find("h1").text
+        else:
+            h1 = soup.find("h1").text
+
+        h1 = re.sub(_REGEX_CHAPTER_TITLE, "", h1)
+        self.story.setMetadata('title', h1)
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+        return
+
+    def _find_chapters_by_parsing(self, soup):
+
+        # store original soup
+        origSoup = soup
+
+        #
+        # find first chapter
+        #
+        firstLink = None
+        firstLinkDiv = soup.find("div", "field-field-erster-teil")
+        if firstLinkDiv is not None:
+            firstLink = "http://%s%s" % (self.getSiteDomain(), firstLinkDiv.findNext("a")['href'])
+            logger.debug("Found first chapter right away <%s>" % firstLink)
+            try:
+                soup = bs.BeautifulSoup(self._fetchUrl(firstLink))
+                self.soupsCache[firstLink] = soup
+                self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(self.url)
+                else:
+                    raise exceptions.StoryDoesNotExist(firstLink)
+        else:
+            logger.debug("DIDN'T find first chapter right away")
+            # parse previous Link until first
+            while True:
+                prevLink = None
+                prevLinkDiv = soup.find("div", "field-field-vorheriger-teil")
+                if prevLinkDiv is not None:
+                    prevLink = prevLinkDiv.find("a")
+                if prevLink is None:
+                    prevLink = soup.find("a", text=re.compile("&lt;&lt;&lt;")) # <<<
+                if prevLink is None:
+                    logger.debug("Couldn't find prev part")
+                    break
+                else:
+                    logger.debug("Previous Chapter <%s>" % prevLink)
+                    if type(prevLink) != bs.Tag or prevLink.name != "a":
+                        prevLink = prevLink.findParent("a")
+                    if prevLink is None or '#' in prevLink['href']:
+                        logger.debug("Couldn't find prev part (false positive) <%s>" % prevLink)
+                        break
+                    prevLink = prevLink['href']
+                try:
+                    soup = bs.BeautifulSoup(self._fetchUrl(prevLink))
+                    self.soupsCache[prevLink] = soup
+                    prevTtitle = soup.find("h1", {'class': 'title'}).string
+                    self.chapterUrls.insert(0, (prevTtitle, prevLink))
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(nextLink)
+                    else:
+                        raise e
+                firstLink = prevLink
+
+        # if first chapter couldn't be determined, assume the URL originally
+        # passed is the first chapter
+        if firstLink is None:
+            logger.debug("Couldn't set first chapter")
+            firstLink = self.url
+            self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
+
+        # set first URL
+        logger.debug("Set first link: %s" % firstLink)
+        self._setURL(firstLink)
+        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(firstLink).group('storyId'))
+
+        #
+        # Parse next chapters
+        #
+        while True:
+            nextLink = None
+            nextLinkDiv = soup.find("div", "field-field-naechster-teil")
+            if nextLinkDiv is not None:
+                nextLink = nextLinkDiv.find("a")
+            if nextLink is None:
+                nextLink = soup.find("a", text=re.compile("&gt;&gt;&gt;"))
+            if nextLink is None:
+                nextLink = soup.find("a", text=re.compile("Fortsetzung"))
+
+            if nextLink is None:
+                logger.debug("Couldn't find next part")
+                break
+            else:
+                if type(nextLink) != bs.Tag or nextLink.name != "a":
+                    nextLink = nextLink.findParent("a")
+                if nextLink is None or '#' in nextLink['href']:
+                    logger.debug("Couldn't find next part (false positive) <%s>" % nextLink)
+                    break
+                nextLink = nextLink['href']
+
+            if not nextLink.startswith('http:'):
+                nextLink = 'http://' + self.getSiteDomain() + nextLink
+
+            for loadedChapter in self.chapterUrls:
+                if loadedChapter[0] == nextLink:
+                    logger.debug("ERROR: Repeating chapter <%s> Try to fix it" % nextLink)
+                    nextLinkMatch = _REGEX_TRAILING_DIGIT.match(nextLink)
+                    if nextLinkMatch is not None:
+                        curChap = nextLinkMatch.group(1)
+                        nextLink = re.sub(_REGEX_TRAILING_DIGIT, str(int(curChap) + 1), nextLink)
+                    else:
+                        break
+            try:
+                data = self._fetchUrl(nextLink)
+                soup = bs.BeautifulSoup(data)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(nextLink)
+                else:
+                    raise e
+            title2 = soup.find("h1", {'class': 'title'}).string
+            self.chapterUrls.append((title2, nextLink))
+            logger.debug("Grabbing next chapter URL " + nextLink)
+            self.soupsCache[nextLink] = soup
+            # [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+        logger.debug("Chapters: %s" % self.chapterUrls)
+
+
+    def _find_chapters_by_guessing(self, title1):
+        step = _INITIAL_STEP
+        curMax = self.maxChapter + step
+        lastHit = True
+        while True:
+            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(curMax), self.url) 
+            if nextChapterUrl == self.url:
+                logger.debug("Unable to guess next chapter because URL doesn't end in numbers")
+                break;
+            try:
+                logger.debug("Trying chapter URL " + nextChapterUrl)
+                data = self._fetchUrl(nextChapterUrl)
+                hit = True
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    hit = False
+                else:
+                    raise e
+            if hit:
+                logger.debug("Found chapter URL " + nextChapterUrl)
+                self.maxChapter = curMax
+                self.soupsCache[nextChapterUrl] = bs.BeautifulSoup(data)
+                if not lastHit:
+                    break
+                lastHit = curMax
+                curMax += step
+            else:
+                lastHit = False
+                curMax -= 1
+            logger.debug(curMax)
+
+        for i in xrange(1, self.maxChapter):
+            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(i), self.url)
+            nextChapterTitle = re.sub("1", str(i), title1)
+            self.chapterUrls.append((nextChapterTitle, nextChapterUrl))
+
+    def getChapterText(self, url):
+
+        if url in self.soupsCache:
+            logger.debug('Getting chapter <%s> from cache' % url)
+            soup = self.soupsCache[url]
+        else:
+            logger.debug('Downloading chapter <%s>' % url)
+            data1 = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data1)
+            #strip comments from soup
+            [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        storyDiv1 = bs.Tag(soup, "div")
+        for para in soup.find("div", "full-node").find('div', 'content').findAll("p"):
+            storyDiv1.append(para)
+        storyDiv1.append('<br />')
+        storytext = self.utf8FromSoup(url,storyDiv1)
+
+        return storytext
+
+
+def getClass():
+    return BdsmGeschichtenAdapter
@@ -0,0 +1,193 @@
+from datetime import timedelta
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+from ..htmlcleanup import stripHTML
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return BloodshedverseComAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class BloodshedverseComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'bvc'
+    SITE_DOMAIN = 'bloodshedverse.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/'
+    READ_URL_TEMPLATE = BASE_URL + 'stories.php?go=read&no=%s'
+
+    STARTED_DATETIME_FORMAT = '%m/%d/%Y'
+    UPDATED_DATETIME_FORMAT = '%m/%d/%Y %I:%M'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_no = query_data['no'][0]
+
+        self.story.setMetadata('storyId', story_no)
+        self._setURL(self.READ_URL_TEMPLATE % story_no)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return BloodshedverseComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL + 'stories.php?go=') + r'(read|chapters)\&no=\d+$'
+
+    # Override stripURLParameters so the "no" parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if stripHTML(soup.title) == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        for option in soup.find('select', {'name': 'chapter'}):
+            title = stripHTML(option)
+            url = self.READ_URL_TEMPLATE % option['value']
+            self.chapterUrls.append((title, url))
+
+        # Get the URL to the author's page and find the correct story entry to
+        # scrape the metadata
+        author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_no = self.story.getMetadata('storyId')
+        # Ignore first list_box div, it only contains the author information
+        for list_box in soup('div', {'class': 'list_box'})[1:]:
+            url = list_box.find('a', {'class': 'fictitle'})['href']
+            query_data = _get_query_data(url)
+
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if query_data['no'] == story_no:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title_anchor = list_box.find('a', {'class': 'fictitle'})
+        self.story.setMetadata('title', stripHTML(title_anchor))
+
+        author_anchor = title_anchor.findNextSibling('a')
+        self.story.setMetadata('author', stripHTML(author_anchor))
+        self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
+
+        list_review = list_box.find('div', {'class': 'list_review'})
+        reviews = stripHTML(list_review.a).split(' ', 1)[0]
+        self.story.setMetadata('reviews', reviews)
+
+        summary_div = list_box.find('div', {'class': 'list_summary'})
+        if not self.getConfig('keep_summary_html'):
+            summary = ''.join(summary_div(text=True))
+        else:
+            summary = self.utf8FromSoup(author_url, summary_div)
+
+        self.story.setMetadata('description', summary)
+
+        # I'm assuming this to be the category, not sure what else it could be
+        first_listinfo = list_box.find('div', {'class': 'list_info'})
+        self.story.addToList('category', stripHTML(first_listinfo.a))
+
+        for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
+            for b_tag in list_info('b'):
+                key = b_tag.string.strip(': ')
+                # Strip colons from the beginning, superfluous spaces and minus
+                # characters from the end, and possibly trailing commas from
+                # the warnings if only one is present
+                value = b_tag.nextSibling.string.strip(': -,')
+
+                if key == 'Genre':
+                    for genre in value.split(', '):
+                        # Ignore the "none" genre
+                        if not genre == 'none':
+                            self.story.addToList('genre', genre)
+
+                elif key == 'Rating':
+                    self.story.setMetadata('rating', value)
+
+                elif key == 'Complete':
+                    self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+                elif key == 'Warning':
+                    for warning in value.split(', '):
+                        # The string here starts with ", " before the actual list
+                        # of values sometimes, so check for an empty warning
+                        # and ignore the "none" warning.
+                        if not warning or warning == 'none':
+                            continue
+
+                        self.story.addToList('warnings', warning)
+
+                elif key == 'Chapters':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Words':
+                    # Apparently only numChapters need to be an integer for
+                    # some strange reason. Remove possible ',' characters as to
+                    # not confuse the codebase down the line
+                    self.story.setMetadata('numWords', value.replace(',', ''))
+
+                elif key == 'Started':
+                    self.story.setMetadata('datePublished', makeDate(value, self.STARTED_DATETIME_FORMAT))
+
+                elif key == 'Updated':
+                    date_string, period = value.rsplit(' ', 1)
+                    date = makeDate(date_string, self.UPDATED_DATETIME_FORMAT)
+
+                    # Rather ugly hack to work around Calibre's changing of
+                    # Python's locale setting, causing am/pm to not be properly
+                    # parsed by strptime() when using a non-english locale
+                    if period == 'pm':
+                        date += timedelta(hours=12)
+                    self.story.setMetadata('dateUpdated', date)
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        storytext_div = soup.find('div', {'class': 'storytext'})
+
+        if self.getConfig('strip_text_links'):
+            for anchor in storytext_div('a', {'class': 'FAtxtL'}):
+                navigable_string = BeautifulSoup.NavigableString(anchor.string)
+                anchor.replaceWith(navigable_string)
+
+        return self.utf8FromSoup(url, storytext_div)
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -89,8 +90,8 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
        return 'bloodties-fans.com' # XXX

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fiction/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
@@ -87,11 +87,11 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
    
    def setHeader(self):
        "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
-        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
        
    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://buffynfaith.net/fanfictions/index.php?act=ovr&id=1234 http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234&ch=2"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=ovr&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234&ch=2"

    def getSiteURLPattern(self):
        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
@@ -101,6 +101,13 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
            r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
        return p

+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
    def extractChapterUrlsAndMetadata(self):

        dateformat = "%d %B %Y"
@@ -109,7 +116,6 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
        
        #set a cookie to get past adult check
        if self.is_adult or self.getConfig("is_adult"):
-            cookieproc = urllib2.HTTPCookieProcessor()
            cookie = cl.Cookie(version=0, name='my_age', value='yes',
                               port=None, port_specified=False,
                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
@@ -121,8 +127,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
                               comment_url=None,
                               rest={'HttpOnly': None},
                               rfc2109=False)
-            cookieproc.cookiejar.set_cookie(cookie)
-            self.opener = urllib2.build_opener(cookieproc)        
+            self.cookiejar.set_cookie(cookie)
            self.setHeader()

        try:
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -91,8 +92,8 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
        return 'castlefans.org' # XXX

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -67,8 +68,8 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
        return 'chaos.sycophanthex.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -65,8 +65,8 @@ class CheckmatedComAdapter(BaseSiteAdapter):
        return 'www.checkmated.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?story=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/story.php?story=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/story.php?story=")+r"\d+$"
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class CSIForensicsComAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'csi-forensics.com'
+
+    @classmethod
+    def getPathToArchive(self):
+        return ''
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'csiforensics'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%d %b %Y" 
+   
+    def handleMetadataPair(self, key, value):
+        if key == 'Warnings':
+            for val in re.split("\s*,\s*", value):
+                if value == 'None':
+                    return
+                else:
+                    self.story.addToList('warnings', val)
+     
+        elif 'Categories' in key:
+            for val in re.split("\s*>\s*", value):
+                self.story.addToList('category', val)
+        else:
+            super(CSIForensicsComAdapter, self).handleMetadataPair(key, value) 
+                           
+def getClass():
+    return CSIForensicsComAdapter
+    
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -70,8 +71,8 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
        return ['www.dark-solace.org','dark-solace.org']

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/elysian/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
@@ -182,6 +183,11 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):

        # first a tag in pagetitle is title
        self.story.setMetadata('title',stripHTML(div.find('a')))
+        div.find('a').extract()
+        # only thing left in div(pagetitle) now should be 'by' and rating.
+        rating = stripHTML(div)
+        if '[' in rating:
+            self.story.setMetadata('rating', rating[rating.index('[')+1:-1])

        for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
                                                       self.story.getMetadata('storyId')+'&chapter=\d+')):
@@ -234,31 +240,28 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                self.setDescription(url,svalue)
                #self.story.setMetadata('description',stripHTML(svalue))

-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
            if 'Word count' in label:
                self.story.setMetadata('numWords', value)

            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                for cat in cats:
                    self.story.addToList('category',cat.string)

            if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)

            if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)

            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)

            if 'Completed' in label:
                if 'Yes' in value:
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
        return 'www.destinysgateway.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -76,8 +76,8 @@ class DokugaComAdapter(BaseSiteAdapter):
        return 'www.dokuga.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfiction/story/1234/1 http://"+cls.getSiteDomain()+"/spark/story/1234/1"

    def getSiteURLPattern(self):
        return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
@@ -66,8 +66,8 @@ class DotMoonNetAdapter(BaseSiteAdapter):
        return 'www.dotmoon.net'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/library_view.php?storyid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/library_view.php?storyid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/library_view.php?storyid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
        return 'www.dracoandginny.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class DramioneOrgAdapter(BaseSiteAdapter):
        return 'dramione.org'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
        return 'efiction.esteliel.de'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,15 +67,16 @@ class EFPFanFicNet(BaseSiteAdapter):
        return 'www.efpfanfic.net'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

    ## Login seems to be reasonably standard across eFiction sites.
    def needToLoginCheck(self, data):
-        if 'Fai il login e leggi la storia!' in data:
+        if( 'Fai il login e leggi la storia!' in data or
+            'Questa storia presenta contenuti non adatti ai minori' in data ):
            return True
        else:
            return False
@@ -203,8 +205,8 @@ class EFPFanFicNet(BaseSiteAdapter):
        noteblock = storyblock.find('div', {'class':'notebloc'})
        #print("%s"%noteblock)
        notetext = ("%s" % noteblock).replace("<br />"," |")
-        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />␍
-        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />␍
+        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />
+        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />
        # Categoria: <a href="categories.php?catid=1&amp;parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&amp;parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&amp;a=">3</a> recensioni</div>

        cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -67,8 +68,8 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
        return 'erosnsappho.sycophanthex.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -0,0 +1,185 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanficHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class FanficHuAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'ffh'
+    SITE_DOMAIN = 'fanfic.hu'
+    SITE_LANGUAGE = 'Hungarian'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/merengo/'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%s'
+
+    DATE_FORMAT = '%m/%d/%Y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanficHuAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&i=1')
+
+        if soup.title.string.encode(_SOURCE_CODE_ENCODING).strip(' :') == 'írta':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        chapter_options = soup.find('form', action='viewstory.php').select('option')
+        # Remove redundant "Fejezetek" option
+        chapter_options.pop(0)
+
+        # If there is still more than one entry remove chapter overview entry
+        if len(chapter_options) > 1:
+            chapter_options.pop(0)
+
+        for option in chapter_options:
+            url = urlparse.urljoin(self.url, option['value'])
+            self.chapterUrls.append((option.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_id = self.story.getMetadata('storyId')
+        for table in soup('table', {'class': 'mainnav'}):
+            title_anchor = table.find('span', {'class': 'storytitle'}).a
+            href = title_anchor['href']
+            if href.startswith('javascript:'):
+                href = href.rsplit(' ', 1)[1].strip("'")
+            query_data = _get_query_data(href)
+
+            if query_data['sid'] == story_id:
+                break
+        else:
+            # This should never happen, the story must be found on the author's
+            # page.
+            raise exceptions.FailedToDownload(self.url)
+
+        self.story.setMetadata('title', title_anchor.string)
+
+        rows = table('tr')
+
+        anchors = rows[0].div('a')
+        author_anchor = anchors[1]
+        query_data = _get_query_data(author_anchor['href'])
+        self.story.setMetadata('author', author_anchor.string)
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.BASE_URL, author_anchor['href']))
+        self.story.setMetadata('reviews', anchors[3].string)
+
+        if self.getConfig('keep_summary_html'):
+            self.story.setMetadata('description', self.utf8FromSoup(author_url, rows[1].td))
+        else:
+            self.story.setMetadata('description', ''.join(rows[1].td(text=True)))
+
+        for row in rows[3:]:
+            index = 0
+            cells = row('td')
+
+            while index < len(cells):
+                cell = cells[index]
+                key = cell.b.string.encode(_SOURCE_CODE_ENCODING).strip(':')
+                try:
+                    value = cells[index+1].string.encode(_SOURCE_CODE_ENCODING)
+                except AttributeError:
+                    value = None
+
+                if key == 'Kategória':
+                    for anchor in cells[index+1]('a'):
+                        self.story.addToList('category', anchor.string)
+
+                elif key == 'Szereplõk':
+                    if cells[index+1].string:
+                        for name in cells[index+1].string.split(', '):
+                            self.story.addToList('character', name)
+
+                elif key == 'Korhatár':
+                    if value != 'nem korhatáros':
+                        self.story.setMetadata('rating', value)
+
+                elif key == 'Figyelmeztetések':
+                    for b_tag in cells[index+1]('b'):
+                        self.story.addToList('warnings', b_tag.string)
+
+                elif key == 'Jellemzõk':
+                    for genre in cells[index+1].string.split(', '):
+                        self.story.addToList('genre', genre)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Megjelenés':
+                    self.story.setMetadata('datePublished', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Frissítés':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Szavak':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                index += 2
+
+        if self.story.getMetadata('rating') == '18':
+            if not (self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        story_cell = soup.find('form', action='viewstory.php').parent.parent
+
+        for div in story_cell('div'):
+            div.extract()
+
+        return self.utf8FromSoup(url, story_cell)
@@ -0,0 +1,218 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanfictionCsodaidokHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FanfictionCsodaidokHuAdapter(BaseSiteAdapter):
+    _SITE_DOMAIN = 'fanfiction.csodaidok.hu'
+    _BASE_URL = 'http://' + _SITE_DOMAIN + '/'
+    _VIEW_STORY_URL_TEMPLATE = _BASE_URL + 'viewstory.php?sid=%s'
+    _VIEW_CHAPTER_URL_TEMPLATE = _VIEW_STORY_URL_TEMPLATE + '&chapter=%s'
+
+    _STORY_DOES_NOT_EXIST_PAGE_TITLE = 'Cím:  Szerző:'
+    _DATE_FORMAT = '%Y.%m.%d'
+    _SITE_LANGUAGE = 'Hungarian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self._VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self._SITE_DOMAIN)
+        self.story.setMetadata('language', self._SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanfictionCsodaidokHuAdapter._SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls._VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self._VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&chapter=1')
+
+        element = soup.find('div', id='pagetitle')
+        page_title = ''.join(element(text=True)).encode(_SOURCE_CODE_ENCODING)
+        if page_title == self._STORY_DOES_NOT_EXIST_PAGE_TITLE:
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        author_url = urlparse.urljoin(self.url, element.a['href'])
+
+        story_id = self.story.getMetadata('storyId')
+        element = soup.find('select', {'name': 'chapter'})
+        if element:
+            for option in element('option'):
+                title = option.string
+                url = self._VIEW_CHAPTER_URL_TEMPLATE % (story_id, option['value'])
+                self.chapterUrls.append((title, url))
+
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+
+        for listbox_div in soup('div', {'class': lambda klass: klass and 'listbox' in klass}):
+            a = listbox_div.div.a
+            if not a['href'].startswith('viewstory.php?sid='):
+                continue
+
+            query_data = _get_query_data(a['href'])
+            if query_data['sid'] == story_id:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = ''.join(a(text=True))
+        self.story.setMetadata('title', title)
+        if not self.chapterUrls:
+            self.chapterUrls.append((title, self.url))
+
+        element = a.findNextSibling('a')
+        self.story.setMetadata('author', element.string)
+        query_data = _get_query_data(element['href'])
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', author_url)
+
+        element = element.findNextSibling('span')
+        rating = element.nextSibling.strip(' [')
+
+        if rating.encode(_SOURCE_CODE_ENCODING) != 'Korhatár nélkül':
+            self.story.setMetadata('rating', rating)
+
+        if rating == '18':
+            raise exceptions.AdultCheckRequired(self.url)
+
+        element = element.findNextSiblings('a')[1]
+        self.story.setMetadata('reviews', element.string)
+
+        sections = listbox_div('div', {'class': lambda klass: klass and klass in ['content', 'tail']})
+        for section in sections:
+            for element in section('span', {'class': 'classification'}):
+                key = element.string.encode(_SOURCE_CODE_ENCODING).strip(' :')
+                try:
+                    value = element.nextSibling.string.encode(_SOURCE_CODE_ENCODING).strip()
+                except AttributeError:
+                    value = None
+
+                if key == 'Tartalom':
+                    contents = []
+                    keep_summary_html = self.getConfig('keep_summary_html')
+
+                    for sibling in _yield_next_siblings(element):
+                        if isinstance(sibling, BeautifulSoup.Tag):
+                            if sibling.name == 'span' and sibling.get('class', None) == 'classification':
+                                break
+
+                            if keep_summary_html:
+                                contents.append(self.utf8FromSoup(author_url, sibling))
+                            else:
+                                contents.append(''.join(sibling(text=True)))
+                        else:
+                            contents.append(sibling)
+                    self.story.setMetadata('description', ''.join(contents))
+
+                elif key == 'Kategória':
+                    for sibling in element.findNextSiblings(['a', 'span']):
+                        if sibling.name == 'span':
+                            break
+
+                        self.story.addToList('category', sibling.string)
+
+                elif key == 'Szereplők':
+                    for name in value.split(', '):
+                        self.story.addToList('characters', name)
+
+                elif key == 'Műfaj':
+                    if value != 'Nincs':
+                        self.story.setMetadata('genre', value)
+
+                elif key == 'Figyelmeztetés':
+                    if value != 'Nincs':
+                        for warning in value.split(', '):
+                            self.story.addToList('warnings', warning)
+
+                elif key == 'Kihívás':
+                    if value != 'Nincs':
+                        self.story.setMetadata('challenge', value)
+
+                elif key == 'Sorozat':
+                    if value != 'Nincs':
+                        self.story.setMetadata('series', value)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                elif key == 'Szavak száma':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Feltöltve':
+                    self.story.setMetadata('datePublished', makeDate(value, self._DATE_FORMAT))
+
+                elif key == 'Frissítve':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self._DATE_FORMAT))
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        contents = []
+
+        notes_div = soup.find('div', id='notes')
+        if notes_div:
+            contents.append(self.utf8FromSoup(url, notes_div))
+            story_div = notes_div.findNextSibling('div')
+        else:
+            element = soup.find('div', {'class': 'jumpmenu'})
+            story_div = element.findNextSibling('div')
+
+        contents.append(self.utf8FromSoup(url, story_div.span))
+        return ''.join(contents)
@@ -52,11 +52,15 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        # latest chapter yet and going back to chapter 1 to pull the
        # chapter list doesn't get the latest.  So save and use the
        # original URL given to pull chapter list & metadata.
+        # Not used by plugin because URL gets normalized first for
+        # eliminating duplicate story urls.
        self.origurl = url
        if "https://m." in self.origurl:
            ## accept m(mobile)url, but use www.
            self.origurl = self.origurl.replace("https://m.","https://www.")

+        self.opener.addheaders.append(('Referer',self.origurl))
+
    @staticmethod
    def getSiteDomain():
        return 'www.fanfiction.net'
@@ -66,13 +70,29 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        return ['www.fanfiction.net','m.fanfiction.net']

    @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
        return "https://www.fanfiction.net/s/1234/1/ https://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"

    def getSiteURLPattern(self):
        return r"https?://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"

-    def extractChapterUrlsAndMetadata(self):
+    def _fetchUrl(self,url,parameters=None,extrasleep=1.0,usecache=True):
+        ## ffnet(and, I assume, fpcom) tends to fail more if hit too
+        ## fast.  This is in additional to what ever the
+        ## slow_down_sleep_time setting is.
+        return BaseSiteAdapter._fetchUrl(self,url,
+                                         parameters=parameters,
+                                         extrasleep=extrasleep,
+                                         usecache=usecache)
+
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
+    def doExtractChapterUrlsAndMetadata(self,get_cover=True):

        # fetch the chapter.  From that we will get almost all the
        # metadata and chapter list
@@ -98,28 +118,31 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        if "not found. Please check to see you are not using an outdated url." in data:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)

-        try:
-            # rather nasty way to check for a newer chapter.  ffnet has a
-            # tendency to send out update notices in email before all
-            # their servers are showing the update on the first chapter.
+        if self.getConfig('check_next_chapter'):
            try:
-                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
-            # get chapter part of url.
+                ## ffnet used to have a tendency to send out update
+                ## notices in email before all their servers were
+                ## showing the update on the first chapter.  It
+                ## generates another server request and doesn't seem
+                ## to be needed lately, so now default it to off.
+                try:
+                    chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+                # get chapter part of url.
+                except:
+                    chapcount = 1
+                chapter = url.split('/',)[5]
+                tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                                self.story.getMetadata('storyId'),
+                                                chapcount+1)
+                logger.debug('=Trying newer chapter: %s' % tryurl)
+                newdata = self._fetchUrl(tryurl)
+                if "not found. Please check to see you are not using an outdated url." \
+                        not in newdata:
+                    logger.debug('=======Found newer chapter: %s' % tryurl)
+                    soup = bs.BeautifulSoup(newdata)
            except:
-                chapcount = 1
-            chapter = url.split('/',)[5]
-            tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            chapcount+1)
-            logger.debug('=Trying newer chapter: %s' % tryurl)
-            newdata = self._fetchUrl(tryurl)
-            if "not found. Please check to see you are not using an outdated url." \
-                    not in newdata:
-                logger.debug('=======Found newer chapter: %s' % tryurl)
-                soup = bs.BeautifulSoup(newdata)
-        except:
-            pass
-        
+                pass
+            
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"^/u/\d+"))
        self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -157,7 +180,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                
                
            
-        a = soup.find('a', href='http://www.fictionratings.com/')
+        a = soup.find('a', href=re.compile(r'https?://www\.fictionratings\.com/'))
        rating = a.string
        if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
            rating = rating[8:]
@@ -244,14 +267,15 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        else:
            self.story.setMetadata('status', 'In-Progress')

-        # Try the larger image first.
-        try:
-            img = soup.find('img',{'class':'lazy cimage'})
-            self.setCoverImage(url,img['data-original'])
-        except:
-            img = soup.find('img',{'class':'cimage'})
-            if img:
-                self.setCoverImage(url,img['src'])
+        if get_cover:
+            # Try the larger image first.
+            try:
+                img = soup.find('img',{'class':'lazy cimage'})
+                self.setCoverImage(url,img['data-original'])
+            except:
+                img = soup.find('img',{'class':'cimage'})
+                if img:
+                    self.setCoverImage(url,img['src'])
            
        # Find the chapter selector 
        select = soup.find('select', { 'name' : 'chapter' } )
@@ -275,27 +299,26 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        return

    def getChapterText(self, url):
-        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
-                        ## more if hit too fast.  This is in
-                        ## additional to what ever the
-                        ## slow_down_sleep_time setting is.
+        # time.sleep(4.0) ## ffnet(and, I assume, fpcom) tends to fail
+        #                 ## more if hit too fast.  This is in
+        #                 ## additional to what ever the
+        #                 ## slow_down_sleep_time setting is.
        logger.debug('Getting chapter text from: %s' % url)
-        data = self._fetchUrl(url)
+        data = self._fetchUrl(url,extrasleep=4.0)

        if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  FanFiction.net Site Error!" % url)
        
        # some ancient stories have body tags inside them that cause
        # soup parsing to discard the content.  For story text we
-        # don't care about anything before "<div class='storytextp"
-        # (there's a space after storytextp, so no close quote(')) and
+        # don't care about anything before "<div role='main'" and
        # this kills any body tags.
        divstr = "<div role='main'"
        if divstr not in data:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
        else:
-            data = data[data.index(divstr):]
-        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
+            data = data[data.index(divstr):] 
+        data = data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
        
        soup = bs.BeautifulSoup(data)

@@ -68,12 +68,19 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        return 'www.fanfiktion.de'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1/story-name"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/s/46ccbef30000616306614050 http://"+cls.getSiteDomain()+"/s/46ccbef30000616306614050/1 http://"+cls.getSiteDomain()+"/s/46ccbef30000616306614050/1/story-name"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?"
        
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
        ## Login seems to be reasonably standard across eFiction sites.
    def needToLoginCheck(self, data):
        if 'Diese Geschichte wurde als entwicklungsbeeintr' in data \
@@ -99,9 +106,8 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        loginUrl = 'https://ssl.fanfiktion.de/'
        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                              params['nickname']))
-        d = self._postUrl(loginUrl,params)
-    
-        if "Login erfolgreich" not in d : #Member Account
+        soup = bs.BeautifulSoup(self._postUrl(loginUrl,params))
+        if not soup.find('a', title='Logout'):
            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                              params['nickname']))
            raise exceptions.FailedToLogin(url,params['nickname'])
@@ -126,7 +132,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        if self.needToLoginCheck(data):
            # need to log in for this one.
            self.performLogin(url)
-            data = self._fetchUrl(url)
+            data = self._fetchUrl(url,usecache=False)
            
        if "Uhr ist diese Geschichte nur nach einer" in data:
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Auserhalb der Zeit von 23:00 Uhr bis 04:00 Uhr ist diese Geschichte nur nach einer erfolgreichen Altersverifikation zuganglich.")
@@ -142,7 +148,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
-        head = soup.find('div', {'class' : 'story-metadata-left-top'})
+        head = soup.find('div', {'class' : 'story-left'})
        a = head.find('a')
        self.story.setMetadata('authorId',a['href'].split('/')[2])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
@@ -156,14 +162,18 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        self.story.setMetadata('language','German')

        #find metadata on the story page
-        self.story.setMetadata('datePublished', makeDate(head.text.split('erstellt: ')[1].split('\n')[0], self.dateformat))
+        headtext = stripHTML(head)
+        self.story.setMetadata('datePublished', makeDate(headtext.split('erstellt: ')[1].split('\n')[0], self.dateformat))
        
-        self.story.setMetadata('dateUpdated', makeDate(head.text.split('letztes Update: ')[1].split('\n')[0], self.dateformat))
-        
-        for genre in head.text.split('&nbsp;&nbsp;&nbsp;')[3].split('/')[0].split(', '):
-            self.story.addToList('genre',genre)
+        self.story.setMetadata('dateUpdated', makeDate(headtext.split('aktualisiert: ')[1].split('\n')[0], self.dateformat))
+
+        # second colspan=3 td in head.
+        genres=stripHTML(head.findAll('td',{'colspan':'3'})[1])
+        self.story.extendList('genre',genres[:genres.index('(')].split(', '))
+        # for genre in head.text.split('&nbsp;&nbsp;&nbsp;')[3].split('/')[0].split(', '):
+        #     self.story.addToList('genre',genre)
            
-        if 'fertiggestellt' in head.text:
+        if 'fertiggestellt' in headtext:
            self.story.setMetadata('status', 'Completed')
        else:
            self.story.setMetadata('status', 'In Progress')
@@ -178,9 +188,9 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        self.setDescription(url,a['onmouseover'].split("', '")[1])
        
        td = tr[i].findAll('td')
-        self.story.addToList('category',stripHTML(td[1]))
-        self.story.setMetadata('rating', stripHTML(td[4]))
-        self.story.setMetadata('numWords', stripHTML(td[5]))
+        self.story.addToList('category',stripHTML(td[2]))
+        self.story.setMetadata('rating', stripHTML(td[5]))
+        self.story.setMetadata('numWords', stripHTML(td[6]))
 
            
    # grab the text for an individual chapter.
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class FanNationAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'fannation.shades-of-moonlight.com'
+
+    @classmethod
+    def getPathToArchive(self):
+        return '/archive'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'fannation'
+
+    def handleMetadataPair(self, key, value):
+        if key == 'Romance':
+            for val in re.split("\s*,\s*", value):
+                self.story.addToList('romance', val)
+        else:
+            super(FanNationAdapter, self).handleMetadataPair(key, value)
+
+def getClass():
+    return FanNationAdapter
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class FHSArchiveComAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'fhsarchive.com'
+
+    @classmethod
+    def getPathToArchive(self):
+        return '/autoarchive'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'fhsa'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%m/%d/%y" 
+   
+    def handleMetadataPair(self, key, value):
+        if key == 'Warnings':
+            for val in re.split("\s*,\s*", value):
+                if value == 'None':
+                    return
+                else:
+                    # toss numbers only.
+                    self.story.addToList('warnings', filter(lambda x : not x.isdigit() , val))
+     
+    #     elif 'Categories' in key:
+    #         for val in re.split("\s*>\s*", value):
+    #             self.story.addToList('category', val)
+        else:
+            super(FHSArchiveComAdapter, self).handleMetadataPair(key, value) 
+                           
+def getClass():
+    return FHSArchiveComAdapter
+    
@@ -70,8 +70,8 @@ class FicBookNetAdapter(BaseSiteAdapter):
        return 'www.ficbook.net'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/readfic/12345 http://"+self.getSiteDomain()+"/readfic/93626/246417#part_content"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/readfic/12345 http://"+cls.getSiteDomain()+"/readfic/93626/246417#part_content"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
@@ -58,8 +58,8 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
        return 'www.fictionalley.org'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/authors/drt/DA.html http://"+self.getSiteDomain()+"/authors/drt/JOTP01a.html"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/authors/drt/DA.html http://"+cls.getSiteDomain()+"/authors/drt/JOTP01a.html"

    def getSiteURLPattern(self):
        # http://www.fictionalley.org/authors/drt/DA.html
@@ -0,0 +1,178 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+from ..BeautifulSoup import NavigableString
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return FictionManiaTVAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FictionManiaTVAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'fmt'
+    SITE_DOMAIN = 'fictionmania.tv'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/stories/'
+    READ_TEXT_STORY_URL_TEMPLATE = BASE_URL + 'readtextstory.html?storyID=%s'
+    DETAILS_URL_TEMPLATE = BASE_URL + 'details.html?storyID=%s'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+    ALTERNATIVE_DATETIME_FORMAT = '%m/%d/%y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['storyID'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.READ_TEXT_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+        # Always single chapters, probably should use the Anthology feature to
+        # merge chapters of a story
+        self.story.setMetadata('numChapters', 1)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FictionManiaTVAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_TEXT_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL) + '(readtextstory|details)\.html\?storyID=\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        url = self.DETAILS_URL_TEMPLATE % self.story.getMetadata('storyId')
+        soup = self._customized_fetch_url(url)
+
+        keep_summary_html = self.getConfig('keep_summary_html')
+        for row in soup.find('table')('tr'):
+            cells = row('td')
+            key = cells[0].b.string.strip(':')
+            try:
+                value = cells[1].string
+            except AttributeError:
+                value = None
+
+            if key == 'Story Name-Title':
+                self.story.setMetadata('title', value)
+                self.chapterUrls.append((value, self.url))
+
+            elif key == 'File Name':
+                self.story.setMetadata('fileName', value)
+
+            elif key == 'File Size':
+                self.story.setMetadata('fileSize', value)
+
+            elif key == 'Author':
+                element = cells[1].a
+                self.story.setMetadata('author', element.string)
+                query_data = _get_query_data(element['href'])
+                self.story.setMetadata('authorId', query_data['word'])
+                self.story.setMetadata('authorUrl', urlparse.urljoin(url, element['href']))
+
+            elif key == 'Date Added':
+                try:
+                    date = makeDate(value, self.DATETIME_FORMAT)
+                except ValueError:
+                    date = makeDate(value, self.ALTERNATIVE_DATETIME_FORMAT)
+                self.story.setMetadata('datePublished', date)
+
+            elif key == 'Old Name':
+                self.story.setMetadata('oldName', value)
+
+            elif key == 'New Name':
+                self.story.setMetadata('newName', value)
+
+            elif key == 'Other Key Names':
+                for name in value.split(', '):
+                    self.story.addToList('characters', name)
+
+            # I have no clue how the rating system works, if you are reading
+            # transgender fanfiction, you are probably an adult.
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Complete':
+                self.story.setMetadata('status', 'Complete' if value == 'Complete' else 'In-Progress')
+
+            elif key == 'Categories':
+                for element in cells[1]('a'):
+                    self.story.addToList('category', element.string)
+
+            elif key == 'Key Words':
+                for element in cells[1]('a'):
+                    self.story.addToList('keyWords', element.string)
+
+            elif key == 'Main Characters Age':
+                element = cells[1].a
+                self.story.setMetadata('mainCharactersAge', element.string)
+
+            elif key == 'Synopsis':
+                element = cells[1]
+
+                # Replace td with div to avoid possible strange formatting in
+                # the ebook later on
+                element.name = 'div'
+
+                if keep_summary_html:
+                    self.story.setMetadata('description', unicode(element))
+                else:
+                    self.story.setMetadata('description', ''.join(element(text=True)))
+
+            elif key == 'Reads':
+                self.story.setMetadata('readings', value)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        element = soup.find('pre')
+        element.name = 'div'
+
+        # The story's content is contained in a <pre> tag, probably taken 1:1
+        # from the source text file. A simple replacement of all newline
+        # characters with a break line tag should take care of formatting.
+
+        # While wrapping in paragraphs would be possible, it's too much work,
+        # I'd rather display the story 1:1 like it was found in the pre tag.
+        content = unicode(element)
+        content = content.replace('\n', '<br />')
+
+        if self.getConfig('non_breaking_spaces'):
+            content = content.replace(' ', '&nbsp;')
+        return content
@@ -57,7 +57,7 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
        return 'fictionpad.com'

    @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
        return "https://fictionpad.com/author/Author/stories/1234/Some-Title"
    
    def getSiteURLPattern(self):
@@ -133,6 +133,7 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
        author = tables['users'][0]
        story = tables['stories'][0]
        story_ver = tables['story_versions'][0]
+        print("story:%s"%story)
        
        self.story.setMetadata('authorId',author['id'])
        self.story.setMetadata('author',author['display_name'])
@@ -151,7 +152,8 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
        self.story.setMetadata('comments',story['comments_count'])
        self.story.setMetadata('views',story['views_count'])
        self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
-        self.story.setMetadata('dislikes',int(story['dislikes']))
+        if 'dislikes' in story:
+            self.story.setMetadata('dislikes',int(story['dislikes']))

        if story_ver['is_complete']:
            self.story.setMetadata('status', 'Completed')
@@ -40,11 +40,11 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
        return ['www.fictionpress.com','m.fictionpress.com']

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
+    def getSiteExampleURLs(cls):
+        return "https://www.fictionpress.com/s/1234/1/ https://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"

    def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"https?://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"

 def getClass():
    return FictionPressComSiteAdapter
@@ -46,7 +46,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
        return 'ficwad.com'

    @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
        return "http://ficwad.com/story/1234"

    def getSiteURLPattern(self):
@@ -65,7 +65,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
        loginUrl = 'http://' + self.getSiteDomain() + '/account/login'
        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                              params['username']))
-        d = self._postUrl(loginUrl,params)
+        d = self._postUrl(loginUrl,params,usecache=False)

        if "Login attempt failed..." in d:
            logger.info("Failed to login to URL %s as %s" % (loginUrl,
@@ -75,6 +75,13 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
        else:
            return True        
    
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
    def extractChapterUrlsAndMetadata(self):

        # fetch the chapter.  From that we will get almost all the
@@ -96,9 +103,15 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
            else:
                raise e
            
-        h3 = soup.find('h3')
-        storya = h3.find('a',href=re.compile("^/story/\d+$"))
-        if storya : # if there's a story link in the h3 header, this is a chapter page.
+        # if blocked, attempt login.
+        if soup.find("div",{"class":"blocked"}):
+            if self.performLogin(url): # performLogin raises
+                                       # FailedToLogin if it fails.
+                soup = bs.BeautifulSoup(self._fetchUrl(url,usecache=False))
+
+        divstory = soup.find('div',id='story')
+        storya = divstory.find('a',href=re.compile("^/story/\d+$"))
+        if storya : # if there's a story link in the divstory header, this is a chapter page.
            # normalize story URL on chapter list.
            self.story.setMetadata('storyId',storya['href'].split('/',)[2])
            url = "http://"+self.getSiteDomain()+storya['href']
@@ -113,17 +126,17 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
                    raise e

        # if blocked, attempt login.
-        if soup.find("li",{"class":"blocked"}):
+        if soup.find("div",{"class":"blocked"}):
            if self.performLogin(url): # performLogin raises
                                       # FailedToLogin if it fails.
-                soup = bs.BeautifulSoup(self._fetchUrl(url))
+                soup = bs.BeautifulSoup(self._fetchUrl(url,usecache=False))

        # title - first h4 tag will be title.
-        titleh4 = soup.find('h4')
+        titleh4 = soup.find('div',{'class':'storylist'}).find('h4')
        self.story.setMetadata('title', stripHTML(titleh4.a))

        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"^/author/\d+"))
+        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/author/\d+"))
        self.story.setMetadata('authorId',a['href'].split('/')[2])
        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
        self.story.setMetadata('author',a.string)
@@ -139,7 +152,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):

        # warnings
        # <span class="req"><a href="/help/38" title="Medium Spoilers">[!!] </a> <a href="/help/38" title="Rape/Sexual Violence">[R] </a> <a href="/help/38" title="Violence">[V] </a> <a href="/help/38" title="Child/Underage Sex">[Y] </a></span>
-        spanreq = metap.find("span",{"class":"req"})
+        spanreq = metap.find("span",{"class":"story-warnings"})
        if spanreq: # can be no warnings.
            for a in spanreq.findAll("a"):
                self.story.addToList('warnings',a['title'])
@@ -165,16 +178,16 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
                if g:
                    self.story.addToList('characters',g)
        
-        m = re.match(r".*?Published: ([0-9/]+?) -.*?",metastr)
+        m = re.match(r".*?Published: ([0-9-]+?) -.*?",metastr)
        if m:
-            self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y/%m/%d"))
+            self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y-%m-%d"))

        # Updated can have more than one space after it. <shrug>
-        m = re.match(r".*?Updated: ([0-9/]+?) +-.*?",metastr) 
+        m = re.match(r".*?Updated: ([0-9-]+?) +-.*?",metastr) 
        if m:
-            self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y/%m/%d"))
+            self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y-%m-%d"))

-        m = re.match(r".*? - ([0-9/]+?) words.*?",metastr)
+        m = re.match(r".*? - ([0-9,]+?) words.*?",metastr)
        if m:
            self.story.setMetadata('numWords',m.group(1))

@@ -185,7 +198,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):

        # get the chapter list first this time because that's how we
        # detect the need to login.
-        storylistul = soup.find('ul',{'id':'storylist'})
+        storylistul = soup.find('ul',{'class':'storylist'})
        if not storylistul:
            # no list found, so it's a one-chapter story.
            self.chapterUrls.append((self.story.getMetadata('title'),url))
@@ -55,16 +55,22 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        return ['www.fimfiction.net','mobile.fimfiction.net', 'www.fimfiction.com', 'mobile.fimfiction.com']

    @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
        return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"

    def getSiteURLPattern(self):
-        return r"http://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
+        return r"https?://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
        
-    def extractChapterUrlsAndMetadata(self):
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
+    def doExtractChapterUrlsAndMetadata(self,get_cover=True):
        
        if self.is_adult or self.getConfig("is_adult"):
-            cookieproc = urllib2.HTTPCookieProcessor()
            cookie = cl.Cookie(version=0, name='view_mature', value='true',
                               port=None, port_specified=False,
                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
@@ -76,16 +82,16 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                               comment_url=None,
                               rest={'HttpOnly': None},
                               rfc2109=False)
-            cookieproc.cookiejar.set_cookie(cookie)
-            self.opener = urllib2.build_opener(cookieproc)
-        
+            self.cookiejar.set_cookie(cookie)
+
+        ##---------------------------------------------------------------------------------------------------
+        ## Get the story's title page. Check if it exists.
+
        try:
-            apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
-            apiData = json.loads(apiResponse)
-            
-            # Unfortunately, we still need to load the story index
-            # page to parse the characters.  And chapters, now, too.
-            data = self._fetchUrl(self.url)
+            # don't use cache if manual is_adult--should only happen
+            # if it's an adult story and they don't have is_adult in ini.
+            data = self.do_fix_blockquotes(self._fetchUrl(self.url,
+                                                          usecache=(not self.is_adult)))
            soup = bs.BeautifulSoup(data)
        except urllib2.HTTPError, e:
            if e.code == 404:
@@ -96,109 +102,97 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
            raise exceptions.StoryDoesNotExist(self.url)

-        # Can cause problems if a missing story is referenced in a comment.
-        # Shouldn't be needed anyway.
-        # if "/images/missing_story.png" in data:
-        #     raise exceptions.StoryDoesNotExist(self.url)
-        
-        if "This story has been marked as having adult content." in data:
+        if "This story has been marked as having adult content. Please click below to confirm you are of legal age to view adult material in your country." in data:
            raise exceptions.AdultCheckRequired(self.url)
        
        if self.password:
            params = {}
            params['password'] = self.password
-            data = self._postUrl(self.url,params)
+            data = self._postUrl(self.url, params)
+            soup = bs.BeautifulSoup(data)

-        if "Enter the password the author set for this story to view it." in data:
+        if not (soup.find('form', {'id' : 'password_form'}) == None):
            if self.getConfig('fail_on_password'):
                raise exceptions.FailedToDownload("%s requires story password and fail_on_password is true."%self.url)
            else:
                raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
-         
-        if "Invalid story id" in apiData.values():
-            raise exceptions.StoryDoesNotExist(self.url)
-        
-        storyMetadata = apiData["story"]    
-            
-        ## Title
-        a = soup.find('a', href=re.compile(r'^/story/'+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # self.story.setMetadata("title", storyMetadata["title"])
-        # if not storyMetadata["title"]:
-        #     raise exceptions.FailedToDownload("%s doesn't have a title in the API.  This is a known fimfiction.net bug with titles containing ."%self.url)
-        
-        self.story.setMetadata("author", storyMetadata["author"]["name"])
-        self.story.setMetadata("authorId", storyMetadata["author"]["id"])
-        self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), storyMetadata["author"]["name"]))

-        # chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
-        
-        # ## this is bit of a kludge based on the assumption all the
-        # ## 'bad' chapters will be at the end.
-        # ## limit down to the number of chapters reported by chapter_count.
-        # chapters = chapters[:storyMetadata["chapter_count"]] 
-        
-        # for chapter in chapters:
-        #     self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
-        # self.story.setMetadata("numChapters", len(self.chapterUrls))
+        ##----------------------------------------------------------------------------------------------------
+        ## Extract metadata

-        for chapter in soup.findAll('a',{'class':'chapter_link'}):
+        storyContentBox = soup.find('div', {'class':'story_content_box'})
+
+        # Title
+        title = storyContentBox.find('a', {'class':re.compile(r'.*\bstory_name\b.*')})
+        self.story.setMetadata('title',stripHTML(title))
+
+        # Author
+        author = storyContentBox.find('span', {'class':'author'})
+        self.story.setMetadata("author", stripHTML(author))
+        #No longer seems to be a way to access Fimfiction's internal author ID
+        self.story.setMetadata("authorId", self.story.getMetadata("author"))
+        self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), stripHTML(author)))
+
+        #Rating text is replaced with full words for historical compatibility after the site changed
+        #on 2014-10-27
+        rating = stripHTML(storyContentBox.find('a', {'class':re.compile(r'.*\bcontent-rating-.*')}))
+        rating = rating.replace("E", "Everyone").replace("T", "Teen").replace("M", "Mature")
+        self.story.setMetadata("rating", rating)
+
+        # Chapters
+        for chapter in storyContentBox.findAll('a',{'class':'chapter_link'}):
            self.chapterUrls.append((stripHTML(chapter), 'http://'+self.host+chapter['href']))

        self.story.setMetadata('numChapters',len(self.chapterUrls))
-            
-        # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
+
+        # Status
+        # In the case of Fimfiction, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
        # For the sake of bringing it in line with the other adapters, 'Incomplete' becomes 'In-Progress'
-        # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
-        # status to leave unchanged.
-        # Nov2012 - 'On Hiatus' is now passed, too.  It's easy now for users to change/remove if they want
-        # with replace_metadata
-        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("Complete", "Completed")
+        # and 'Complete' becomes 'Completed'. 'Cancelled' and 'On Hiatus' are passed through, it's easy now for users
+        # to change/remove if they want with replace_metadata
+        status = stripHTML(storyContentBox.find('span', {'class':re.compile(r'.*\bcompleted-status-.*')}))
+        status = status.replace("Incomplete", "In-Progress").replace("Complete", "Completed")
        self.story.setMetadata("status", status)
-        self.story.setMetadata("rating", storyMetadata["content_rating_text"])

-        ## Warnings aren't included in the API.
-        bottomli = soup.find('li',{'class':'bottom'})
-        if bottomli:
-            bottomspans = bottomli.findAll('span')
-            # the first span in bottom is the rating, obtained above.
-            if bottomspans and len(bottomspans) > 1:
-                for warning in bottomspans[1:]:
-                    self.story.addToList('warnings',warning.string)
-                
-        
-        for category in storyMetadata["categories"]:
-            if storyMetadata["categories"][category]:
-                self.story.addToList("genre", category) 
-
-        self.story.setMetadata("numWords", str(storyMetadata["words"]))
-        
-        # fimfic is the first site with an explicit cover image.
-        if "image" in storyMetadata.keys():
-            if "full_image" in storyMetadata:
-                coverurl = storyMetadata["full_image"]
+        # Genres and Warnings
+        # warnings were folded into general categories in the 2014-10-27 site update
+        categories = storyContentBox.findAll('a', {'class':re.compile(r'.*\bstory_category\b.*')})
+        for category in categories:
+            category = stripHTML(category)
+            if category == "Gore" or category == "Sex":
+                self.story.addToList('warnings', category)
            else:
-                coverurl = storyMetadata["image"]
+                self.story.addToList('genre', category)
+
+        # Word count
+        wordCountText = stripHTML(storyContentBox.find('li', {'class':'bottom'}).find('div', {'class':'word_count'}))
+        self.story.setMetadata("numWords", re.sub(r'[^0-9]', '', wordCountText))
+
+        # Cover image
+        storyImage = storyContentBox.find('div', {'class':'story_image'})
+        if storyImage:
+            coverurl = storyImage.find('a')['href']
            if coverurl.startswith('//'): # fix for img urls missing 'http:'
                coverurl = "http:"+coverurl
+            if get_cover:
+                self.setCoverImage(self.url,coverurl)

-            self.setCoverImage(self.url,coverurl)
+            coverSource = storyImage.find('a', {'class':'source'})
+            if coverSource:
+                self.story.setMetadata('coverSourceUrl', coverSource['href'])
+                #There's no text associated with the cover source link, so just
+                #reuse the URL. Makes it clear it's an external link leading
+                #outside of the fanfic site, at least.
+                self.story.setMetadata('coverSource', coverSource['href'])

        # fimf has started including extra stuff inside the description div.
-        descdivstr = "%s"%soup.find("div", {"class":"description"})
-        hrstr="<hr />"
-        descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
+        descdivstr = u"%s"%storyContentBox.find("div", {"class":"description"})
+        hrstr=u"<hr />"
+        descdivstr = u'<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
        self.setDescription(self.url,descdivstr)

-        # Can't trust dates from API anymore I'm told.
-        # Dates are in Unix time
-        # Take the publish date from the first chapter posted
-        # rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        # rawDateUpdated = storyMetadata["date_modified"]
-        # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
-        
+        # Find the newest and oldest chapter dates
+        storyData = storyContentBox.find('div', {'class':'story_data'})
        oldestChapter = None
        newestChapter = None
        self.newestChapterNum = None # save for comparing during update.
@@ -206,18 +200,21 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        # FiMFiction it's possible for authors to insert new chapters
        # out-of-order or change the dates of earlier ones by editing
        # them--That WILL break epub update.
-        for index, chapterDate in enumerate(soup.findAll('span', {'class':'date'})):
-            date=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",chapterDate.contents[1].strip())
-            chapterDate = makeDate(date,self.dateformat)
+        for index, chapterDate in enumerate(storyData.findAll('span', {'class':'date'})):
+            dateString=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",chapterDate.contents[1].strip())
+            chapterDate = makeDate(dateString,self.dateformat)
            if oldestChapter == None or chapterDate < oldestChapter:
                oldestChapter = chapterDate
            if newestChapter == None or chapterDate > newestChapter:
                newestChapter = chapterDate
                self.newestChapterNum = index

+        # Date updated
        self.story.setMetadata("dateUpdated", newestChapter)
-        
-        pubdatetag = soup.find('span', {'class':'date_approved'})
+
+        # Date published
+        # falls back to oldest chapter date for stories that haven't been officially published yet
+        pubdatetag = storyContentBox.find('span', {'class':'date_approved'})
        if pubdatetag is None:
            self.story.setMetadata("datePublished", oldestChapter)            
        else:
@@ -225,47 +222,103 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
            datestripped=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",pubdateraw.strip())
            pubDate = makeDate(datestripped,self.dateformat)
            self.story.setMetadata("datePublished", pubDate)
-            
-        chars = soup.find("div", {"class":"inner_data"})
-        # fimfic stopped putting the char name on or around the char
-        # icon now for some reason.  Pull it from the image name with
-        # some heuristics.
-        for character in [character_icon["src"] for character_icon in chars.findAll("img", {"class":"character_icon"})]:
-            # //static.fimfiction.net/images/characters/twilight_sparkle.png
-            # 5th split /, remove last four, replace _, capitolize every word(title())
-            char = character.split('/')[5][:-4].replace('_',' ').title()
-            if char == 'Oc':
-                char = "OC"
-            if char == 'Cmc':
-                char = "Cutie Mark Crusaders"
-            self.story.addToList("characters", char)
-            
-        # extra site specific metadata
-        extralist = ["likes","dislikes","views","total_views","short_description"]
-        for metakey in extralist:
-            if metakey in storyMetadata:
-                value = storyMetadata[metakey]
-                if not isinstance(value,basestring):
-                    value = unicode(value)
-                self.story.setMetadata(metakey, value)

-        rawGroupList = soup.find('ul', {'id':'story_group_list'})
-        if rawGroupList is not None:
-            for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
+        # Characters
+        chars = storyContentBox.find("div", {"class":"extra_story_data"})
+        for character in chars.findAll("a", {"class":"character_icon"}):
+            self.story.addToList("characters", character['title'])
+
+        # Likes and dislikes
+        storyToolbar = soup.find('div', {'class':'story-toolbar'})
+        likes = storyToolbar.find('span', {'class':'likes'})
+        if not likes is None:
+            self.story.setMetadata("likes", stripHTML(likes))
+        dislikes = storyToolbar.find('span', {'class':'dislikes'})
+        if not dislikes is None:
+            self.story.setMetadata("dislikes", stripHTML(dislikes))
+
+        # Highest view for a chapter and total views
+        viewSpan = storyToolbar.find('span', {'title':re.compile(r'.*\btotal views\b.*')})
+        self.story.setMetadata("views", re.sub(r'[^0-9]', '', stripHTML(viewSpan)))
+        self.story.setMetadata("total_views", re.sub(r'[^0-9]', '', viewSpan['title']))
+
+        # Comment count
+        commentSpan = storyToolbar.find('span', {'title':re.compile(r'.*\bcomments\b.*')})
+        self.story.setMetadata("comment_count", re.sub(r'[^0-9]', '', stripHTML(commentSpan)))
+
+        # Short description
+        descriptionMeta = soup.find('meta', {'property':'og:description'})
+        self.story.setMetadata("short_description", stripHTML(descriptionMeta['content']))
+
+        #groups
+        if soup.find('button', {'id':'button-view-all-groups'}):
+            groupResponse = self._fetchUrl("http://www.fimfiction.net/ajax/groups/story_groups_list.php?story=%s" % (self.story.getMetadata("storyId")))
+            groupData = json.loads(groupResponse)
+            groupList = bs.BeautifulSoup(groupData["content"])
+        else:
+            groupList = soup.find('ul', {'id':'story-groups-list'})
+
+        if not (groupList == None):
+            for groupName in groupList.findAll('a'):
+                self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) 
                self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
-            
+
+        #sequels
+        sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels")
+        if not sequelStoryHeader == None:
+            sequelContainer = sequelStoryHeader.parent.parent
+            for sequel in sequelContainer.findAll('a', {'class':'story_link'}):
+                self.story.addToList("sequelsUrl", 'http://'+self.host+sequel["href"]) 
+                self.story.addToList("sequels", stripHTML(sequel).replace(',', ';'))
+                
+        #The link to the prequel is embedded in the description text, so erring
+        #on the side of caution and wrapping this whole thing in a try block.
+        #If anything goes wrong this probably wasn't a valid prequel link.
+        try:
+            description = soup.find('div', {'class':'description'})
+            firstHR = description.find("hr")
+            nextSib = firstHR.nextSibling
+            if "This story is a sequel to" in nextSib.string:
+                link = nextSib.nextSibling
+                if link.name == "a":
+                    self.story.setMetadata("prequelUrl", 'http://'+self.host+link["href"])
+                    self.story.setMetadata("prequel", stripHTML(link))
+        except:
+            pass
+        
    def hookForUpdates(self,chaptercount):
        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
            self.oldchapters = self.oldchapters[:self.newestChapterNum]
        return len(self.oldchapters)
-            
+
+    def do_fix_blockquotes(self,data):
+        if self.getConfig('fix_fimf_blockquotes'):
+            # <p class="double"><blockquote>
+            # </blockquote></p>
+            # include > in re groups so there's always something in the group.
+            data = re.sub(r'<p([^>]*>\s*)<blockquote([^>]*>)',r'<blockquote\2<p\1',data)
+            data = re.sub(r'</blockquote(>\s*)</p>',r'</p\1</blockquote>',data)
+        return data
        
    def getChapterText(self, url):
        logger.debug('Getting chapter text from: %s' % url)
-        
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
+
+        data = self._fetchUrl(url)
+
+        soup = bs.BeautifulSoup(data)
+        if not (soup.find('form', {'id' : 'password_form'}) == None):
+            if self.password:
+                params = {}
+                params['password'] = self.password
+                data = self._postUrl(url, params)
+            else:
+                print("Chapter %s needed password but no password was present" % url)
+
+        data = self.do_fix_blockquotes(data)
+
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
        if soup == None:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
        return self.utf8FromSoup(url,soup)
-        
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-

-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,11 +37,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)

-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
        self.password = ""
        self.is_adult=False
@@ -51,7 +46,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
        if 'storyInfo' in self.story.getMetadata('storyId'):
            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
        
-        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
        
@@ -68,8 +62,8 @@ class FineStoriesComAdapter(BaseSiteAdapter):
        return 'finestories.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/s/1234 http://"+cls.getSiteDomain()+"/s/1234:4010 http://"+cls.getSiteDomain()+"/library/storyInfo.php?id=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -75,7 +76,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
        return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']

    @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
        return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -57,7 +58,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
        return ['www.harrypotterfanfiction.com','harrypotterfanfiction.com']

    @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234"

    def getSiteURLPattern(self):
@@ -66,8 +66,8 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
        return 'www.henneth-annun.net'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/stories/chapter.cfm?stid=1234"

    def getSiteURLPattern(self):
        return "http://"+self.getSiteDomain()+"/stories/chapter(_view)?.cfm\?stid="+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class HLFictionNetAdapter(BaseSiteAdapter):
        return 'hlfiction.net'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -71,8 +72,8 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
        return 'www.hpfandom.net' # XXX

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/eff/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
        return 'www.hpfanficarchive.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/stories/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=")+r"\d+$"
@@ -128,23 +129,22 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
        # <span class="label">Rated:</span> NC-17<br /> etc
        labels = soup.findAll('span',{'class':'label'})
        for labelspan in labels:
-            value = labelspan.nextSibling
+            val = labelspan.nextSibling
+            value = unicode('')
+            while val and not defaultGetattr(val,'class') == 'label':
+                value += unicode(val)
+                val = val.nextSibling
            label = labelspan.string
+            #print("label:%s\nvalue:%s"%(label,value))

            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,value)

            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
+                self.story.setMetadata('rating', stripHTML(value))

            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
+                self.story.setMetadata('numWords', stripHTML(value))

            if 'Categories' in label:
                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
@@ -161,13 +161,18 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                for genre in genres:
                    self.story.addToList('genre',genre.string)

+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
            if 'Warnings' in label:
                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                for warning in warnings:
                    self.story.addToList('warnings',warning.string)

            if 'Completed' in label:
-                if 'Yes' in value:
+                if 'Yes' in stripHTML(value):
                    self.story.setMetadata('status', 'Completed')
                else:
                    self.story.setMetadata('status', 'In-Progress')
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class IkEternalNetAdapter(BaseSiteAdapter):
        return 'www.ik-eternal.net'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
        return 'imagine.e-fic.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -73,8 +73,8 @@ class InDeathNetAdapter(BaseSiteAdapter):


    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/blog/archive/123-story-in-death/"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/blog/archive/123-story-in-death/"

    def getSiteURLPattern(self):
        # http://www.indeath.net/blog/archive/169-ransom-in-death/
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -75,8 +76,8 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        return 'ksarchive.com' # XXX

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -67,8 +68,8 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
        return 'www.libraryofmoria.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/a/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/a/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/a/viewstory.php?sid=")+r"\d+$"
@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+class LiteroticaSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+
+        self.story.setMetadata('siteabbrev','litero')
+
+        # normalize to first chapter.  Not sure if they ever have more than 2 digits.
+        storyId = self.parsedUrl.path.split('/',)[2]
+        # replace later chapters with first chapter but don't remove numbers
+        # from the URL that disambiguate stories with the same title.
+        storyId = re.sub("-ch-?\d\d", "", storyId)
+        self.story.setMetadata('storyId', storyId)
+
+        ## accept m(mobile)url, but use www.
+        url = re.sub("^(www|german|spanish|french|dutch|italian|romanian|portuguese|other)\.i",
+                              "\1",
+                              url)
+
+        ## strip ?page=...
+        url = re.sub("\?page=.*$", "", url)
+
+        ## set url
+        self._setURL(url)
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = '%m/%d/%y'
+
+    @staticmethod
+    def getSiteDomain():
+        return 'literotica.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.literotica.com',
+                'www.i.literotica.com',
+                'german.literotica.com',
+                'german.i.literotica.com',
+                'spanish.literotica.com',
+                'spanish.i.literotica.com',
+                'french.literotica.com',
+                'french.i.literotica.com',
+                'dutch.literotica.com',
+                'dutch.i.literotica.com',
+                'italian.literotica.com',
+                'italian.i.literotica.com',
+                'romanian.literotica.com',
+                'romanian.i.literotica.com',
+                'portuguese.literotica.com',
+                'portuguese.i.literotica.com',
+                'other.literotica.com',
+                'other.i.literotica.com']
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title http://portuguese.literotica.com/s/story-title http://german.literotica.com/s/story-title"
+
+    def getSiteURLPattern(self):
+        return r"https?://(www|german|spanish|french|dutch|italian|romanian|portuguese|other)(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+        """
+        NOTE: Some stories can have versions, 
+              e.g. /my-story-ch-05-version-10
+        NOTE: If two stories share the same title, a running index is added,
+              e.g.: /my-story-ch-02-1
+        Strategy:
+            * Go to author's page, search for the current story link,
+            * If it's in a tr.root-story => One-part story
+                * , get metadata and be done
+            * If it's in a tr.sl => Chapter in series
+                * Search up from there until we find a tr.ser-ttl (this is the
+                story)
+                * Gather metadata
+                * Search down from there for all tr.sl until the next
+                tr.ser-ttl, foreach
+                    * Chapter link is there
+        """
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+
+        logger.debug("Chapter/Story URL: <%s> " % self.url)
+        try:
+            data1 = self._fetchUrl(self.url)
+            soup1 = bs.BeautifulSoup(data1)
+            #strip comments from soup
+            [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # author
+        a = soup1.find("span", "b-story-user-y")
+        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'][0])
+        authorurl = a.a['href']
+        if authorurl.startswith('//'):
+            authorurl = self.parsedUrl.scheme+':'+authorurl
+        self.story.setMetadata('authorUrl', authorurl)
+        self.story.setMetadata('author', a.text)
+
+        # get the author page
+        try:
+            dataAuth = self._fetchUrl(authorurl)
+            soupAuth = bs.BeautifulSoup(dataAuth)
+            #strip comments from soup
+            [comment.extract() for comment in soupAuth.findAll(text=lambda text:isinstance(text, bs.Comment))]
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(authorurl)
+            else:
+                raise e
+
+        ## Find link to url in author's page
+        ## site has started using //domain.name/asdf urls remove https?: from front
+        storyLink = soupAuth.find('a', href=self.url[self.url.index(':')+1:])
+
+        if storyLink is not None:
+            urlTr = storyLink.parent.parent
+            if urlTr['class'] == "sl":
+                isSingleStory = False
+            else:
+                isSingleStory = True
+        else:
+            raise exceptions.FailedToDownload("Couldn't find story <%s> on author's page <%s>" % (url, authorurl))
+
+        if isSingleStory:
+            self.story.setMetadata('title', storyLink.text)
+            self.story.setMetadata('description', urlTr.findAll("td")[1].text)
+            self.story.addToList('eroticatags', urlTr.findAll("td")[2].text)
+            date = urlTr.findAll('td')[-1].text
+            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
+            self.chapterUrls = [(storyLink.text, self.url)]
+        else:
+            seriesTr = urlTr.previousSibling
+            while seriesTr['class'] != 'ser-ttl':
+                seriesTr = seriesTr.previousSibling
+            m = re.match("^(?P<title>.*?):\s(?P<numChapters>\d+)\sPart\sSeries$", seriesTr.find("strong").text)
+            self.story.setMetadata('title', m.group('title'))
+
+            ## Walk the chapters
+            chapterTr = seriesTr.nextSibling
+            self.chapterUrls = []
+            dates = []
+            descriptions = []
+            while chapterTr is not None and chapterTr['class'] == 'sl':
+                descriptions.append(chapterTr.findAll("td")[1].text)
+                chapterLink = chapterTr.find("td", "fc").find("a")
+                self.chapterUrls.append((chapterLink.text, "http:" + chapterLink["href"]))
+                self.story.addToList('eroticatags', chapterTr.findAll("td")[2].text)
+                dates.append(makeDate(chapterTr.findAll('td')[-1].text, self.dateformat))
+                chapterTr = chapterTr.nextSibling
+
+            ## Set description to joint chapter descriptions
+            self.story.setMetadata('description', " / ".join(descriptions))
+
+            ## Set the oldest date as publication date, the newest as update date
+            dates.sort()
+            self.story.setMetadata('datePublished', dates[0])
+            self.story.setMetadata('dateUpdated', dates[-1])
+
+            # normalize on first chapter URL.
+            self._setURL(self.chapterUrls[0][1])
+
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+                
+        # set storyId to 'title-author' to avoid duplicates
+        # self.story.setMetadata('storyId', 
+        #     re.sub("[^a-z0-9]", "", self.story.getMetadata('title').lower())
+        #     + "-"
+        #     + re.sub("[^a-z0-9]", "", self.story.getMetadata('author').lower()))
+
+        return
+
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from <%s>' % url)
+        data1 = self._fetchUrl(url)
+        # brute force approach to replace the wrapping <p> tag.  If
+        # done by changing tag name, it causes problems with nested
+        # <p> tags.
+        data1 = data1.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>')
+        soup1 = bs.BeautifulSoup(data1)
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        story1 = soup1.find('div', 'b-story-body-x').div
+        #print("story1:%s"%story1)
+        # story1.name='div'
+        story1.append('<br />')
+        storytext = self.utf8FromSoup(url,story1)
+
+        # find num pages
+        pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0])
+        logger.debug("pages: "+str(pgs))
+
+        # get all the pages
+        for i in xrange(2, pgs+1):
+            try:
+                logger.debug("fetching page "+str(i))
+                time.sleep(0.5)
+                data2 = self._fetchUrl(url, {'page': i})
+                # brute force approach to replace the wrapping <p> tag.  If
+                # done by changing tag name, it causes problems with nested
+                # <p> tags.
+                data2 = data2.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>')
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+                story2 = soup2.find('div', 'b-story-body-x').div
+                # story2.name='div'
+                story2.append('<br />')
+                storytext += self.utf8FromSoup(url,story2)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        return storytext
+
+
+def getClass():
+    return LiteroticaSiteAdapter
+
+
+
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+from base_efiction_adapter import BaseEfictionAdapter
+
+class TheLOTRFanFictionSiteAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'lotrfanfiction.com'
+
+    @classmethod
+    def getSiteAbbrev(seluuf):
+        return 'lotrff'
+
+    @classmethod
+    def getDateFormat(self):
+        return "%d/%m/%y"
+
+def getClass():
+    return TheLOTRFanFictionSiteAdapter
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -67,8 +68,8 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
        return 'lumos.sycophanthex.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -56,8 +56,8 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
        return 'www.mediaminer.org'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+self.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+cls.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"

    def getSiteURLPattern(self):
        ##  http://www.mediaminer.org/fanfic/view_st.php/76882
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
        return 'merlinfic.dtwins.co.uk'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -71,8 +72,8 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
        return 'www.midnightwhispers.ca' # XXX

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -74,8 +75,8 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
        return ['fanfiction.mugglenet.com','fanfic.mugglenet.com']

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+r"fanfic(tion)?\.mugglenet\.com"+re.escape("/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -69,9 +70,9 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
        return ['www.national-library.net','national-library.net']

    @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
        # ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: 
-        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?storyid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -69,8 +70,8 @@ class NCISFicComAdapter(BaseSiteAdapter):
        return ['www.ncisfic.com','ncisfic.com']

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?storyid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
@@ -75,8 +75,8 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
        return ['www.ncisfiction.net','www.ncisfiction.com']

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/story.php?stid=01234 http://"+cls.getSiteDomain()+"/chapters.php?stid=1234"

    def getSiteURLPattern(self):
        return r'http://www\.ncisfiction\.(net|com)/(chapters|story)?.php\?stid=\d+'
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -64,8 +65,8 @@ class NetRaptorOrgAdapter(BaseSiteAdapter):
        return 'netraptor.org'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -75,8 +76,8 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
        return 'nfacommunity.com' # XXX

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -59,7 +60,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):

        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = " %m/%d/%y"
+        self.dateformat = " %d/%m/%y"
            
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
@@ -67,8 +68,8 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
        return 'nha.magical-worlds.us'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -86,6 +87,28 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
            else:
                raise e
            
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
            
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -67,8 +68,8 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
        return 'www.nickandgreg.net'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$"
@@ -0,0 +1,177 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return NocturnalLightNetAdapter
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class NocturnalLightNetAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'nln'
+    SITE_DOMAIN = 'nocturnal-light.net'
+    BASE_URL = 'http://' + SITE_DOMAIN + '/fanfiction/'
+    STORY_URL_TEMPLATE = BASE_URL + 'story/%s'
+    AUTHORS_URL_TEMPLATE = BASE_URL + 'authors/%s'
+
+    DATETIME_FORMAT = '%m-%d-%y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        url_tokens = self.parsedUrl.path.split('/')
+        story_id = url_tokens[url_tokens.index('story') + 1]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return NocturnalLightNetAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.STORY_URL_TEMPLATE[:-2]) + r'\d+.*$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if soup.title.string.strip() == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        # "storycontent" is found in a single-chapter story
+        author_anchor = soup.find('div', id=lambda id: id in ('main', 'storycontent')).h1.a
+        self.story.setMetadata('author', author_anchor.string)
+
+        url_tokens = author_anchor['href'].split('/')
+        author_id = url_tokens[url_tokens.index('authors')+1]
+        self.story.setMetadata('authorId', author_id)
+        self.story.setMetadata('authorUrl', self.AUTHORS_URL_TEMPLATE % author_id)
+
+        chapter_anchors = soup('a', href=lambda href: href and href.startswith('/fanfiction/story/'))
+        for chapter_anchor in chapter_anchors:
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((chapter_anchor.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+        for listbox in soup('div', {'class': 'listbox'}):
+            url_tokens = listbox.a['href'].split('/')
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if story_id == url_tokens[url_tokens.index('story')+1]:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = listbox.a.string
+        self.story.setMetadata('title', title)
+
+        # No chapter anchors found in the original story URL, so the story has
+        # only a single chapter.
+        if not chapter_anchors:
+            self.chapterUrls.append((title, self.url))
+
+        for b_tag in listbox('b'):
+            key = b_tag.string.strip(':')
+            try:
+                value = b_tag.nextSibling.string.replace('&bull;', '').strip(': ')
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                for sibling in _yield_next_siblings(b_tag):
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(author_url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+
+                # Pop last break line tag
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Category':
+                for sibling in b_tag.findNextSiblings(['a', 'b']):
+                    if sibling.name == 'b':
+                        break
+
+                    self.story.addToList('category', sibling.string)
+
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Chapters':
+                self.story.setMetadata('numChapters', int(value))
+
+                # Also parse reviews number which lies right after the chapters
+                # section
+                reviews_anchor = b_tag.findNextSibling('a')
+                reviews = reviews_anchor.string.split(' ')[1].strip('()')
+                self.story.setMetadata('reviews', reviews)
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Date Added':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Last Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Read':
+                self.story.setMetadata('readings', value.split()[0])
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        return self.utf8FromSoup(url, soup.find('div', id='storytext'))
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -67,8 +68,8 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
        return 'occlumency.sycophanthex.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -71,8 +72,8 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):


    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -192,7 +193,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
            if 'Summary' in label:
                ## Everything until the next span class='label'
                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
+                while value and not defaultGetattr(value,'class') == 'label':
                    svalue += str(value)
                    value = value.nextSibling
                self.setDescription(url,svalue)
@@ -66,8 +66,8 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
        return 'www.phoenixsong.net'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfiction/story/1234/"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/story/")+r"\d+/?$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -76,8 +77,8 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
        return 'pommedesang.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/sds/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+cls.getSiteDomain()+"/sds/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return r"http://"+self.getSiteDomain()+"/(efiction|sds)?/viewstory.php\?sid=\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -70,8 +71,8 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
        return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234 http://explicit."+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234 http://explicit."+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -73,12 +73,19 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
        return 'fanfiction.portkey.org' # XXX

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story/1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/story/1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/story/")+r"\d+(/\d+)?$"

+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
    ## Getting the chapter list and the meta data, plus 'is adult' checking.
    def extractChapterUrlsAndMetadata(self):

@@ -88,7 +95,6 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
        # portkey screws around with using a different URL to set the
        # cookie and it's a pain.  So... cheat!
        if self.is_adult or self.getConfig("is_adult"):
-            cookieproc = urllib2.HTTPCookieProcessor()
            cookie = cl.Cookie(version=0, name='verify17', value='1',
                               port=None, port_specified=False,
                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
@@ -99,9 +105,8 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
                               comment=None,
                               comment_url=None,
                               rest={'HttpOnly': None},
-                               rfc2109=False)
-            cookieproc.cookiejar.set_cookie(cookie)
-            self.opener = urllib2.build_opener(cookieproc)        
+                               rfc2109=False) 
+            self.cookiejar.set_cookie(cookie)

        try:
            data = self._fetchUrl(url)
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -56,7 +57,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
        return ['www.potionsandsnitches.net','potionsandsnitches.net']

    @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
@@ -163,12 +164,11 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                    self.story.setMetadata('status', 'In-Progress')

            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
+                # limit date values, there's some extra chars.
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y"))
            
            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y"))

        try:
            # Find Series name from series URL.
@@ -73,7 +73,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
        return 'www.potterfics.com'

    @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
        return "http://www.potterfics.com/historias/12345 http://www.potterfics.com/historias/12345/capitulo-1 "

    def getSiteURLPattern(self):
@@ -86,6 +86,40 @@ class PotterFicsComAdapter(BaseSiteAdapter):
            r"(?P<id>\d+)(/capitulo-(?P<ch>\d+))?/?$"
        return p

+    def needToLoginCheck(self, data):
+        # partials used to avoid having to figure out what was wrong
+        # with included utf8 higher chars.
+        if 'Para ver esta historia, por favor inicia tu sesi' in data \
+                or '<script>alert("El nombre de usuario o contrase' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self,url):
+        params = {}
+
+        if self.password:
+            params['login_usuario'] = self.username
+            params['login_password'] = self.password
+        else:
+            params['login_usuario'] = self.getConfig("username")
+            params['login_password'] = self.getConfig("password")
+        params['login_ck'] = '1'
+
+        loginUrl = 'http://www.potterfics.com/secciones/usuarios/login.php'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['login_usuario']))
+        d = self._postUrl(loginUrl,params)
+
+        #print("d:%s"%d)
+        if '<script>alert("El nombre de usuario o contrase' in d:
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['login_usuario']))
+            raise exceptions.FailedToLogin(url,params['login_usuario'])
+            return False
+        else:
+            return True
+
    def extractChapterUrlsAndMetadata(self):

        #this converts '/historias/12345' to 'http://www.potterfics.com/historias/12345'
@@ -123,8 +157,12 @@ class PotterFicsComAdapter(BaseSiteAdapter):

        #print data

-        #deal with adult content warnings - doesn't seem to apply to this site
-
+        #deal with adult content login
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url,usecache=False)
+            
        #set constant meta for this site:
        #Set Language = Spanish
        self.story.setMetadata('language', 'Spanish')
@@ -144,8 +182,8 @@ class PotterFicsComAdapter(BaseSiteAdapter):
        #within that, we want the second row, first cell
        cell = table.tr.findNextSibling('tr').td

-        #find first metadata block
-        mb = cell.div.findNextSibling('div')
+        #find first metadata block--isn't first if logged in
+        mb = cell.div.findNextSibling('div',{'align':'left'})
        #Get meta...
        self.story.setMetadata('title', stripHTML(mb.b))
        #strip out brackets on rating
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
        return 'fanfic.potterheadsanonymous.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -70,8 +71,8 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
        return ['www.pretendercentre.com','pretendercentre.com']

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/missingpieces/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class PsychFicComAdapter(BaseSiteAdapter):
        return 'www.psychfic.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class QafFicComAdapter(BaseSiteAdapter):
        return 'www.qaf-fic.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/atp/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=")+r"\d+$"
@@ -69,8 +69,8 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
        return 'www.restrictedsection.org'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?story=1234 http://"+self.getSiteDomain()+"/file.php?file=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/story.php?story=1234 http://"+cls.getSiteDomain()+"/file.php?file=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain())+r"/(?P<filestory>file|story).php\?(file|story)=(?P<id>\d+)$"
@@ -0,0 +1,342 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return SamAndJackNetAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sjn') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'samandjack.net' # XXX
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfics/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 10, 3 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=5" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+             
+        # Find authorid and URL from... author url.
+        # (fetch multiple authors)
+        alist = soup.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        for a in alist:
+            self.story.addToList('authorId',a['href'].split('=')[1])
+            self.story.addToList('authorUrl','http://'+self.host+'/fanfics/'+a['href'])
+            self.story.addToList('author',a.string)
+                             
+        # Reviews
+        reviewdata = soup.find('div', {'id' : 'sort'})
+        a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
+        self.story.setMetadata('reviews',stripHTML(a))              
+        
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfics/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+               
+        # <span class="label">Rated:</span> NC-17<br /> etc 
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string  
+            
+            if 'Summary' in label:
+                self.setDescription(url,value)                     
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                value=value.replace(' | ','')
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                         
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfics/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -70,8 +71,8 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
        return ['www.samdean.archive.nu','samdean.archive.nu']

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+r"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class ScarHeadNetAdapter(BaseSiteAdapter):
        return 'scarhead.net'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
        return 'www.scarvesandcoffee.net'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -15,6 +15,7 @@
 # limitations under the License.
 #

+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -75,8 +76,8 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
        return 'sg1-heliopolis.com'

    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+cls.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+cls.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return r"http://sg1-heliopolis.com/(archive|adult|atlantis)?/viewstory.php\?sid=\d+$"
--- a/Show More
+++ b/Show More