Fix for smarten_punctuation setting in pre-0.9.39 calibre.

Added tag FanFictionDownLoader-4.4.66 for changeset c32e4193a161
Added tag calibre-plugin-1.7.32 for changeset c32e4193a161
2026-09-15 11:36:30 +08:00 · 2013-07-24 13:02:14 -05:00 · 2013-07-23 15:19:01 -05:00 · 2013-07-23 15:18:48 -05:00 · 2013-07-23 15:18:37 -05:00 · 2013-07-23 15:17:16 -05:00
119 changed files with 3714 additions and 2297 deletions
@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-50
+version: 4-4-66
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -1,11 +1,10 @@
 #!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 # -*- coding: utf-8 -*-
 from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'

 # The class that all Interface Action plugin wrappers must inherit from
@@ -27,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
    description         = 'UI plugin to download FanFiction stories from various sites.'
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (1, 7, 17)
+    version             = (1, 7, 33)
    minimum_calibre_version = (0, 8, 57)

    #: This field defines the GUI plugin class that contains all the code
@@ -239,8 +239,8 @@ class SizePersistedDialog(QDialog):
            self.restoreGeometry(self.geom)

    def dialog_closing(self, result):
-        geom = bytearray(self.saveGeometry())
-        gprefs[self.unique_pref_name] = geom
+        self.geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = self.geom


 class ReadOnlyTableWidgetItem(QTableWidgetItem):
@@ -13,10 +13,11 @@ from collections import OrderedDict
 from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
-                      QDialogButtonBox )
+                      QDialogButtonBox, QGroupBox )

-from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
+from calibre.gui2 import dynamic, info_dialog
+from calibre.constants import numeric_version as calibre_version

 from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
@@ -177,6 +178,7 @@ class ConfigWidget(QWidget):
        prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
        prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
        prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
+        prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()

        if self.readinglist_tab:
            # lists
@@ -266,13 +268,16 @@ class BasicTab(QWidget):
        self.plugin_action = plugin_action
        QWidget.__init__(self)
        
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
+        topl = QVBoxLayout()
+        self.setLayout(topl)

        label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
        label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
+        topl.addWidget(label)
+
+        defs_gb = groupbox = QGroupBox("Defaults Options on Download")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)

        tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
        horz = QHBoxLayout()
@@ -317,7 +322,15 @@ class BasicTab(QWidget):
        self.updateepubcover.setChecked(prefs['updateepubcover'])
        self.l.addWidget(self.updateepubcover)

-        self.l.addSpacing(10)        
+        self.smarten_punctuation = QCheckBox('Smarten Punctuation (EPUB only)',self)
+        self.smarten_punctuation.setToolTip("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update.")
+        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
+        if calibre_version >= (0, 9, 39):
+            self.l.addWidget(self.smarten_punctuation)
+
+        cali_gb = groupbox = QGroupBox("Updating Calibre Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)

        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
@@ -334,14 +347,24 @@ class BasicTab(QWidget):
        self.keeptags.setChecked(prefs['keeptags'])
        self.l.addWidget(self.keeptags)

-        self.l.addSpacing(10)        
+        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
+        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
+        self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
+        self.l.addWidget(self.checkforseriesurlid)
+
+        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
+        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
+        self.l.addWidget(self.lookforurlinhtml)

        self.showmarked = QCheckBox("Show added/updated books when finished?",self)
        self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
        self.showmarked.setChecked(prefs['showmarked'])
        self.l.addWidget(self.showmarked)

-        self.l.addSpacing(10)        
+        gui_gb = groupbox = QGroupBox("GUI Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)

        self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
@@ -359,7 +382,9 @@ class BasicTab(QWidget):
        self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
        self.l.addWidget(self.adddialogstaysontop)

-        self.l.addSpacing(10)        
+        misc_gb = groupbox = QGroupBox("Misc Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)

        # this is a cheat to make it easier for users to realize there's a new include_images features.
        self.includeimages = QCheckBox("Include images in EPUBs?",self)
@@ -367,43 +392,43 @@ class BasicTab(QWidget):
        self.includeimages.setChecked(prefs['includeimages'])
        self.l.addWidget(self.includeimages)

-        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
-        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
-        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
-        self.l.addWidget(self.lookforurlinhtml)
-
-        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
-        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
-        self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
-        self.l.addWidget(self.checkforseriesurlid)
-
        self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
        self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
        self.injectseries.setChecked(prefs['injectseries'])
        self.l.addWidget(self.injectseries)

-        self.l.addSpacing(10)        
+        rej_gb = groupbox = QGroupBox("Reject List")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)

-        horz = QHBoxLayout()
-        
        self.rejectlist = QPushButton('Edit Reject URL List', self)
        self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
        self.rejectlist.clicked.connect(self.show_rejectlist)
-        horz.addWidget(self.rejectlist)
+        self.l.addWidget(self.rejectlist)
        
        self.reject_urls = QPushButton('Add Reject URLs', self)
        self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
        self.reject_urls.clicked.connect(self.add_reject_urls)
-        horz.addWidget(self.reject_urls)
+        self.l.addWidget(self.reject_urls)
        
        self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
        self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
        self.reject_reasons.clicked.connect(self.show_reject_reasons)
-        horz.addWidget(self.reject_reasons)
+        self.l.addWidget(self.reject_reasons)
+
+        topl.addWidget(defs_gb)
+
+        horz = QHBoxLayout()
+        topl.addLayout(horz)
+        horz.addWidget(cali_gb)
+        horz.addWidget(rej_gb)
        
-        self.l.addLayout(horz)
-        
-        self.l.insertStretch(-1)
+        horz = QHBoxLayout()
+        topl.addLayout(horz)
+        horz.addWidget(gui_gb)
+        horz.addWidget(misc_gb)
+
+        topl.insertStretch(-1)
        
    def set_collisions(self):
        prev=self.collision.currentText()
@@ -17,7 +17,8 @@ from PyQt4 import QtGui
 from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
-                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal)
+                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal,
+                      QGroupBox, QFrame)

 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
@@ -48,18 +49,21 @@ anthology_collision_order=[UPDATE,
                           UPDATEALWAYS,
                           OVERWRITEALWAYS]

+gpstyle='QGroupBox {border:0; padding-top:10px; padding-bottom:0px; margin-bottom:0px;}' #  background-color:red;
+
 class RejectUrlEntry:

    matchpat=re.compile(r"^(?P<url>[^,]+)(,(?P<fullnote>(((?P<title>.+) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
    
    def __init__(self,url_or_line,note=None,title=None,auth=None,
-                 addreasontext=None,fromline=False):
-        
+                 addreasontext=None,fromline=False,book_id=None):
+
        self.url=url_or_line
        self.note=note
        self.title=title
        self.auth=auth
        self.valid=False
+        self.book_id=book_id

        if fromline:
            mc = re.match(self.matchpat,url_or_line)
@@ -199,6 +203,24 @@ class AddNewDialog(SizePersistedDialog):
        # elements to show again when doing *update* merge
        self.mergeupdateshow = []

+        self.groupbox = QGroupBox("Show Download Options")
+        self.groupbox.setCheckable(True)
+        self.groupbox.setChecked(False)
+        self.groupbox.setFlat(True)
+        print("style:%s"%self.groupbox.styleSheet())
+        self.groupbox.setStyleSheet(gpstyle);
+
+        self.gbf = QFrame()
+        self.gbl = QVBoxLayout()
+        self.gbl.addWidget(self.gbf)
+        self.groupbox.setLayout(self.gbl)
+        self.gbl = QVBoxLayout()
+        self.gbf.setLayout(self.gbl)
+        self.l.addWidget(self.groupbox)
+
+        self.gbf.setVisible(False)
+        self.groupbox.toggled.connect(self.gbf.setVisible)
+        
        horz = QHBoxLayout()
        label = QLabel('Output &Format:')
        self.mergehide.append(label)
@@ -214,7 +236,7 @@ class AddNewDialog(SizePersistedDialog):
        horz.addWidget(label)
        label.setBuddy(self.fileform)
        horz.addWidget(self.fileform)
-        self.l.addLayout(horz)
+        self.gbl.addLayout(horz)
        self.mergehide.append(self.fileform)

        horz = QHBoxLayout()
@@ -229,7 +251,7 @@ class AddNewDialog(SizePersistedDialog):
            self.collision.setCurrentIndex(i)
        self.collisionlabel.setBuddy(self.collision)
        horz.addWidget(self.collision)
-        self.l.addLayout(horz)
+        self.gbl.addLayout(horz)
        self.mergehide.append(self.collisionlabel)
        self.mergehide.append(self.collision)
        self.mergeupdateshow.append(self.collisionlabel)
@@ -249,7 +271,7 @@ class AddNewDialog(SizePersistedDialog):
        horz.addWidget(self.updateepubcover)
        self.mergehide.append(self.updateepubcover)
        
-        self.l.addLayout(horz)
+        self.gbl.addLayout(horz)

        self.button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        self.button_box.accepted.connect(self.ok_clicked)
@@ -258,8 +280,8 @@ class AddNewDialog(SizePersistedDialog):

    # invoke the 
    def ok_clicked(self):
+        self.dialog_closing(None) # save persistent size.
        self.hide()
-        print("ok_clicked called")
        self.go_signal.emit( self.get_ffdl_options(),
                             self.get_urlstext(),
                             self.merge,
@@ -287,6 +309,8 @@ class AddNewDialog(SizePersistedDialog):
        self.newmerge = newmerge
        self.extraoptions = extraoptions
        self.extrapayload = extrapayload
+
+        self.groupbox.setVisible(not(self.merge and self.newmerge))
        
        if self.merge:
            self.toplabel.setText('Story URL(s) for anthology, one per line:')
@@ -359,6 +383,7 @@ class AddNewDialog(SizePersistedDialog):
            'collision': unicode(self.collision.currentText()),
            'updatemeta': self.updatemeta.isChecked(),
            'updateepubcover': self.updateepubcover.isChecked(),
+            'smarten_punctuation':self.prefs['smarten_punctuation']
                }
        
        if self.merge:
@@ -594,7 +619,8 @@ class UpdateExistingDialog(SizePersistedDialog):
    def __init__(self, gui, header, prefs, icon, books,
                 save_size_name='fanfictiondownloader_plugin:update list dialog'):
        SizePersistedDialog.__init__(self, gui, save_size_name)
-        
+
+        self.prefs = prefs
        self.setWindowTitle(header)
        self.setWindowIcon(icon)
        
@@ -624,8 +650,25 @@ class UpdateExistingDialog(SizePersistedDialog):

        options_layout = QHBoxLayout()

+        groupbox = QGroupBox("Show Download Options")
+        groupbox.setCheckable(True)
+        groupbox.setChecked(False)
+        groupbox.setFlat(True)
+        groupbox.setStyleSheet(gpstyle)
+
+        gbf = QFrame()
+        gbl = QVBoxLayout()
+        gbl.addWidget(gbf)
+        groupbox.setLayout(gbl)
+        gbl = QHBoxLayout()
+        gbf.setLayout(gbl)
+        options_layout.addWidget(groupbox)
+
+        gbf.setVisible(False)
+        groupbox.toggled.connect(gbf.setVisible)
+        
        label = QLabel('Output &Format:')
-        options_layout.addWidget(label)
+        gbl.addWidget(label)
        self.fileform = QComboBox(self)
        self.fileform.addItem('epub')
        self.fileform.addItem('mobi')
@@ -635,10 +678,10 @@ class UpdateExistingDialog(SizePersistedDialog):
        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
        self.fileform.activated.connect(self.set_collisions)
        label.setBuddy(self.fileform)
-        options_layout.addWidget(self.fileform)
+        gbl.addWidget(self.fileform)
        
        label = QLabel('Update Mode:')
-        options_layout.addWidget(label)
+        gbl.addWidget(label)
        self.collision = QComboBox(self)
        self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
        # add collision options
@@ -647,17 +690,19 @@ class UpdateExistingDialog(SizePersistedDialog):
        if i > -1:
            self.collision.setCurrentIndex(i)
        label.setBuddy(self.collision)
-        options_layout.addWidget(self.collision)
+        gbl.addWidget(self.collision)

        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
        self.updatemeta.setChecked(prefs['updatemeta'])
-        options_layout.addWidget(self.updatemeta)
+        gbl.addWidget(self.updatemeta)
                
        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
        self.updateepubcover.setChecked(prefs['updateepubcover'])
-        options_layout.addWidget(self.updateepubcover)
+        gbl.addWidget(self.updateepubcover)
+
+
        
        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        button_box.accepted.connect(self.accept)
@@ -693,6 +738,7 @@ class UpdateExistingDialog(SizePersistedDialog):
            'collision': unicode(self.collision.currentText()),
            'updatemeta': self.updatemeta.isChecked(),
            'updateepubcover': self.updateepubcover.isChecked(),
+            'smarten_punctuation':self.prefs['smarten_punctuation']
            }

 class StoryListTableWidget(QTableWidget):
@@ -828,7 +874,9 @@ class RejectListTableWidget(QTableWidget):

    def populate_table_row(self, row, rej):

-        self.setItem(row, 0, ReadOnlyTableWidgetItem(rej.url))
+        url_cell = ReadOnlyTableWidgetItem(rej.url)
+        url_cell.setData(Qt.UserRole, QVariant(rej.book_id))
+        self.setItem(row, 0, url_cell)
        self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
        self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
        
@@ -950,10 +998,19 @@ class RejectListDialog(SizePersistedDialog):
        rejectrows = []
        for row in range(self.rejects_table.rowCount()):
            url = unicode(self.rejects_table.item(row, 0).text()).strip()
+            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
            title = unicode(self.rejects_table.item(row, 1).text()).strip()
            auth = unicode(self.rejects_table.item(row, 2).text()).strip()
            note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
-            rejectrows.append(RejectUrlEntry(url,note,title,auth,self.get_reason_text()))
+            rejectrows.append(RejectUrlEntry(url,note,title,auth,self.get_reason_text(),book_id=book_id))
+        return rejectrows
+
+    def get_reject_list_ids(self):
+        rejectrows = []
+        for row in range(self.rejects_table.rowCount()):
+            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            if book_id:
+                rejectrows.append(book_id)
        return rejectrows

    def get_reason_text(self):
@@ -12,8 +12,10 @@ from StringIO import StringIO
 from functools import partial
 from datetime import datetime
 from string import Template
+import urllib
+import email

-from PyQt4.Qt import (QApplication, QMenu, QToolButton)
+from PyQt4.Qt import (QApplication, QMenu, QToolButton, QTimer)

 from PyQt4.Qt import QPixmap, Qt
 from PyQt4.QtCore import QBuffer
@@ -38,7 +40,7 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin

 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page, get_urls_from_html, get_urls_from_text

 from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config, get_ffdl_personalini)
 from calibre_plugins.fanfictiondownloader_plugin.config import (permitted_values, rejecturllist)
@@ -140,10 +142,76 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            file_path = os.path.join(calibre_config_dir,
                                     *("plugins/fanfictiondownloader_macmenuhack.txt".split('/')))
            file_path = os.path.abspath(file_path)
-            print("macmenuhack file_path:%s"%file_path)
+            print("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
            self.macmenuhack = os.access(file_path, os.F_OK)
            return self.macmenuhack

+    accepts_drops = True
+
+    def accept_enter_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library") or \
+                mime_data.hasFormat("text/plain") or \
+                mime_data.hasFormat("text/uri-list"):
+            return True
+                
+        return False
+
+    def accept_drag_move_event(self, event, mime_data):
+        return self.accept_enter_event(event, mime_data)
+
+    def drop_event(self, event, mime_data):
+
+        dropped_ids=None
+        urllist=[]
+        
+        mime = 'application/calibre+from_library'
+        if mime_data.hasFormat(mime):
+            dropped_ids = tuple(map(int, str(mime_data.data(mime)).split()))
+        
+        mimetype='text/uri-list'
+        filelist="%s"%event.mimeData().data(mimetype)
+        if filelist:
+            for f in filelist.splitlines():
+                #print("filename:%s"%f)
+                if f.endswith(".eml"):
+                    fhandle = urllib.urlopen(f)
+                    msg = email.message_from_file(fhandle)
+                    if msg.is_multipart():
+                        for part in msg.walk():
+                            #print("part type:%s"%part.get_content_type())
+                            if part.get_content_type() == "text/html":
+                                #print("URL list:%s"%get_urls_from_data(part.get_payload(decode=True)))
+                                urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
+                            if part.get_content_type() == "text/plain":
+                                #print("part content:text/plain")
+                                #print("part content:%s"%part.get_payload(decode=True))
+                                urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
+                    else:
+                        urllist.extend(get_urls_from_text("%s"%msg))
+                else:
+                    urllist.extend(get_urls_from_text(f))
+        else:
+            mimetype='text/plain'
+            if mime_data.hasFormat(mimetype):
+                #print("text/plain:%s"%event.mimeData().data(mimetype))
+                urllist.extend(get_urls_from_text(event.mimeData().data(mimetype)))
+
+        #print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
+        if urllist or dropped_ids:
+            QTimer.singleShot(1, partial(self.do_drop,
+                                         dropped_ids=dropped_ids,
+                                         urllist=urllist))
+            return True
+        
+        return False
+
+    def do_drop(self,dropped_ids=None,urllist=None):
+        # shouldn't ever be both.
+        if dropped_ids:
+            self.update_dialog(dropped_ids)
+        elif urllist:
+            self.add_dialog("\n".join(urllist))
+    
    def about_to_show_menu(self):
        self.rebuild_menus()

@@ -408,12 +476,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            
    def reject_list_urls_finish(self, book_list):

-        # construct reject list of tuples:
-        # (calibre_id, url, "title, authors", old reject note).
+        # construct reject list of objects
        reject_list = [ RejectUrlEntry(x['url'],
                                       x['oldrejnote'],
                                       x['title'],
-                                       ', '.join(x['author']))
+                                       ', '.join(x['author']),
+                                       book_id=x['calibre_id'])
                        for x in book_list if x['good'] ]
        if reject_list:
            d = RejectListDialog(self.gui,reject_list,
@@ -426,7 +494,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            rejecturllist.add(d.get_reject_list())

            if d.get_deletebooks():
-                self.gui.iactions['Remove Books'].delete_books()
+                self.gui.iactions['Remove Books'].do_library_delete(d.get_reject_list_ids())
            
        else:
            message="<p>Rejecting FFDL URLs: None of the books selected have FanFiction URLs.</p><p>Proceed to Remove?</p>"
@@ -554,19 +622,21 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        # Now that we've 
        self.prep_downloads( options, update_books, merge=True )
        
-    def update_dialog(self):
+    def update_dialog(self, id_list=None):
        if not self.is_library_view():
            self.gui.status_bar.show_message(_('Cannot Update Books from Device View'), 3000)
            return
+
+        if not id_list:
+            id_list = self.gui.library_view.get_selected_ids();
        
-        if len(self.gui.library_view.get_selected_ids()) == 0:
+        if len(id_list) == 0:
            self.gui.status_bar.show_message(_('No Selected Books to Update'), 3000)
            return
        #print("update_dialog()")
        
        db = self.gui.current_db
-        book_list = map( partial(self.make_book_id_only), self.gui.library_view.get_selected_ids() )
-        #book_ids = self.gui.library_view.get_selected_ids()
+        book_list = map( self.make_book_id_only, id_list )

        for j, book in enumerate(book_list):
            book['listorder'] = j
@@ -661,6 +731,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

        url = book['url']
        print("url:%s"%url)
+        mi = None

        if not merge: # skip reject list when merging.
            if rejecturllist.check(url):
@@ -845,6 +916,42 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                    book['icon'] = 'edit-redo.png'
                    book['status'] = 'Update'
    
+                if book_id and mi: # book_id and mi only set if matched by title/author.
+                    liburl = self.get_story_url(db,book_id)
+                    if book['url'] != liburl:
+                        if collision in (OVERWRITE,OVERWRITEALWAYS):
+                            updat="overwrit"
+                        else:
+                            updat="updat"
+                        if not question_dialog(self.gui, 'Change Story URL?',
+                                               '<h3>Change Story URL?</h3>'+
+                                               '<p><b>%s</b> by <b>%s</b> is already in your library with a different source URL:</p>'%
+                                                   (mi.title,', '.join(mi.author))+
+                                               '<p>In library: <a href="%(liburl)s">%(liburl)s</a></p><p>New URL: <a href="%(newurl)s">%(newurl)s</a></p>'%
+                                                   {'liburl':liburl,'newurl':book['url']}+
+                                               "<p>Click '<b>Yes</b>' to %se book with new URL.</p>"%updat+
+                                               "<p>Click '<b>No</b>' to skip %sing this book.</p>"%updat,
+                                               show_copy_button=False):
+                            if question_dialog(self.gui, 'Download as New Book?',
+                                               '<h3>Download as New Book?</h3>'+
+                                               '<p><b>%s</b> by <b>%s</b> is already in your library with a different source URL.</p>'%
+                                               (mi.title,', '.join(mi.author))+
+                                               '<p>You chose not to update the existing book.  Do you want to add a new book for this URL?</p>'+
+                                               '<p>New URL: <a href="%(newurl)s">%(newurl)s</a></p>'%
+                                               {'newurl':book['url']}+
+                                               "<p>Click '<b>Yes</b>' to a new book with new URL.</p>"+
+                                               "<p>Click '<b>No</b>' to skip URL.</p>",
+                                               show_copy_button=False):
+                                book_id = None
+                                mi = None
+                                book['calibre_id'] = None
+                            else:
+                                book['comment'] = "Update declined by user due to differing story URL(%s)"%liburl
+                                book['good']=False
+                                book['icon']='rotate-right.png'
+                                book['status'] = 'Different URL'
+                                return
+        
            if book_id != None and collision != ADDNEW:
                if collision in (CALIBREONLY):
                    book['comment'] = 'Metadata collected.'
@@ -889,7 +996,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                    db.copy_format_to(book_id,fileform,tmp,index_is_id=True)
                    print("existing epub tmp:"+tmp.name)
                    book['epub_for_update'] = tmp.name
-    
+
            if book_id != None and prefs['injectseries']:
                mi = db.get_metadata(book_id,index_is_id=True)
                if not book['series'] and mi.series != None:
@@ -1159,7 +1266,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if 'mergebook' in options:
            existingbook = options['mergebook']
        #print("existingbook:\n%s"%existingbook)
-        mergebook = self.merge_meta_books(existingbook,good_list)
+        mergebook = self.merge_meta_books(existingbook,good_list,options['fileform'])

        if 'mergebook' in options:
            mergebook['calibre_id'] = options['mergebook']['calibre_id']
@@ -1281,16 +1388,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
            epubmi = get_metadata(existingepub,'EPUB')
            if epubmi.cover_data[1] is not None:
-                db.set_cover(book_id, epubmi.cover_data[1])
-
-        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
-        if 'authorUrl' in book['all_metadata']:
-            authurls = book['all_metadata']['authorUrl'].split(", ")
-            for i, auth in enumerate(book['author']):
-                #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                autid=db.get_author_id(auth)
-                db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
+                try:
+                    db.set_cover(book_id, epubmi.cover_data[1])
+                except:
+                    print("Failed to set_cover, skipping")

        # implement 'newonly' flags here by setting to the current
        # value again.
@@ -1369,7 +1470,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

                    if flag == 'r' or book['added']: # flag 'n' isn't actually needed--*always* set if configured and new book.
                        if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
-                            val = unicode(book['all_metadata'][meta]).replace(",","")
+                            if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
+                                # re-split list, strip commas, convert to floats, sum up.
+                                val = sum([ float(x.replace(",","")) for x in book['all_metadata'][meta].split(", ") ])
+                            else:
+                                val = unicode(book['all_metadata'][meta]).replace(",","")
                        else:
                            val = book['all_metadata'][meta]
                        db.set_custom(book_id, val, label=label, commit=False)
@@ -1390,6 +1495,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                            
                        db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)

+        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
+        # Moved down so author's already in the DB.
+        if 'authorUrl' in book['all_metadata']:
+            authurls = book['all_metadata']['authorUrl'].split(", ")
+            for i, auth in enumerate(book['author']):
+                #print("===Update author url for %s to %s"%(auth,authurls[i]))
+                autid=db.get_author_id(auth)
+                db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
+
        db.commit()

        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
@@ -1533,7 +1648,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
    def convert_url_to_book(self, url):
        book = self.make_book()
        # look here for [\d,\d] at end of url, and remove?
-        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>,)?(?P<end>\d+)?\])?$",url)
+        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>[,-])?(?P<end>\d+)?\])?$",url)
        #print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
        url = mc.group('url')
        book['begin'] = mc.group('begin')
@@ -1623,7 +1738,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                    # identifiers have :->| in uri.
                    # print("uri from ident uri:%s"%identifiers['uri'].replace('|',':'))
                    return identifiers['uri'].replace('|',':')
-            elif path.lower().endswith('.epub'):
+            elif path and path.lower().endswith('.epub'):
                existingepub = path
                
            ## only epub has URL in it--at least where I can easily find it.
@@ -1642,12 +1757,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
    def is_good_downloader_url(self,url):
        return adapters.getNormalStoryURL(url)

-    def merge_meta_books(self,existingbook,book_list):
+    def merge_meta_books(self,existingbook,book_list,fileform):
        book = self.make_book()
        book['author'] = []
        book['tags'] = []
        book['url'] = ''
        book['all_metadata'] = {}
+        book['anthology_meta_list'] = {}
        book['comment'] = ''
        book['added'] = True
        book['good'] = True
@@ -1714,12 +1830,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        book['all_metadata'][k]=book['all_metadata'][k]+"\n\n"+v
                    else:
                        book['all_metadata'][k]=book['all_metadata'][k]+", "+v
+                        # flag psuedo list element.  Used so numeric
+                        # cust cols can convert back to numbers and
+                        # add.
+                        book['anthology_meta_list'][k]=True
    
        if existingbook:
            book['title'] = deftitle = existingbook['title']
            book['comments'] = existingbook['comments']
        else:
-            book['title'] = deftitle = book_list[0]['title']+" Anthology"
+            book['title'] = deftitle = book_list[0]['title']
            book['comments'] = "Anthology containing:\n" + \
                "\n".join([ "%s by %s"%(b['title'],', '.join(b['author'])) for b in book_list ])
            # book['all_metadata']['description']
@@ -1727,12 +1847,22 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            # if all same series, use series for name.  But only if all and not previous named
            if len(serieslist) == len(book_list):
                series = serieslist[0]
-                book['title'] = series+" Anthology"
+                book['title'] = series
                for sr in serieslist:
                    if series != sr:
                        book['title'] = deftitle;
                        break
            
+            configuration = get_ffdl_config(book['url'],fileform)
+            print("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
+            if configuration.getConfig('anthology_title_pattern'):
+                tmplt = Template(configuration.getConfig('anthology_title_pattern'))
+                book['title'] = tmplt.safe_substitute({'title':book['title']}).encode('utf8')
+            else:
+                # No setting, do fall back default.  Shouldn't happen,
+                # should always have a version in defaults.
+                book['title'] = book['title']+" Anthology"
+        
        book['all_metadata']['title'] = book['title'] # because custom columns are set from all_metadata
        book['all_metadata']['author'] = ", ".join(book['author'])
        book['author_sort']=book['author']
@@ -1741,6 +1871,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                book['tags'].remove(v)
        book['tags'].append('Anthology')
        book['all_metadata']['anthology'] = "true"
+        
        return book

 def split_text_to_urls(urls):
@@ -14,6 +14,7 @@ from StringIO import StringIO

 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
+from calibre.constants import numeric_version as calibre_version

 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
@@ -49,7 +50,7 @@ def do_download_worker(book_list, options,
            args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
                    'do_download_for_worker',
                    (book,options)]
-            job = ParallelJob('arbitrary',
+            job = ParallelJob('arbitrary_n',
                              "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
                              done=None,
                              args=args)
@@ -84,7 +85,7 @@ def do_download_worker(book_list, options,
        book_id = job._book['calibre_id']
        #title = job._title
        count = count + 1
-        notification(float(count)/total, 'Downloaded Story')
+        notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
        # Add this job's output to the current log
        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
        print(job.details)
@@ -106,7 +107,7 @@ def do_download_worker(book_list, options,
    # return the book list as the job result
    return book_list

-def do_download_for_worker(book,options):
+def do_download_for_worker(book,options,notification=lambda x,y:x):
    '''
    Child job, to extract isbn from formats for this specific book,
    when run as a worker job
@@ -206,6 +207,23 @@ def do_download_for_worker(book,options):
            book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
                (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
        
+        if options['smarten_punctuation'] and options['fileform'] == "epub" \
+                and calibre_version >= (0, 9, 39):
+            # do smarten_punctuation from calibre's polish feature
+            from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+            from calibre.utils.logging import Log
+            from collections import namedtuple
+        
+            data = {'smarten_punctuation':True}
+            opts = ALL_OPTS.copy()
+            opts.update(data)
+            O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+            opts = O(**opts)
+            
+            log = Log(level=Log.DEBUG)
+            # report = []
+            polish({outfile:outfile}, opts, log, print) # report.append
+        
    except NotGoingToDownload as d:
        book['good']=False
        book['comment']=unicode(d)
@@ -218,6 +236,6 @@ def do_download_for_worker(book,options):
        book['status'] = 'Error'
        print("Exception: %s:%s"%(book,unicode(e)))
        traceback.print_exc()
-
+        
    #time.sleep(10)
    return book
@@ -41,6 +41,7 @@ default_prefs['includeimages'] = False
 default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
 default_prefs['injectseries'] = False
+default_prefs['smarten_punctuation'] = False

 default_prefs['send_lists'] = ''
 default_prefs['read_lists'] = ''
@@ -61,6 +62,13 @@ default_prefs['allow_custcol_from_ini'] = True

 default_prefs['std_cols_newonly'] = {}

+# This is where all preferences for this plugin *were* stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
 def set_library_config(library_config,db):
    db.prefs.set_namespaced(PREFS_NAMESPACE,
                            PREFS_KEY_SETTINGS,
@@ -74,7 +82,7 @@ def get_library_config(db):
    if library_id in old_prefs:
        #print("get prefs from old_prefs")
        library_config = old_prefs[library_id]
-        set_library_config(library_config)
+        set_library_config(library_config,db)
        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
        del old_prefs[library_id]

@@ -84,13 +92,6 @@ def get_library_config(db):
                                                 copy.deepcopy(default_prefs))
    return library_config

-# This is where all preferences for this plugin *were* stored
-# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
-# in a global namespace, so make it as unique as possible.
-# You should always prefix your config file name with plugins/,
-# so as to ensure you dont accidentally clobber a calibre config file
-old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
 # fake out so I don't have to change the prefs calls anywhere.  The
 # Java programmer in me is offended by op-overloading, but it's very
 # tidy.
@@ -1,4 +1,4 @@
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -133,6 +133,9 @@ zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 ## zip_filename.
 allow_unsafe_filename: false

+## The regex pattern of 'unsafe' filename chars for above.
+#output_filename_safepattern:[^a-zA-Z0-9_\. \[\]\(\)&'-]+
+
 ## entries to make epub subjects and calibre tags
 ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
 include_subject_tags: extratags, genre, category, characters, ships, lastupdate, status
@@ -169,6 +172,13 @@ extratags: FanFiction
 ## *Five* part lines.  Effect only when trailing conditional key=>regexp matches
 ## metakey[,metakey]=>pattern=>replacement[&&conditionalkey=>regexp]
 ## Note that if metakey == conditionalkey the conditional is ignored.
+## You can use \s in the replacement to add explicit spaces.  (The config parser
+## tends to discard trailing spaces.)
+## replace_metadata <entry>_LIST options: FFDL replace_metadata lines
+## operate on individual list items for list entries. But if you
+## want to do a replacement on the joined string for the whole list,
+## you can by using <entry>_LIST. Example, if you added
+## calibre_author: calibre_author_LIST=>^(.{,100}).*$=>\1
 #replace_metadata:
 # genre,category=>Sci-Fi=>SF
 # Puella Magi Madoka Magica.* => Madoka
@@ -189,6 +199,10 @@ extratags: FanFiction
 ## summary.
 keep_summary_html:true

+## If set true, any style attributes on tags in the story HTML will be
+## kept.  Useful for keeping extra colors & formatting from original.
+#keep_style_attr: false
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -227,6 +241,24 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## "The Beginning" => "1. The Beginning" 
 chapter_title_add_pattern:${index}. ${title}

+## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
+## on '/', so use replace_metadata to change separator first if
+## needed. Something like: ships=>[ ]*(/|&amp;|&)[ ]*=>/ You can use
+## ships_LIST to change the / back to something else if you want.
+sort_ships:false
+
+## join_string_<entry> options -- FFDL list entries are comma
+## separated by default. You can use this to change that. For example,
+## if you want authors separated with ' & ' instead, use
+## join_string_calibre_author:\s&\s. (\s == space)
+#join_string_author:,\s
+
+## keep_in_order_<entry> options: FFDL sorts list entries by default
+## (except for author/authorUrl/authorId). But if you want to use an
+## extra entry derived from author, it ends up sorted. For example, if
+## you added calibre_author: keep_in_order_calibre_author:true
+#keep_in_order_author:true
+

 ## Each output format has a section that overrides [defaults]
 [html]
@@ -273,7 +305,7 @@ output_css:

 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,seriesUrl,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description

 ## Width to word wrap text output.  0 indicates no wrapping.
 wrap_width: 78
@@ -507,6 +539,15 @@ extraships:Severus Snape/Hermione Granger
 #username:YourName
 #password:yourpassword

+[asr3.slashzone.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Sentinel
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [bloodties-fans.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Blood Ties
@@ -571,6 +612,11 @@ extraships:Spike/Buffy
 #username:YourName
 #password:yourpassword

+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [dramione.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -602,7 +648,7 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## composite metadata entries.  dramione.org, for example, adds
 ## 'cliches' and then defines as the composite of hermiones,dracos in
 ## include_in_cliches.
-extra_valid_entries:themes,hermiones,dracos,timeline,cliches
+extra_valid_entries:themes,hermiones,dracos,timeline,cliches,read,reviews
 include_in_cliches:hermiones,dracos

 ## For another example, you could, by uncommenting this line, include
@@ -647,6 +693,11 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter

+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -654,11 +705,22 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword

-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+[fanfic.potterheadsanonymous.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [fanfiction.portkey.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -702,6 +764,10 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:read,reviews
+
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
@@ -762,6 +828,8 @@ extracategories:West Wing
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS

+[netraptor.org]
+
 [nfacommunity.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
@@ -844,6 +912,22 @@ extracategories:Supernatural
 extracharacters:Sam,Dean
 extraships:Sam/Dean

+[scarhead.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -918,6 +1002,15 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true

+[tokra.fandomnet.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: SG-1
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.adastrafanfic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1148,6 +1241,11 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS

+[www.nickandgreg.net]
+## Site dedicated to these categories/characters/ships
+extracategories:CSI
+extraships:Nick Stokes/Greg Sanders
+
 [www.phoenixsong.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1160,6 +1258,12 @@ extraships:Harry Potter/Ginny Weasley
 #username:YourName
 #password:yourpassword

+## phoenixsong.net, oddly, can have high rated chapters (login
+## required) in the middle of a lower rated story.  Use this to force
+## FFDL to always login to phoenixsong.net so those stories download
+## correctly.
+#force_login:true
+
 [www.potionsandsnitches.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1210,6 +1314,17 @@ extracharacters:Kurt Hummel,Blaine Anderson
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+[www.simplyundeniable.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.sinful-desire.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Supernatural
@@ -1356,3 +1471,48 @@ extracategories:Stargate: Atlantis
 ## extratags params in all other sections.  Only commandline options
 ## beat overrides.
 #extratags:fanficdownloader
+
+
+[teststory:defaults]
+valid_entries:title,author_list,authorId_list,authorUrl_list,storyUrl,
+ datePublished,dateUpdated,numWords,status,language,series,seriesUrl,
+ rating,category_list,genre_list,warnings_list,characters_list,ships_list,
+ description,site,extratags
+
+# {{storyId}} is a special case--it's the only one that works.
+title:Test Story Title {{storyId}}
+author_list:Test Author aa
+authorId_list:1
+authorUrl_list:http://test1.com?authid=1
+storyUrl:http://test1.com?sid={{storyId}}
+datePublished:1975-03-15
+dateUpdated:1975-04-15
+numWords:123,456
+status:In-Progress
+language:English
+
+chaptertitles:Prologue
+
+## Add additional sections with different numbers to get different
+## parameters for different story urls.
+## test1.com?sid=1000
+[teststory:1000]
+# note the leading commas when doing add_to_ with valid_entries and *_list
+add_to_valid_entries:,favs
+title:Testing New Feature {{storyId}}
+author_list:Bob Smith
+authorId_list:45
+authorUrl_list:http://test1.com?authid=45
+datePublished:2013-03-15
+dateUpdated:2013-04-15
+numWords:1456
+favs:56
+series:The Great Test [4]
+seriesUrl:http://test1.com?seriesid=1
+rating:Tweenie
+category_list:Harry Potter,Furbie,Crossover,Puella Magi Madoka Magica/魔法少女まどか★マギカ,Magical Girl Lyrical Nanoha
+genre_list:Fantasy,Comedy,Sci-Fi,Noir
+warnings_list:Swearing,Violence
+characters_list:Bob Smith,George Johnson,Fred Smythe
+
+chaptertitles:Prologue,Chapter 1\, Xenos on Cinnabar,Chapter 2\, Sinmay on Kintikin,3. Chapter 3
@@ -33,16 +33,24 @@ if sys.version_info >= (2, 7):
    rootlogger.addHandler(loghandler)

 try:
-    from fanficdownloader import adapters,writers,exceptions
-    from fanficdownloader.configurable import Configuration
-    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-    from fanficdownloader.geturls import get_urls_from_page
+    from calibre.constants import numeric_version as calibre_version
+    is_calibre = True
 except:
+    is_calibre = False
+
+# using try/except directly was masking errors during development.
+if is_calibre:
    # running under calibre
    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
+else:
+    from fanficdownloader import adapters,writers,exceptions
+    from fanficdownloader.configurable import Configuration
+    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficdownloader.geturls import get_urls_from_page
+    

 if sys.version_info < (2, 5):
    print "This program requires Python 2.5 or newer."
@@ -94,6 +102,12 @@ def main(argv,
   parser.add_option("-l", "--list",
                     action="store_true", dest="list",
                     help="Get list of valid story URLs from page given.",)
+   parser.add_option("-n", "--normalize-list",
+                     action="store_true", dest="normalize",default=False,
+                     help="Get list of valid story URLs from page given, but normalized to standard forms.",)
+   parser.add_option("-s", "--sites-list",
+                     action="store_true", dest="siteslist",default=False,
+                     help="Get list of valid story URLs examples.",)
   parser.add_option("-d", "--debug",
                     action="store_true", dest="debug",
                     help="Show debug output while downloading.",)
@@ -104,17 +118,31 @@ def main(argv,
       logger = logging.getLogger("fanficdownloader")
       logger.setLevel(logging.INFO)
   
-   if len(args) != 1:
+   if not options.siteslist and len(args) != 1:
       parser.error("incorrect number of arguments")

+   if options.siteslist:
+       for (site,examples) in adapters.getSiteExamples():
+           print("\n====%s====\n\nExample URLs:"%site)
+           for u in examples:
+               print("  * %s"%u)
+       return
+
   if options.update and options.format != 'epub':
       parser.error("-u/--update-epub only works with epub")

   ## Attempt to update an existing epub.
+   chaptercount = None
+   output_filename = None
   if options.update:
-       (url,chaptercount) = get_dcsource_chaptercount(args[0])
-       print "Updating %s, URL: %s" % (args[0],url)
-       output_filename = args[0]
+       try:
+           (url,chaptercount) = get_dcsource_chaptercount(args[0])
+           print "Updating %s, URL: %s" % (args[0],url)
+           output_filename = args[0]
+       except:
+           # if there's an error reading the update file, maybe it's a URL?
+           # we'll look for an existing outputfile down below.
+           url = args[0]
   else:
       url = args[0]
           
@@ -153,8 +181,8 @@ def main(argv,
   if options.force:
       configuration.set("overrides","always_overwrite","true")

-   if options.update:
-       configuration.set("overrides","output_filename",args[0])
+   if options.update and chaptercount:
+       configuration.set("overrides","output_filename",output_filename)
       
   if options.update and not options.updatecover:
       configuration.set("overrides","never_make_cover","true")
@@ -169,8 +197,8 @@ def main(argv,
           (var,val) = opt.split('=')
           configuration.set("overrides",var,val)

-   if options.list:
-       retlist = get_urls_from_page(args[0], configuration)
+   if options.list or options.normalize:
+       retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
       print "\n".join(retlist)
               
       return
@@ -178,7 +206,18 @@ def main(argv,
   try:
       adapter = adapters.getAdapter(configuration,url)
       adapter.setChaptersRange(options.begin,options.end)
-           
+
+       # check for updating from URL (vs from file)
+       if options.update and not chaptercount:
+           try:
+               writer = writers.getWriter("epub",configuration,adapter)
+               output_filename=writer.getOutputFileName()
+               (noturl,chaptercount) = get_dcsource_chaptercount(output_filename)
+               print "Updating %s, URL: %s" % (output_filename,url)
+           except:
+               options.update = False
+               pass
+       
       ## Check for include_images and absence of PIL, give warning.
       if adapter.getConfig('include_images'):
           try:
@@ -214,11 +253,11 @@ def main(argv,

       if options.update and not options.force:
           urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
-           
+
           if chaptercount == urlchaptercount and not options.metaonly:
-               print "%s already contains %d chapters." % (args[0],chaptercount)
+               print "%s already contains %d chapters." % (output_filename,chaptercount)
           elif chaptercount > urlchaptercount:
-               print "%s contains %d chapters, more than source: %d." % (args[0],chaptercount,urlchaptercount)
+               print "%s contains %d chapters, more than source: %d." % (output_filename,chaptercount,urlchaptercount)
           else:
               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
               if not options.metaonly:
@@ -232,7 +271,7 @@ def main(argv,
                    adapter.oldimgs,
                    adapter.oldcover,
                    adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(args[0])
+                    adapter.logfile) = get_update_data(output_filename)

                   writeStory(configuration,adapter,"epub")
                   
@@ -71,7 +71,6 @@ import adapter_thehexfilesnet
 import adapter_dokugacom
 import adapter_iketernalnet
 import adapter_onedirectionfanfictioncom
-import adapter_prisonbreakficnet
 import adapter_storiesofardacom
 import adapter_samdeanarchivenu
 import adapter_destinysgatewaycom
@@ -88,7 +87,6 @@ import adapter_pretendercentrecom
 import adapter_darksolaceorg
 import adapter_finestoriescom
 import adapter_hpfanficarchivecom
-import adapter_svufictioncom
 import adapter_twilightarchivescom
 import adapter_wizardtalesnet
 import adapter_nhamagicalworldsus
@@ -103,7 +101,6 @@ import adapter_merlinficdtwinscouk
 import adapter_thehookupzonenet
 import adapter_bloodtiesfancom
 import adapter_indeathnet
-import adapter_jlaunlimitedcom
 import adapter_qafficcom
 import adapter_efpfanficnet
 import adapter_potterficscom
@@ -115,6 +112,13 @@ import adapter_imagineeficcom
 import adapter_buffynfaithnet
 import adapter_psychficcom
 import adapter_hennethannunnet
+import adapter_tokrafandomnetcom
+import adapter_netraptororg
+import adapter_asr3slashzoneorg
+import adapter_nickandgregnet
+import adapter_potterheadsanonymouscom
+import adapter_simplyundeniablecom
+import adapter_scarheadnet

 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -163,6 +167,12 @@ def getAdapter(config,url):
 def getConfigSections():
    return [cls.getConfigSection() for cls in __class_list]

+def getSiteExamples():
+    l=[]
+    for cls in sorted(__class_list, key=lambda x : x.getConfigSection()):
+        l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split()))
+    return l
+
 def getConfigSectionFor(url):
    (cls,fixedurl) = getClassFor(url)
    if cls:
@@ -52,6 +52,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
    def getSiteDomain():
        return 'www.adastrafanfic.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#  -*- coding: utf-8 -*-

 # Copyright 2011 Fanficdownloader team
 #
@@ -77,8 +77,9 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'archiveofourown.org'

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456"
+        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"

    def getSiteURLPattern(self):
        # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
@@ -292,7 +293,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
            a = metasoup.find('dd',{'class':"series"})
            b = a.find('a', href=re.compile(r"/series/\d+"))
            series_name = b.string
-            series_url = 'http://'+self.host+'/fanfic/'+b['href']
+            series_url = 'http://'+self.host+b['href']
            series_index = int(a.text.split(' ')[1])
            self.setSeries(series_name, series_index)
            self.story.setMetadata('seriesUrl',series_url)
@@ -70,6 +70,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['archive.skyehawke.com','www.skyehawke.com']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"

@@ -66,6 +66,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'ashwinder.sycophanthex.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return Asr3SlashzoneOrgAdapter
+
+class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','asr3')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'asr3.slashzone.org'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Rating
+        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
+        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
+        self.story.setMetadata('rating', rate)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        metadiv = soup.find('div',{'class':'content'})
+        smalldiv = metadiv.find('div',{'class':'small'})
+
+        categorys = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for category in categorys:
+            self.story.addToList('category',category.string)
+        
+        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        ships = smalldiv.parent.findAll('a',href=re.compile(r'browse\.php\?type=class&type_id=2&classid=1'))
+        for ship in ships:
+            self.story.addToList('ships',ship.string)
+
+        metatext = stripHTML(smalldiv)
+
+        if 'Completed: Yes' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        wordstart=metatext.rindex('Word count:')+12
+        words = metatext[wordstart:metatext.index(' ',wordstart)]
+        self.story.setMetadata('numWords', words)
+
+        datesdiv = soup.find('div',{'class':'bottom'})
+        dates = stripHTML(datesdiv).split()
+        # Published: 04/26/2011 Updated: 03/06/2013
+        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        # remove 'small' leaving only summary.
+        smalldiv.extract()
+        self.setDescription(url,metadiv)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -88,6 +88,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
        # The site domain.  Does have www here, if it uses it.
        return 'bloodties-fans.com' # XXX

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"

@@ -211,7 +212,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fiction/'+a['href'])
        self.story.setMetadata('author',a.string)

        # Find the chapters:
@@ -89,8 +89,9 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
        "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
        
+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://buffynfaith.net/fanfictions/index.php?act=vie&id=963 http://buffynfaith.net/fanfictions/index.php?act=vie&id=949 http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2"
+        return "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://buffynfaith.net/fanfictions/index.php?act=ovr&id=1234 http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234&ch=2"

    def getSiteURLPattern(self):
        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
@@ -90,6 +90,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
        # The site domain.  Does have www here, if it uses it.
        return 'castlefans.org' # XXX

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"

@@ -184,7 +185,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfic/'+a['href'])
        self.story.setMetadata('author',a.string)

        # Find the chapters:
@@ -66,6 +66,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'chaos.sycophanthex.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -64,6 +64,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.checkmated.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/story.php?story=1234"

@@ -49,7 +49,6 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -59,7 +58,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):

        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %B %Y"
+        self.dateformat = "%B %d, %Y"
            
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
@@ -70,6 +69,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.dark-solace.org','dark-solace.org']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"

@@ -79,7 +79,8 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
    ## Login seems to be reasonably standard across eFiction sites.
    def needToLoginCheck(self, data):
        if 'This story contains adult content not suitable for children' in data \
-                or "That password doesn't match the one in our database" in data:
+                or "That password doesn't match the one in our database" in data \
+                or "Registered Users Only" in data:
            return True
        else:
            return False
@@ -94,17 +95,16 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
            params['penname'] = self.getConfig("username")
            params['password'] = self.getConfig("password")
        params['rememberme'] = '1'
-        params['sid'] = ''
-        params['intent'] = ''
+        params['action'] = 'login'
        params['submit'] = 'Submit'
    
        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                              params['penname']))
    
-        d = self._fetchUrl(loginUrl, params)
+        d = self._postUrl(loginUrl, params)
    
-        if "User Account Page" not in d : #Member Account
+        if "Member Account" not in d : #User Account Page
            logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
            raise exceptions.FailedToLogin(url,params['penname'])
            return False
@@ -113,9 +113,19 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):

    ## Getting the chapter list and the meta data, plus 'is adult' checking.
    def extractChapterUrlsAndMetadata(self):
+        
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
        # index=1 makes sure we see the story chapter index.  Some
        # sites skip that for one-chapter stories.
-        url = self.url
+        url = self.url+'&index=1'+addurl
        logger.debug("URL: "+url)

        try:
@@ -128,9 +138,30 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):

        if self.needToLoginCheck(data):
            # need to log in for this one.
-            addurl="&ageconsent=ok"
            self.performLogin(url)
-            data = self._fetchUrl(url+addurl)
+            data = self._fetchUrl(url)
+            
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
            
        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
@@ -142,33 +173,41 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
        # Now go hunting for all the meta data and the chapter list.
        
        ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
+        div = soup.find('div', {'id' : 'pagetitle'})
        
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        aut = div.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',aut['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
        self.story.setMetadata('author',aut.string)
        aut.extract()
-        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # first a tag in pagetitle is title
+        self.story.setMetadata('title',stripHTML(div.find('a')))

        # Find the chapters:
-        chapters=soup.find('select', {'name' : 'chapter'})
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        # chapters=soup.find('select', {'name' : 'chapter'})
+        # if chapters != None:
+        #     for chapter in chapters.findAll('option'):
+        #         self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        # else:
+        #     self.chapterUrls.append((self.story.getMetadata('title'),url+"&chapter=1"))

+        for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
+                                                       self.story.getMetadata('storyId')+'&chapter=\d+')):
+            self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
+        
        self.story.setMetadata('numChapters',len(self.chapterUrls))
        
        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
        
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a', href=re.compile(r'viewstory.php\?sid='))
-            if a != None:
-                if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
-                    break
+        # for metalist in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+        #     a = metalist.find('a', href=re.compile(r'viewstory.php\?sid='))
+        #     if a != None:
+        #         if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
+        #             break
+
+        metalist = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                   self.story.getMetadata('storyId')+'($|[^\d])')).parent.parent

        # eFiction sites don't help us out a lot with their meta data
        # formating, so it's a little ugly.
@@ -182,7 +221,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                

        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
+        labels = metalist.findAll('span', {'class' : 'label'})
        for labelspan in labels:
            label = labelspan.text
            value = labelspan.nextSibling
@@ -190,7 +229,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
            if 'Summary' in label:
                ## Everything until the next span class='label'
                svalue = ""
-                while not (defaultGetattr(value,'class') == 'classification' or "Chapters: " in stripHTML(value)):
+                while not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
                    svalue += str(value)
                    value = value.nextSibling
                self.setDescription(url,svalue)
@@ -238,7 +277,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):

        try:
            # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            a = metalist.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
            series_name = a.string
            series_url = 'http://'+self.host+'/elysian/'+a['href']

@@ -265,8 +304,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):

        logger.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
        
        div = soup.find('div', {'id' : 'story'})

@@ -65,6 +65,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.destinysgateway.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -75,6 +75,7 @@ class DokugaComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.dokuga.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"

@@ -65,6 +65,7 @@ class DotMoonNetAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.dotmoon.net'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/library_view.php?storyid=1234"

@@ -65,6 +65,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.dracoandginny.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'dramione.org'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -237,6 +238,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
            if 'Word count' in label:
                self.story.setMetadata('numWords', value)

+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
            if 'Categories' in label:
                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                for cat in cats:
@@ -282,6 +286,14 @@ class DramioneOrgAdapter(BaseSiteAdapter):
            # I find it hard to care if the series parsing fails
            pass
            
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('h2',{'id':'pagetitle'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
    # grab the text for an individual chapter.
    def getChapterText(self, url):

@@ -65,6 +65,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'efiction.esteliel.de'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class EFPFanFicNet(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.efpfanfic.net'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -66,6 +66,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'erosnsappho.sycophanthex.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -64,8 +64,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.fanfiction.net','m.fanfiction.net']

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
+        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"

    def getSiteURLPattern(self):
        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
@@ -132,14 +133,28 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        ## For 1, use the second link.
        ## For 2, fetch the crossover page and pull the two categories from there.

-        categories = soup.findAll('a',{'class':'xcontrast_txt'})
+        categories = soup.find('div',{'id':'pre_story_links'}).findAll('a',{'class':'xcontrast_txt'})
+        #print("xcontrast_txt a:%s"%categories)
        if len(categories) > 1:
-            self.story.addToList('category',stripHTML(categories[-1]))
+            # Strangely, the ones with *two* links are the
+            # non-crossover categories.  Each is in a category itself
+            # of Book, Movie, etc.
+            self.story.addToList('category',stripHTML(categories[1]))
        elif 'Crossover' in categories[0]['href']:
            caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
            catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
                self.story.addToList('category',stripHTML(a))
+            else:
+                # Fall back.  I ran across a story with a Crossver
+                # category link to a broken page once.
+                # http://www.fanfiction.net/s/2622060/1/
+                # Naruto + Harry Potter Crossover
+                logger.info("Fall back category collection")
+                for c in stripHTML(categories[0]).replace(" Crossover","").split(' + '):
+                    self.story.addToList('category',c)
+                
+                
            
        a = soup.find('a', href='http://www.fictionratings.com/')
        rating = a.string
@@ -150,7 +165,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):

        # after Rating, the same bit of text containing id:123456 contains
        # Complete--if completed.
-        gui_table1i = soup.find(id="gui_table1i")
+        gui_table1i = soup.find('table',{'cellpadding':'5'})

        self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i

@@ -159,15 +174,16 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
            self.setDescription(url,stripHTML(summarydiv))
            

-        graydiv = gui_table1i.find('div', {'class':'xgray'})
-        for b in graydiv.findAll('button'):
-            b.extract()
-        metatext = stripHTML(graydiv).replace('Hurt/Comfort','Hurt-Comfort')
+        grayspan = gui_table1i.find('span', {'class':'xgray xcontrast_txt'})
+        # for b in grayspan.findAll('button'):
+        #     b.extract()
+        metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
        #logger.debug("metatext:(%s)"%metatext)
        metalist = metatext.split(" - ")
        #logger.debug("metalist:(%s)"%metalist)

        # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
+        # Rated: Fiction T - English - Adventure/Sci-Fi - Naruto U. - Chapters: 22 - Words: 114,414 - Reviews: 395 - Favs: 779 - Follows: 835 - Updated: 03-21-13 - Published: 04-28-12 - id: 8067258 

        # rating is obtained above more robustly.
        if metalist[0].startswith('Rated:'):
@@ -257,12 +273,18 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                        ## slow_down_sleep_time setting is.
        data = self._fetchUrl(url)

+        if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  FanFiction.net Site Error!" % url)
+        
        # some ancient stories have body tags inside them that cause
        # soup parsing to discard the content.  For story text we
        # don't care about anything before "<div class='storytextp"
        # (there's a space after storytextp, so no close quote(')) and
        # this kills any body tags.
-        data = data[data.index("<div class='storytextp"):]
+        if "<div class='storytextp" not in data:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        else:
+            data = data[data.index("<div class='storytextp"):]
        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
        
        soup = bs.BeautifulSoup(data)
@@ -67,11 +67,12 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.fanfiktion.de'

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
+        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1/story-name"

    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
+        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?"
        
        ## Login seems to be reasonably standard across eFiction sites.
    def needToLoginCheck(self, data):
@@ -137,15 +138,15 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        # Now go hunting for all the meta data and the chapter list.
        
        ## Title
-        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"/"))
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        head = soup.find('div', {'class' : 'story-metadata-left-top'})
        a = head.find('a')
        self.story.setMetadata('authorId',a['href'].split('/')[2])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))

        # Find the chapters:
        for chapter in soup.find('select').findAll('option'):
@@ -178,8 +179,8 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        
        td = tr[i].findAll('td')
        self.story.addToList('category',stripHTML(td[1]))
-        self.story.setMetadata('rating', td[4].string)
-        self.story.setMetadata('numWords', td[5].string)
+        self.story.setMetadata('rating', stripHTML(td[4]))
+        self.story.setMetadata('numWords', stripHTML(td[5]))
 
            
    # grab the text for an individual chapter.
@@ -69,8 +69,9 @@ class FicBookNetAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.ficbook.net'

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/readfic/12345"
+        return "http://"+self.getSiteDomain()+"/readfic/12345 http://"+self.getSiteDomain()+"/readfic/93626/246417#part_content"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
@@ -57,6 +57,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
    def getSiteDomain():
        return 'www.fictionalley.org'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/authors/drt/DA.html http://"+self.getSiteDomain()+"/authors/drt/JOTP01a.html"

@@ -39,8 +39,9 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.fictionpress.com','m.fictionpress.com']

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title"
+        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"

    def getSiteURLPattern(self):
        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
@@ -45,8 +45,9 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
    def getSiteDomain():
        return 'ficwad.com'

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://ficwad.com/story/137169"
+        return "http://ficwad.com/story/1234"

    def getSiteURLPattern(self):
        return re.escape(r"http://"+self.getSiteDomain())+"/story/\d+?$"
@@ -51,6 +51,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        # mobile.fimifction.com isn't actually a valid domain, but we can still get the story id from URLs anyway
        return ['www.fimfiction.net','mobile.fimfiction.net', 'www.fimfiction.com', 'mobile.fimfiction.com']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"

@@ -67,8 +67,9 @@ class FineStoriesComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'finestories.com'

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=10537"
+        return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=1234"

    def getSiteURLPattern(self):
        return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
@@ -74,6 +74,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"

@@ -227,6 +228,9 @@ class GrangerEnchantedCom(BaseSiteAdapter):
            if 'Word count' in label:
                self.story.setMetadata('numWords', value)

+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
            if 'Categories' in label:
                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                for cat in cats:
@@ -278,7 +282,14 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                        self.story.setMetadata('seriesUrl',series_url)
                        break
                    i+=1
-            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('div',{'id':'sort'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
        except:
            # I find it hard to care if the series parsing fails
            pass
@@ -56,8 +56,9 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.harrypotterfanfiction.com','harrypotterfanfiction.com']

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234 http://harrypotterfanfiction.com/viewstory.php?psid=5678"
+        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+r"(www\.)?"+re.escape("harrypotterfanfiction.com/viewstory.php?psid=")+r"\d+$"
@@ -65,6 +65,7 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.henneth-annun.net'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/stories/chapter.cfm?stid=1234"

@@ -65,6 +65,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'hlfiction.net'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -70,6 +70,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
        # The site domain.  Does have www here, if it uses it.
        return 'www.hpfandom.net' # XXX

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.hpfanficarchive.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.ik-eternal.net'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'imagine.e-fic.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -72,6 +72,7 @@ class InDeathNetAdapter(BaseSiteAdapter):
        return 'indeath.net'


+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/blog/archive/123-story-in-death/"

@@ -74,6 +74,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        # The site domain.  Does have www here, if it uses it.
        return 'ksarchive.com' # XXX

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -66,6 +66,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.libraryofmoria.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/a/viewstory.php?sid=1234"

@@ -133,7 +134,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/a/'+a['href'])
        self.story.setMetadata('author',a.string)

        # Find the chapters:
@@ -66,6 +66,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'lumos.sycophanthex.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -55,6 +55,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
    def getSiteDomain():
        return 'www.mediaminer.org'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+self.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"

@@ -65,6 +65,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'merlinfic.dtwins.co.uk'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -70,6 +70,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
        # The site domain.  Does have www here, if it uses it.
        return 'www.midnightwhispers.ca' # XXX

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -66,14 +66,19 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
            
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfiction.mugglenet.com' # XXX
+        # The site domain.
+        return 'fanfiction.mugglenet.com'

+    @classmethod
+    def getAcceptDomains(cls):
+        return ['fanfiction.mugglenet.com','fanfic.mugglenet.com']
+
+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://")+r"fanfic(tion)?\.mugglenet\.com"+re.escape("/viewstory.php?sid=")+r"\d+$"

    ## Login seems to be reasonably standard across eFiction sites.
    def needToLoginCheck(self, data):
@@ -68,8 +68,10 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.national-library.net','national-library.net']

+    @classmethod
    def getSiteExampleURLs(self):
-        return "ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+        # ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: 
+        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
@@ -68,6 +68,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.ncisfic.com','ncisfic.com']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"

@@ -79,6 +79,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.ncisfiction.net','www.ncisfiction.com']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-

-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,11 +28,9 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate

 def getClass():
-    return PrisonBreakFicNetAdapter
+    return NetRaptorOrgAdapter

-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PrisonBreakFicNetAdapter(BaseSiteAdapter):
+class NetRaptorOrgAdapter(BaseSiteAdapter):

    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)
@@ -51,31 +49,30 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
        
        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
        
        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','pbf')
+        self.story.setMetadata('siteabbrev','netrap')

        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
+        self.dateformat = "%d/%m/%y"
            
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
        # The site domain.  Does have www here, if it uses it.
-        return 'www.prisonbreakfic.net'
+        return 'netraptor.org'

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=")+r"\d+$"

    ## Getting the chapter list and the meta data, plus 'is adult' checking.
    def extractChapterUrlsAndMetadata(self):

-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
        url = self.url+'&index=1'
        logger.debug("URL: "+url)

@@ -86,6 +83,10 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                raise exceptions.StoryDoesNotExist(self.url)
            else:
                raise e
+
+
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
            
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)
@@ -94,19 +95,20 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
        # Now go hunting for all the meta data and the chapter list.
        
        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
        self.story.setMetadata('title',a.string)
        
        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfiction/'+a['href'])
        self.story.setMetadata('author',a.string)

        # Find the chapters:
        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href']))

        self.story.setMetadata('numChapters',len(self.chapterUrls))

@@ -119,10 +121,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                return d[k]
            except:
                return ""
-                
-# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
+				
        labels = soup.findAll('span',{'class':'label'})
        for labelspan in labels:
            value = labelspan.nextSibling
@@ -145,21 +144,18 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):

            if 'Categories' in label:
                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
                    self.story.addToList('category',cat.string)

            if 'Characters' in label:
                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
+                charstext = [char.string for char in chars]
+                for char in charstext:
                    self.story.addToList('characters',char.string)

-            if 'Pairing' in label:
-                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
-                for ship in ships:
-                    self.story.addToList('ships',ship.string.replace(" and ","/"))
-
            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                for genre in genres:
                    self.story.addToList('genre',genre.string)

@@ -184,7 +180,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
            # Find Series name from series URL.
            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
+            series_url = 'http://'+self.host+'/fanfiction/'+a['href']

            # use BeautifulSoup HTML parser to make everything easier to find.
            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -206,8 +202,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):

        logger.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
        
        div = soup.find('div', {'id' : 'story'})

@@ -74,6 +74,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
        # The site domain.  Does have www here, if it uses it.
        return 'nfacommunity.com' # XXX

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -66,6 +66,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'nha.magical-worlds.us'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -112,7 +113,8 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
 		
        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
            a = info.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
+                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
                self.story.setMetadata('title',a.string)
                break
 		
@@ -142,14 +144,14 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
        for cat in cats:
            self.story.addToList('category',cat.string)
 					
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        a = info.find('a', href=re.compile(r'viewuser.php'))
        val = a.nextSibling
        svalue = ""
        while not defaultGetattr(val) == 'br':
            val = val.nextSibling
        val = val.nextSibling
        while not defaultGetattr(val) == 'br':
-            svalue += str(val)
+            svalue += unicode(val)
            val = val.nextSibling
        self.setDescription(url,svalue)
        
@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NickAndGregNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NickAndGregNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nag')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.nickandgreg.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$"
+    
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&i=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters = soup.find('select')
+        for chapter in chapters.findAll('option'):
+            if chapter.text != 'Story Index' and chapter.text != 'Chapters':
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for div in asoup.findAll('td', {'class' : 'tblborder6'}):
+            a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+        
+        self.setDescription(url,div.find('br').nextSibling)
+        
+        a=div.text.split('Rating:')
+        if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0])
+        
+        a=div.text.split('Characters:')
+        if len(a) == 2: 
+        	for char in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('characters',char)
+        	    
+        a=div.text.split('Genres:')
+        if len(a) == 2: 
+        	for genre in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('genre',genre)
+        	    
+        a=div.text.split('Warnings:')
+        if len(a) == 2: 
+        	for warn in a[1].split(' -')[0].split(', '):
+        	    if 'none' not in warn:
+        	        self.story.addToList('warnings',warn)
+        	    
+        a=div.text.split('Completed:')
+        if len(a) ==2:
+            if 'Yes' in a[1]:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+                
+        a=div.text.split('Published:')
+        if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+        
+        a=div.text.split('Updated:')
+        if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('table', {'class' : 'tblborder6'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -66,6 +66,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'occlumency.sycophanthex.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -70,6 +70,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
        return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com']


+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.phoenixsong.net'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/"

@@ -112,6 +113,8 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
        logger.debug("URL: "+url)

        try:
+            if self.getConfig('force_login'):
+                self.performLogin(url)
            data = self._fetchUrl(url)
        except urllib2.HTTPError, e:
            if e.code == 404:
@@ -123,7 +126,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
            # need to log in for this one.
            self.performLogin(url)
            data = self._fetchUrl(url)
-			
            
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)
@@ -75,6 +75,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'pommedesang.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/sds/viewstory.php?sid=1234"

@@ -70,8 +70,9 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234 http://explicit."+self.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -72,6 +72,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
        # The site domain.  Does have www here, if it uses it.
        return 'fanfiction.portkey.org' # XXX

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/story/1234"

@@ -55,8 +55,9 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.potionsandsnitches.net','potionsandsnitches.net']

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234 http://potionsandsnitches.net/fanfiction/viewstory.php?sid=5678"
+        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$"
@@ -72,13 +72,9 @@ class PotterFicsComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.potterfics.com'

+    @classmethod
    def getSiteExampleURLs(self):
-        return \
-                "http://www.potterfics.com/historias/127583 "\
-                "http://www.potterfics.com/historias/127583/capitulo-1 "\
-                "http://www.potterfics.com/historias/127583/capitulo-4 "\
-                "http://www.potterfics.com/historias/92810 "\
-                "http://www.potterfics.com/historias/111194"
+        return "http://www.potterfics.com/historias/12345 http://www.potterfics.com/historias/12345/capitulo-1 "

    def getSiteURLPattern(self):
        #http://www.potterfics.com/historias/127583
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-

-# Copyright 2011 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,12 +27,12 @@ from .. import exceptions as exceptions

 from base_adapter import BaseSiteAdapter,  makeDate

-
 def getClass():
-    return JLAUnlimitedComAdapter
+    return PotterHeadsAnonymousComAdapter

-
-class JLAUnlimitedComAdapter(BaseSiteAdapter):
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):

    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)
@@ -42,36 +42,70 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.username = "" # if left empty, site doesn't return any message at all.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
        self.password = ""
        self.is_adult=False
-
+        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-
-        self._setURL('http://' + self.getSiteDomain() + '/eFiction1.1/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','jla')
+        self.story.setMetadata('siteabbrev','pha')

        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-
+        self.dateformat = "%d %b %Y"
+            
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
        # The site domain.  Does have www here, if it uses it.
-        return 'www.jlaunlimited.com'
+        return 'fanfic.potterheadsanonymous.com'

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=")+r"\d+$"
-
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}

+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True

    ## Getting the chapter list and the meta data, plus 'is adult' checking.
    def extractChapterUrlsAndMetadata(self):
@@ -81,7 +115,7 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
            # If the title search below fails, there's a good chance
            # you need a different number.  print data at that point
            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4" # XXX
+            addurl = "&ageconsent=ok&warning=4"
        else:
            addurl=""

@@ -98,8 +132,21 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
            else:
                raise e

-        # Assume that if there is a url with 'warning=#' in the page then it is a 'check' page
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)",data)
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
        if m != None:
            if self.is_adult or self.getConfig("is_adult"):
                # We tried the default and still got a warning, so
@@ -120,27 +167,32 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
                        raise e    
            else:
                raise exceptions.AdultCheckRequired(self.url)
-                
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)
        # print data

        # Now go hunting for all the meta data and the chapter list.

+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        
        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
        self.story.setMetadata('title',a.string)
        
        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
        self.story.setMetadata('author',a.string)

        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+")):
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eFiction1.1/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))

        self.story.setMetadata('numChapters',len(self.chapterUrls))

@@ -177,34 +229,22 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):

            if 'Categories' in label:
                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
+                for cat in cats:
                    self.story.addToList('category',cat.string)

            if 'Characters' in label:
                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
+                for char in chars:
                    self.story.addToList('characters',char.string)

-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
                    self.story.addToList('genre',genre.string)

-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for warning in warnings:
                    self.story.addToList('warnings',warning.string)

            if 'Completed' in label:
@@ -225,7 +265,7 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
            # Find Series name from series URL.
            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
            series_name = a.string
-            series_url = 'http://'+self.host+'/fanfic/'+a['href']
+            series_url = 'http://'+self.host+'/'+a['href']

            # use BeautifulSoup HTML parser to make everything easier to find.
            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -241,20 +281,18 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
        except:
            # I find it hard to care if the series parsing fails
            pass
-
-
+            
    # grab the text for an individual chapter.
    def getChapterText(self, url):

        logger.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulSoup(self._fetchUrl(url),
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        div = soup.find('div', {'id' : 'story'})
        
+        div = soup.find('div', {'id' : 'story'})
+
        if None == div:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
+    
        return self.utf8FromSoup(url,div)
-
@@ -69,6 +69,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.pretendercentre.com','pretendercentre.com']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.psychfic.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class QafFicComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.qaf-fic.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=1234"

@@ -70,6 +70,7 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
    def getSiteDomain():
        return 'www.restrictedsection.org'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/story.php?story=1234 http://"+self.getSiteDomain()+"/file.php?file=1234"

@@ -69,6 +69,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.samdean.archive.nu','samdean.archive.nu']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-

-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,11 +28,11 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate

 def getClass():
-    return SVUFictionComAdapter
+    return ScarHeadNetAdapter

 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class SVUFictionComAdapter(BaseSiteAdapter):
+class ScarHeadNetAdapter(BaseSiteAdapter):

    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)
@@ -54,17 +54,18 @@ class SVUFictionComAdapter(BaseSiteAdapter):
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
        
        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','svuf')
+        self.story.setMetadata('siteabbrev','shn')

        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
+        self.dateformat = "%d/%m/%y"
            
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
        # The site domain.  Does have www here, if it uses it.
-        return 'svufiction.com'
+        return 'scarhead.net'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -108,13 +109,13 @@ class SVUFictionComAdapter(BaseSiteAdapter):

    ## Getting the chapter list and the meta data, plus 'is adult' checking.
    def extractChapterUrlsAndMetadata(self):
-    
+
        if self.is_adult or self.getConfig("is_adult"):
            # Weirdly, different sites use different warning numbers.
            # If the title search below fails, there's a good chance
            # you need a different number.  print data at that point
            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=6"
+            addurl = "&ageconsent=ok&warning=5"
        else:
            addurl=""

@@ -130,12 +131,21 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                raise exceptions.StoryDoesNotExist(self.url)
            else:
                raise e
-                
+
        if self.needToLoginCheck(data):
            # need to log in for this one.
            self.performLogin(url)
            data = self._fetchUrl(url)
-                
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
        if m != None:
            if self.is_adult or self.getConfig("is_adult"):
@@ -166,24 +176,19 @@ class SVUFictionComAdapter(BaseSiteAdapter):
        # print data

        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('tr',{'valign':'top'})
        
        ## Title
-        pt = soup.find('div', {'class' : 'title'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        if a.text != "":
-            self.story.setMetadata('title',a.text)
-        else:
-            self.story.setMetadata('title','Banner')
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
        
        # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
        self.story.setMetadata('author',a.string)

-        rating=pt.text.split('[')[1].split(']')[0]
-        self.story.setMetadata('rating', rating)
-
        # Find the chapters:
        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
            # just in case there's tags, like <i> in chapter titles.
@@ -193,46 +198,70 @@ class SVUFictionComAdapter(BaseSiteAdapter):

        # eFiction sites don't help us out a lot with their meta data
        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        bottom=soup.find('div', {'class' : 'bottom'}).text
-        self.story.setMetadata('numWords', bottom.split('Word count: ')[1].split(' Read:')[0])
-        self.story.setMetadata('datePublished', makeDate(bottom.split('Published: ')[1].split(' Updated:')[0], self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(bottom.split('Updated: ')[1], self.dateformat))
-
        
-        content=soup.find('div', {'class' : 'content'})
-        value=content.find('h1').nextSibling
-        svalue = ""
-        while not defaultGetattr(value,'class') == 'smaller':
-            svalue += str(value)
-            value = value.nextSibling
-        self.setDescription(url,svalue)
-        
-        status = content.text.split('Completed: ')[1]
-        if 'Yes' in status:
+        cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            if '/' == cat.string[0]:
+                self.story.addToList('ships','Harry Potter'+cat.string.split('(')[0])
+            elif 'Harry' in cat.string:
+                self.story.addToList('ships',cat.string.split('(')[0])
+            else:
+                self.story.addToList('category',cat.string)
+            if '(' in cat.string:
+                self.story.addToList('category',cat.string.split('(')[1].split(')')[0])
+            
+                
+            
+
+        chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+
+        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)
+            
+        textsoup = stripHTML(soup)
+
+        a = textsoup.split('Published: ')[1].split(' ')[0]
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a), self.dateformat))
+        a = textsoup.split('Updated: ')[1].split(' ')[0]
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a), self.dateformat))
+        a = textsoup.split('Rating: ')[1].split(' ')[0]
+        self.story.setMetadata('rating', a)
+        a = textsoup.split('Length: ')[1].split('(')[1].split(' ')[0]
+        self.story.setMetadata('numWords', a)
+        a = textsoup.split('Completed: ')[1].split(' ')[0]
+        if 'Yes' in a:
            self.story.setMetadata('status', 'Completed')
        else:
            self.story.setMetadata('status', 'In-Progress')
+        #a = textsoup.split('Summary: ')[1].split('Add Story to Favorites')[0]
+        #self.setDescription(url,a)
        
-        cats = content.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-            
-        chars = content.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-            
-        genres = content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-            
+
+        
+        a=soup.find(text=re.compile("Summary: "))
+        i=0
+        svalue = ""
+        while i == 0:
+            try:
+                b = str(a)
+                svalue += b.split('Summary: ')[1]
+            except:
+                svalue += str(a)
+            if a.nextSibling != None:
+                a = a.nextSibling
+            else:
+                a = a.parent.nextSibling
+            if 'Disclaimer: ' in stripHTML(a):
+                i=1
+        self.setDescription(url,svalue)
+
        try:
            # Find Series name from series URL.
            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
@@ -241,17 +270,14 @@ class SVUFictionComAdapter(BaseSiteAdapter):

            # use BeautifulSoup HTML parser to make everything easier to find.
            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
            i=1
            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
            
        except:
            # I find it hard to care if the series parsing fails
@@ -262,7 +288,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):

        logger.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
        
        div = soup.find('div', {'id' : 'story'})
@@ -65,6 +65,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.scarvesandcoffee.net'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -141,7 +142,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
        self.story.setMetadata('title',a.string)
        
        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
        self.story.setMetadata('author',a.string)
@@ -74,6 +74,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'sg1-heliopolis.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SimplyUndeniableComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SimplyUndeniableComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sud')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.simplyundeniable.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Please log in now' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        if "Please log in now" in data:
+                raise exceptions.FailedToDownload(self.getSiteDomain() +" says: You need to have access to the restricted section.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for info in asoup.findAll('table', {'cellpadding' : '5'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.string)
+                break
+                
+        # Find the chapters:
+        if "Disclaimer" in data:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')):
+            # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        
+        a = info.find('td', {'valign' : 'top'}).find('p')
+        self.setDescription(url,a)
+        
+        a = info.find('td', {'class' : 'greysm'}).findAll('b')
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a[0].nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].nextSibling), self.dateformat))
+        if 'Yes' in a[2].nextSibling:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        self.story.setMetadata('rating', a[3].nextSibling)
+        self.story.setMetadata('numWords', a[4].nextSibling)
+        
+        
+        warnings = info.find('td', {'width' : '45'}).nextSibling.nextSibling.text.split(', ')
+        for warning in warnings:
+            if 'none' not in warning:
+                self.story.addToList('warnings',warning)
+        chars = info.find('td', {'width' : '51'}).nextSibling.nextSibling.text.split(', ')
+        for char in chars:
+            if '&' in char:
+                self.story.addToList('ships',char)
+            else:
+                self.story.addToList('characters',char)
+        genres = info.find('td', {'width' : '36'}).nextSibling.nextSibling.text.split(', ')
+        for genre in genres:
+            self.story.addToList('genre',genre)
+            
+        cat = info.find('a', href=re.compile(r'categories.php'))
+        self.story.addToList('category',cat.string)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('span', {'class' : 'style'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -65,6 +65,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.sinful-desire.org'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"

@@ -70,6 +70,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
    def getAcceptDomains(cls):
        return ['www.siye.co.uk','siye.co.uk']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"

@@ -110,7 +111,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX

        ## Title
        titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',titlea.string)
+        self.story.setMetadata('title',stripHTML(titlea))
        
        # Find the chapters (from soup, not authsoup):
        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
@@ -81,6 +81,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
        # The config section name.  Only override if != site domain.
        return cls.getSiteDomain()+'/peja'
    
+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'stargate-atlantis.org'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.storiesofarda.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=1234"

@@ -55,6 +55,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
    def getSiteDomain():
        return 'fanfiction.tenhawkpresents.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -40,6 +40,7 @@ class TestSiteAdapter(BaseSiteAdapter):
    def getSiteDomain():
        return 'test1.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"?sid=1234"

@@ -47,31 +48,63 @@ class TestSiteAdapter(BaseSiteAdapter):
        return BaseSiteAdapter.getSiteURLPattern(self)+r'/?\?sid=\d+$'

    def extractChapterUrlsAndMetadata(self):
+        idstr = self.story.getMetadata('storyId')
+        idnum = int(idstr)

-        if self.story.getMetadata('storyId') == '665' and not (self.is_adult or self.getConfig("is_adult")):
+        if idnum >= 1000:
+            logger.warn("storyId:%s - Custom INI data will be used."%idstr)
+
+            sections = ['teststory:%s'%idstr,'teststory:defaults']
+            #print("self.get_config_list(sections,'valid_entries'):%s"%self.get_config_list(sections,'valid_entries'))
+            for key in self.get_config_list(sections,'valid_entries'):
+                if key.endswith("_list"):
+                    nkey = key[:-len("_list")]
+                    #print("addList:%s"%(nkey))
+                    for val in self.get_config_list(sections,key):
+                        #print("addList:%s->%s"%(nkey,val))
+                        self.story.addToList(nkey,val.decode('utf-8').replace('{{storyId}}',idstr))
+                else:
+                    # Special cases:
+                    if key in ['datePublished','dateUpdated']:
+                        self.story.setMetadata(key,makeDate(self.get_config(sections,key),"%Y-%m-%d"))
+                    else:
+                        self.story.setMetadata(key,self.get_config(sections,key).decode('utf-8').replace('{{storyId}}',idstr))
+                    #print("set:%s->%s"%(key,self.story.getMetadata(key)))
+
+            self.chapterUrls = []
+            for (j,chap) in enumerate(self.get_config_list(sections,'chaptertitles'),start=1):
+                self.chapterUrls.append( (chap,self.url+"&chapter=%d"%j) )
+            # self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
+            #                 ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
+            #                 ]
+            self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+            return
+
+        if idstr == '665' and not (self.is_adult or self.getConfig("is_adult")):
            logger.warn("self.is_adult:%s"%self.is_adult)
            raise exceptions.AdultCheckRequired(self.url)

-        if self.story.getMetadata('storyId') == '666':
+        if idstr == '666':
            raise exceptions.StoryDoesNotExist(self.url)

-        if self.story.getMetadata('storyId').startswith('670'):
+        if idstr.startswith('670'):
            time.sleep(1.0)
            
-        if self.story.getMetadata('storyId').startswith('671'):
+        if idstr.startswith('671'):
            time.sleep(1.0)
            
        if self.getConfig("username"):
            self.username = self.getConfig("username")
        
-        if self.story.getMetadata('storyId') == '668' and self.username != "Me" :
+        if idstr == '668' and self.username != "Me" :
            raise exceptions.FailedToLogin(self.url,self.username)

-        if self.story.getMetadata('storyId') == '664':
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring)
+        if idstr == '664':
+            self.story.setMetadata(u'title',"Test Story Title "+idstr+self.crazystring)
            self.story.setMetadata('author','Test Author aa bare amp(&) quote(&#39;) amp(&amp;)')
        else:
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
+            self.story.setMetadata(u'title',"Test Story Title "+idstr)
            self.story.setMetadata('author','Test Author aa')
        self.story.setMetadata('storyUrl',self.url)
        self.setDescription(self.url,u'Description '+self.crazystring+u''' Done
@@ -79,13 +112,12 @@ class TestSiteAdapter(BaseSiteAdapter):
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
        self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
-        if self.story.getMetadata('storyId') == '669':
+        if idstr == '669':
            self.story.setMetadata('dateUpdated',datetime.datetime.now())
        else:
            self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
        self.story.setMetadata('numWords','123456')

-        idnum = int(self.story.getMetadata('storyId'))
        if idnum % 2 == 1:
            self.story.setMetadata('status','In-Progress')
        else:
@@ -108,15 +140,67 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
            
        self.story.setMetadata('rating','Tweenie')

-        if self.story.getMetadata('storyId') == '673':
-            self.story.addToList('author','Author From List')
+        if idstr == '673':
+            self.story.addToList('author','Author From List 1')
            self.story.addToList('author','Author From List 2')
+            self.story.addToList('author','Author From List 3')
+            self.story.addToList('author','Author From List 4')
+            self.story.addToList('author','Author From List 5')
+            self.story.addToList('author','Author From List 6')
+            self.story.addToList('author','Author From List 7')
+            self.story.addToList('author','Author From List 8')
+            self.story.addToList('author','Author From List 9')
+            self.story.addToList('author','Author From List 0')
+            self.story.addToList('author','Author From List q')
+            self.story.addToList('author','Author From List w')
+            self.story.addToList('author','Author From List e')
+            self.story.addToList('author','Author From List r')
+            self.story.addToList('author','Author From List t')
+            self.story.addToList('author','Author From List y')
+            self.story.addToList('author','Author From List u')
+            self.story.addToList('author','Author From List i')
+            self.story.addToList('author','Author From List o')
        
-            self.story.addToList('authorId','98765')
+            self.story.addToList('authorId','98765-1')
            self.story.addToList('authorId','98765-2')
+            self.story.addToList('authorId','98765-3')
+            self.story.addToList('authorId','98765-4')
+            self.story.addToList('authorId','98765-5')
+            self.story.addToList('authorId','98765-6')
+            self.story.addToList('authorId','98765-7')
+            self.story.addToList('authorId','98765-8')
+            self.story.addToList('authorId','98765-9')
+            self.story.addToList('authorId','98765-0')
+            self.story.addToList('authorId','98765-q')
+            self.story.addToList('authorId','98765-w')
+            self.story.addToList('authorId','98765-e')
+            self.story.addToList('authorId','98765-r')
+            self.story.addToList('authorId','98765-t')
+            self.story.addToList('authorId','98765-y')
+            self.story.addToList('authorId','98765-u')
+            self.story.addToList('authorId','98765-i')
+            self.story.addToList('authorId','98765-o')
        
-            self.story.addToList('authorUrl','http://author/url')
+            self.story.addToList('authorUrl','http://author/url-1')
            self.story.addToList('authorUrl','http://author/url-2')
+            self.story.addToList('authorUrl','http://author/url-3')
+            self.story.addToList('authorUrl','http://author/url-4')
+            self.story.addToList('authorUrl','http://author/url-5')
+            self.story.addToList('authorUrl','http://author/url-6')
+            self.story.addToList('authorUrl','http://author/url-7')
+            self.story.addToList('authorUrl','http://author/url-8')
+            self.story.addToList('authorUrl','http://author/url-9')
+            self.story.addToList('authorUrl','http://author/url-0')
+            self.story.addToList('authorUrl','http://author/url-q')
+            self.story.addToList('authorUrl','http://author/url-w')
+            self.story.addToList('authorUrl','http://author/url-e')
+            self.story.addToList('authorUrl','http://author/url-r')
+            self.story.addToList('authorUrl','http://author/url-t')
+            self.story.addToList('authorUrl','http://author/url-y')
+            self.story.addToList('authorUrl','http://author/url-u')
+            self.story.addToList('authorUrl','http://author/url-i')
+            self.story.addToList('authorUrl','http://author/url-o')
+
            self.story.addToList('category','Power Rangers')
            self.story.addToList('category','SG-1')
            self.story.addToList('genre','Porn')
@@ -128,18 +212,19 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
        self.story.addToList('warnings','Swearing')
        self.story.addToList('warnings','Violence')

-        if self.story.getMetadata('storyId') == '80':
-            self.story.addToList('category',u'Rizzoli & Isles')
+        if idstr == '80':
+            self.story.addToList('category',u'Rizzoli &amp; Isles')
            self.story.addToList('characters','J. Rizzoli')
-        elif self.story.getMetadata('storyId') == '81':
+        elif idstr == '81':
            self.story.addToList('category',u'Pitch Perfect')
            self.story.addToList('characters','Chloe B.')
-        elif self.story.getMetadata('storyId') == '83':
-            self.story.addToList('category',u'Rizzoli & Isles')
+        elif idstr == '83':
+            self.story.addToList('category',u'Rizzoli &amp; Isles')
            self.story.addToList('characters','J. Rizzoli')
            self.story.addToList('category',u'Pitch Perfect')
            self.story.addToList('characters','Chloe B.')
-        elif self.story.getMetadata('storyId') == '82':
+            self.story.addToList('ships','Chloe B. &amp; J. Rizzoli')
+        elif idstr == '82':
            self.story.addToList('characters','Henry (Once Upon a Time)')        
            self.story.addToList('category',u'Once Upon a Time (TV)')
        else:
@@ -152,6 +237,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
            self.story.addToList('characters','Bob Smith')
            self.story.addToList('characters','George Johnson')
            self.story.addToList('characters','Fred Smythe')
+            self.story.addToList('ships','Harry Potter/Ginny Weasley')
+            self.story.addToList('ships','Harry Potter/Ginny Weasley/Albus Dumbledore')
+            self.story.addToList('ships','Harry Potter &amp; Hermione Granger')
            
        self.story.addToList('genre','Fantasy')
        self.story.addToList('genre','Comedy')
@@ -217,6 +305,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <div>
 <h3>Prologue</h3>
 <p>This is a fake adapter for testing purposes.  Different sid's will give different errors:</p>
+<h4>Config(personal.ini)</h4>
+<p>sid&gt;=1000 will use custom test story data from your configuration(personal.ini)</p>
+<p>Hard coded ids:</p>
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
@@ -224,10 +315,6 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
-
-
-
-
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
 <p>http://test1.com?sid=673 - Succeeds, multiple authors, extra categories, genres</p>
@@ -263,7 +350,7 @@ Don't&#8212e;ver&#8212d;o&#8212;that&#8212a;gain, &#27861; &#xE9;
 <hr>
 horizontal rules
 <hr size=1 noshade>
-<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<p>"Lorem ipsum dolor sit amet", consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore--et dolore magna aliqua. 'Ut enim ad minim veniam', quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 </div>
 '''
@@ -65,6 +65,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.thealphagate.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -69,6 +69,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.thehexfiles.net','thehexfiles.net']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -102,7 +103,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
 		
        try:
@@ -117,7 +118,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
        for info in asoup.findAll('table', {'cellspacing' : '4'}):
            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
            if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                break
 		

@@ -90,6 +90,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
        # The site domain.  Does have www here, if it uses it.
        return 'thehookupzone.net' # XXX

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=1234"

@@ -183,7 +184,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/CriminalMinds/'+a['href'])
        self.story.setMetadata('author',a.string)

        # Find the chapters:
@@ -72,6 +72,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'themasque.net'
        
+    @classmethod
    def getSiteExampleURLs(self):
        return "http://themasque.net/wiktt/efiction/viewstory.php?sid=1234 http://themasque.net/efiction/viewstory.php?sid=1234"

@@ -65,6 +65,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.thepetulantpoetess.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -74,6 +74,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
    def getAcceptDomains(cls):
        return ['www.thequidditchpitch.org','thequidditchpitch.org']

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

@@ -55,6 +55,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
    def getSiteDomain():
        return 'www.thewriterscoffeeshop.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/library/viewstory.php?sid=1234"

@@ -244,16 +245,23 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
        # issues with different SGML parsers in python.  This is a
        # nasty hack, but it works.
        data = data[data.index("<body"):]
-        
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        span = soup.find('div', {'id' : 'story'})

-        if None == span:
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        
+        soup = bs.BeautifulSoup(data)
+
+        found=False
+        for div in soup.findAll('div'):
+            if div.has_key('class') and div['class'] == 'notes':
+                chapter.append(div)
+            if div.has_key('id') and div['id'] == 'story':
+                chapter.append(div)
+                found=True
+
+        if not found:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,span)
+
+        return self.utf8FromSoup(url,chapter)

 def getClass():
    return TheWritersCoffeeShopComSiteAdapter
@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TokraFandomnetComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TokraFandomnetComAdapter(BaseSiteAdapter): 
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tokra')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.  But it
+        # doesn't matter too much anymore.
+        return 'tokra.fandomnet.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Rating
+        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
+        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
+        self.story.setMetadata('rating', rate)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        metadiv = soup.find('div',{'class':'content'})
+        smalldiv = metadiv.find('div',{'class':'small'})
+
+        # tokra categories -> genre
+        # categories will be filled from ini.
+        genres = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+        
+        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        metatext = stripHTML(smalldiv)
+
+        if 'Completed: Yes' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        wordstart=metatext.rindex('Word count:')+12
+        words = metatext[wordstart:metatext.index(' ',wordstart)]
+        self.story.setMetadata('numWords', words)
+
+        datesdiv = soup.find('div',{'class':'bottom'})
+        dates = stripHTML(datesdiv).split()
+        # Published: 04/26/2011 Updated: 03/06/2013
+        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        # remove 'small' leaving only summary.
+        smalldiv.extract()
+        self.setDescription(url,metadiv)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'class' : 'content'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        # remove some decorations while keeping notes.
+        remove = div.find('div', {'id' : 'pagetitle'})
+        remove.extract()
+    
+        for remove in div.findAll('div', {'class' : 'right'}):
+            remove.extract()
+    
+        for remove in div.findAll('div', {'class' : 'left'}):
+            remove.extract()
+    
+        return self.utf8FromSoup(url,div)
@@ -54,13 +54,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
    def getSiteDomain():
        return 'www.tthfanfic.org'

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm http://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
+        return "http://www.tthfanfic.org/Story-1234 http://www.tthfanfic.org/Story-1234/Author+Story+Title.htm http://www.tthfanfic.org/T-99999999/Story-1234-1/Author+Story+Title.htm http://www.tthfanfic.org/story.php?no=12345"

-    # http://www.tthfanfic.org/T-526321777848988007890480555880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm
-    # http://www.tthfanfic.org/Story-5583
-    # http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm
-    # http://www.tthfanfic.org/story.php?no=26093
+    # http://www.tthfanfic.org/T-999999999999/Story-12345-1/Author+Story+Title.htm
+    # http://www.tthfanfic.org/Story-12345
+    # http://www.tthfanfic.org/Story-12345/Author+Story+Title.htm
+    # http://www.tthfanfic.org/story.php?no=12345
    def getSiteURLPattern(self):
        return r"http://www.tthfanfic.org(/(T-\d+/)?Story-|/story.php\?no=)(?P<id>\d+)(-\d+)?(/.*)?$"

@@ -154,26 +155,6 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
        self.story.setMetadata('author',stripHTML(a))
        authorurl = 'http://'+self.host+a['href']

-        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
-        if ainfo != None: # indicates multiple authors/contributors.
-            try:
-                # going to pull part of the meta data from author list page.
-                infourl = 'http://'+self.host+ainfo['href']
-                logger.debug("**StoryInfo** URL: "+infourl)
-                infodata = self._fetchUrl(infourl)
-                infosoup = bs.BeautifulSoup(infodata)
-
-                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
-                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
-                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
-                    self.story.addToList('author',stripHTML(a))                
-                                            
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(url)
-                else:
-                    raise e
-    
        try:
            # going to pull part of the meta data from *primary* author list page.
            logger.debug("**AUTHOR** URL: "+authorurl)
@@ -198,6 +179,53 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
        #self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'})))
        self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'})))

+        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
+        if ainfo != None: # indicates multiple authors/contributors.
+            try:
+                # going to pull part of the meta data from author list page.
+                infourl = 'http://'+self.host+ainfo['href']
+                logger.debug("**StoryInfo** URL: "+infourl)
+                infodata = self._fetchUrl(infourl)
+                infosoup = bs.BeautifulSoup(infodata)
+
+                # for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
+                #     self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
+                #     self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
+                #     self.story.addToList('author',stripHTML(a))
+
+                # second verticaltable is the chapter list.
+                table = infosoup.findAll('table',{'class':'verticaltable'})[1]
+                for a in table.findAll('a',href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))):
+                    autha = a.findNext('a',href=re.compile(r"^/Author-\d+"))
+                    self.story.addToList('authorId',autha['href'].split('/')[1].split('-')[1])
+                    self.story.addToList('authorUrl','http://'+self.host+autha['href'].replace("/Author-","/AuthorStories-"))
+                    self.story.addToList('author',stripHTML(autha))
+                    # include leading number to match 1. ... 2. ...
+                    self.chapterUrls.append(("%d. %s by %s"%(len(self.chapterUrls)+1,
+                                                             stripHTML(a),
+                                                             stripHTML(autha)),'http://'+self.host+a['href']))
+                                            
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        else: # single author:
+            # Find the chapter selector 
+            select = soup.find('select', { 'name' : 'chapnav' } )
+        	 
+            if select is None:
+        	   # no selector found, so it's a one-chapter story.
+        	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+            else:
+                allOptions = select.findAll('option')
+                for o in allOptions:
+                    url = "http://"+self.host+o['value']
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(o),url))
+    
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
        verticaltable = soup.find('table', {'class':'verticaltable'})

        BtVS = True
@@ -235,21 +263,6 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
        if BtVS:
            self.story.addToList('category','Buffy: The Vampire Slayer')
            
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'chapnav' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = "http://"+self.host+o['value']
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(o),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
        pseries = soup.find('p', {'style':'margin-top:0px'})
        m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
                     pseries.text)
@@ -66,6 +66,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
        return 'www.twilightarchives.com'
       

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/read/1234"

@@ -57,8 +57,9 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.twilighted.net','twilighted.net']

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://www.twilighted.net/viewstory.php?sid=1234 http://twilighted.net/viewstory.php?sid=5678"
+        return "http://www.twilighted.net/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+r"(www\.)?"+re.escape("twilighted.net/viewstory.php?sid=")+r"\d+$"
@@ -58,8 +58,9 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
    def getAcceptDomains(cls):
        return ['www.twiwrite.net','twiwrite.net']

+    @classmethod
    def getSiteExampleURLs(self):
-        return "http://www.twiwrite.net/viewstory.php?sid=1234 http://twiwrite.net/viewstory.php?sid=5678"
+        return "http://www.twiwrite.net/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
        return re.escape("http://")+r"(www\.)?"+re.escape("twiwrite.net/viewstory.php?sid=")+r"\d+$"
@@ -63,6 +63,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
        # The site domain.  Does have www here, if it uses it.
        return 'www.walkingtheplank.org'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"

@@ -119,7 +120,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
        self.story.setMetadata('author',a.string)

        # Find the chapters:
@@ -41,6 +41,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
    def getSiteDomain():
        return 'www.whofic.com'

+    @classmethod
    def getSiteExampleURLs(self):
        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Jim Miller	7ac00b53ab	Fix for smarten_punctuation setting in pre-0.9.39 calibre.	2013-07-24 13:02:14 -05:00
Jim Miller	9c0519316d	Added tag FanFictionDownLoader-4.4.66 for changeset c32e4193a161	2013-07-23 15:19:01 -05:00
Jim Miller	d9fbccbef5	Added tag calibre-plugin-1.7.32 for changeset c32e4193a161	2013-07-23 15:18:48 -05:00
Jim Miller	0c622b379e	Bump versions.	2013-07-23 15:18:37 -05:00
Jim Miller	83d210566b	Fixes for fanfiktion.de & thehexfiles.net, add entries to teststory valid list.	2013-07-23 15:17:16 -05:00
Jim Miller	ab0b25317b	Added tag FanFictionDownLoader-4.4.65 for changeset 1def4c08104d	2013-07-18 11:05:16 -05:00
Jim Miller	e10ae5fbe4	Added tag calibre-plugin-1.7.31 for changeset 1def4c08104d	2013-07-18 11:04:59 -05:00
Jim Miller	62eb103ff5	Bump versions, update ini files for new parameters.	2013-07-18 11:04:47 -05:00
Jim Miller	8e984e62fb	Only do Smarten Punctuation on calibre 0.9.39+.	2013-07-18 11:00:10 -05:00
Jim Miller	fd270789ac	Smarten Punctuation option on epub download/update.	2013-07-17 16:53:00 -05:00
Jim Miller	f6bd941369	Drag and drop onto the plugin button on toolbar.	2013-07-17 15:21:39 -05:00
Jim Miller	54ae77082a	sort_ships option. Ship b/a is changed to a/b, etc.	2013-07-15 14:17:52 -05:00
Jim Miller	b3c30f6a21	Make output_filename_safepattern configurable.	2013-07-13 16:46:54 -05:00
Jim Miller	84bc9b3a3e	Add join_string_<entry>, keep_in_order_<entry> and replace_metadata <entry>_LIST options.	2013-07-12 15:36:36 -05:00
Jim Miller	e59673fdd7	Change CLI u/--update-epub option to update existing file when given URL, too.	2013-07-12 15:34:54 -05:00
Jim Miller	8d2b9646fb	Commit Previous Version update.	2013-07-10 14:24:20 -05:00
Jim Miller	7fc3a2ad2d	Added tag FanFictionDownLoader-4.4.64 for changeset d771f0b4b2f2	2013-07-10 14:17:27 -05:00
Jim Miller	8c0a216e46	Make 'supported list' generated from adapter Examples. Bump web version.	2013-07-10 14:17:06 -05:00
Jim Miller	9b08fe3bfb	Added tag calibre-plugin-1.7.30 for changeset d1e6cd170b48	2013-07-09 19:29:54 -05:00
Jim Miller	b4ffb2f9d9	Added tag FanFictionDownLoader-4.4.63 for changeset d1e6cd170b48	2013-07-09 19:29:45 -05:00
Jim Miller	f276ca081f	Bump versions.	2013-07-09 19:29:16 -05:00
iatheia	dc2116dd3e	New adapter for scarhead.net	2013-07-08 22:19:35 -04:00
Jim Miller	737bed8bfa	GUI improvements to Basic config and Add/Update dialogs.	2013-07-08 15:58:04 -05:00
Jim Miller	178d83d733	Redo Basic config layout into better groupings.	2013-07-05 16:08:54 -05:00
Jim Miller	ccc07676fb	Phoenixsong can have individual chapters req login. Add force_login parameter.	2013-07-03 09:51:39 -05:00
Jim Miller	a6b9ca52bb	Improved error handling when ffnet is having problems.	2013-07-02 23:40:10 -05:00
Jim Miller	cbb8eb14db	Added tag FanFictionDownLoader-4.4.62 for changeset 1fd9bacc4f62	2013-06-28 11:06:25 -05:00
Jim Miller	b472f8d216	Added tag calibre-plugin-1.7.29 for changeset 1fd9bacc4f62	2013-06-28 11:06:08 -05:00
Jim Miller	a4d97500b5	Bump versions.	2013-06-28 11:05:30 -05:00
Jim Miller	22d9a674a3	Allow domain fanfic.mugglenet.com for mugglenet.com.	2013-06-28 10:09:26 -05:00
iatheia	1e908c3440	New adapter for fanfic.potterheadsanonymous.com and www.simplyundeniable.com (latter needs user input for restricted stories)	2013-06-28 00:06:29 -04:00
Jim Miller	ec7c02f2ec	Added tag calibre-plugin-1.7.28 for changeset 1a40ab7c9ba4	2013-06-22 19:20:18 -05:00
Jim Miller	2049910bb0	Added tag FanFictionDownLoader-4.4.61 for changeset 1a40ab7c9ba4	2013-06-22 19:20:03 -05:00
Jim Miller	1e61bebf21	Fix for siye.co.uk stories with HTML in the title, bump versions.	2013-06-22 19:19:42 -05:00
Jim Miller	7f47100db4	Couple minor fixes.	2013-06-18 13:53:21 -05:00
Jim Miller	83b48d15d8	Added tag FanFictionDownLoader-4.4.60 for changeset de6789e70c76	2013-06-10 21:12:06 -05:00
Jim Miller	5aec8188a2	Added tag calibre-plugin-1.7.27 for changeset de6789e70c76	2013-06-10 21:11:53 -05:00
Jim Miller	5a34f4c86a	Bump versions, update index.html.	2013-06-10 21:11:39 -05:00
Jim Miller	8e5ba7b634	Restrict url list search on scarvesandcoffee.net.	2013-06-10 19:46:25 -05:00
Jim Miller	7a47d1fad2	Fix for scarvesandcoffee.net taking author from featured stories.	2013-06-10 19:45:24 -05:00
Jim Miller	7f4749a022	Fix for nha.magical-worlds.us: reviews link disappeared, fix finding story url for meta section, fix highbyte chars in description.	2013-06-10 19:44:58 -05:00
Jim Miller	b44f059e57	Added tag calibre-plugin-1.7.26 for changeset 127933a4d5f3	2013-06-09 11:54:53 -05:00
Jim Miller	6b9058a9eb	Added tag FanFictionDownLoader-4.4.59 for changeset 127933a4d5f3	2013-06-09 11:54:30 -05:00
Jim Miller	5acf9a8d0b	Add 'Download as New Book?' dialog after 'Change Story URL?', fix author URLs when new author.	2013-06-08 23:11:55 -05:00
Jim Miller	454c7ffb2f	Fall back category parse for ffnet when broken crossover cat link.	2013-06-08 23:10:51 -05:00
Jim Miller	f6dcb447b0	Bump versions, update index.html.	2013-05-30 21:37:48 -05:00
Jim Miller	5ce064bf92	Fix so non-anthology numeric custom columns populate correctly.	2013-05-30 19:37:50 -05:00
Jim Miller	c9a1537190	Fix for numeric site specific values into float/int custom columns.	2013-05-30 12:52:57 -05:00
Jim Miller	36e192e82c	Add Read & Review counts to dramione.org and grangerenchanted.com	2013-05-30 12:52:27 -05:00
iatheia	42058d02b3	Adapter for nickandgreg.net	2013-05-30 01:18:53 -04:00
Jim Miller	e0832b9deb	Update index.html	2013-05-26 16:15:19 -05:00
Jim Miller	4ccb94fca0	Added tag FanFictionDownLoader-4.4.58 for changeset b53a5015e7b3	2013-05-26 16:07:46 -05:00
Jim Miller	0e5b64bee0	Added tag calibre-plugin-1.7.25 for changeset b53a5015e7b3	2013-05-26 16:07:30 -05:00
Jim Miller	53a325ffa3	Bump versions.	2013-05-26 16:07:19 -05:00
Jim Miller	29f900199e	Configurable testing stories in test1.com.	2013-05-22 21:18:36 -05:00
Jim Miller	f56fb0efed	Add "keep_style_attr" option.	2013-05-22 21:14:56 -05:00
Jim Miller	a703011aef	Correct a misspelling.	2013-05-02 13:16:53 -05:00
Jim Miller	a635272bcf	Added tag FanFictionDownLoader-4.4.57 for changeset c5d34faaec8f	2013-04-28 11:35:48 -05:00
Jim Miller	1079ed565c	Added tag calibre-plugin-1.7.24 for changeset c5d34faaec8f	2013-04-28 11:35:33 -05:00
Jim Miller	17ab4a3d25	Bump versions.	2013-04-28 11:35:23 -05:00
Jim Miller	2566e434ee	Tweak 'skip update when the story URL would be changed' feature.	2013-04-28 11:34:46 -05:00
Jim Miller	91f5269453	Offer user chance to skip update when the story URL would be changed.	2013-04-27 18:17:41 -05:00
Jim Miller	235f00a5e6	After adding to reject list, delete only rejects, not those removed from list.	2013-04-27 17:36:28 -05:00
Jim Miller	aad4a26131	Don't strip lead/trail whitespace from replace_metadata, add \s->' ' in replacements	2013-04-26 10:40:13 -05:00
Jim Miller	f20a02a2bc	Catch errors on db.set_cover().	2013-04-25 22:00:58 -05:00
Jim Miller	1ba1da4d65	Moved tag FanFictionDownLoader-4.4.56 to changeset 6ea4b9910113 (from changeset 0252e03d11c4)	2013-04-12 10:35:27 -05:00
Jim Miller	a951fc3d6c	Fix index.html. Really this time.	2013-04-12 10:35:08 -05:00
Jim Miller	606785f6e7	Moved tag FanFictionDownLoader-4.4.56 to changeset 0252e03d11c4 (from changeset 14a9a3ca53c1)	2013-04-12 10:32:37 -05:00
Jim Miller	842ab4feeb	Fix index.html.	2013-04-12 10:32:18 -05:00
Jim Miller	029af794cf	Added tag FanFictionDownLoader-4.4.56 for changeset 14a9a3ca53c1	2013-04-12 10:25:11 -05:00
Jim Miller	87033bd4a3	Added tag calibre-plugin-1.7.23 for changeset 14a9a3ca53c1	2013-04-12 10:24:59 -05:00
Jim Miller	c8577893c4	Bump versions.	2013-04-12 10:24:48 -05:00
Jim Miller	a822064da0	Remove dead site jlaunlimited.com	2013-04-11 19:51:29 -05:00
Jim Miller	af7c717c20	Add new sites to index.html	2013-04-11 17:11:54 -05:00
Jim Miller	56c75350a6	anthology_title_pattern feature	2013-04-11 17:04:45 -05:00
Jim Miller	34b5076753	Add 'url' to chapter custom formats and class="skip_on_ffdl_update" for updates.	2013-04-10 13:30:05 -05:00
Jim Miller	f0847809ad	Improved error handling for mobi issues.	2013-04-09 20:17:37 -05:00
Jim Miller	976ddf827e	Fix for no chapter name for one chapter stories on TtH.	2013-04-09 20:17:16 -05:00
Jim Miller	da188234ac	Sync up plugin-defaults.ini	2013-04-09 17:38:18 -05:00
Jim Miller	89a676d0b6	Add site: asr3.slashzone.org	2013-04-09 17:36:54 -05:00
Jim Miller	1d48b7c72c	Add site: netraptor.org	2013-04-09 17:12:23 -05:00
Jim Miller	ce82d96163	Fix author URLs for sites with leading 'dir' in URL.	2013-04-09 16:58:36 -05:00
Jim Miller	049486a059	Add site: tokra.fandomnet.com.	2013-04-09 16:30:24 -05:00
Jim Miller	d098bdbdc8	Improve error reporting on CLI.	2013-04-09 16:29:56 -05:00
Jim Miller	2dd85dd044	Added tag FanFictionDownLoader-4.4.55 for changeset 69dd14744c87	2013-04-08 12:28:10 -05:00
Jim Miller	fcc5c7ffcc	Added tag calibre-plugin-1.7.22 for changeset 69dd14744c87	2013-04-08 12:27:58 -05:00
Jim Miller	9c25271a59	Remove dead sites prisonbreakfic.net, svufiction.com; bump versions.	2013-04-08 12:27:13 -05:00
Jim Miller	6dd5522b7a	Include notes with chapter text for thewriterscoffeeshop.com.	2013-04-07 12:13:16 -05:00
Jim Miller	aaccb45df5	Fix AO3 series url.	2013-04-07 12:12:51 -05:00
Jim Miller	7b052d353a	Added tag FanFictionDownLoader-4.4.54 for changeset fa9a7e44cf2f	2013-04-05 12:12:22 -05:00
Jim Miller	d969fbd251	Added tag calibre-plugin-1.7.21 for changeset fa9a7e44cf2f	2013-04-05 12:11:59 -05:00
Jim Miller	bcc4ee5efd	Bump versions.	2013-04-05 12:11:49 -05:00
Jim Miller	6b2a03dc8f	Use ini sections both w/ and w/o www. for all sites. Without takes precedence.	2013-04-05 12:08:55 -05:00
Jim Miller	a5a3d284b5	Fix persisting AddDialog size. PI.	2013-04-04 21:30:57 -05:00
Jim Miller	b3bf2fc50a	Update dark-solace.org/elysian adapter to properly do is_adult.	2013-04-04 17:04:12 -05:00
Jim Miller	22f4cc76fc	Use ini sections both w/ and w/o www. for all sites. Canonical takes precedence.	2013-04-04 17:03:44 -05:00
Jim Miller	c02ad3f6d5	Update dark-solace.org/elysian adapter for site changes.	2013-04-03 12:15:47 -05:00
Jim Miller	e65e209ab3	Fix for drag & drop of text emails.	2013-04-03 12:12:34 -05:00
Jim Miller	d940936713	Added tag FanFictionDownLoader-4.4.53 for changeset 5b6e38359ca4	2013-04-01 10:06:06 -05:00
Jim Miller	423e1d2840	Added tag calibre-plugin-1.7.20 for changeset 5b6e38359ca4	2013-04-01 10:05:47 -05:00
Jim Miller	e1c2c1a2a2	Another fix for fanfiction.net changes.	2013-04-01 10:05:30 -05:00
Jim Miller	0277ef7590	Added tag FanFictionDownLoader-4.4.52 for changeset b7d0e24dc68a	2013-03-28 20:42:13 -05:00
Jim Miller	63f30c12ab	Added tag calibre-plugin-1.7.19 for changeset b7d0e24dc68a	2013-03-28 20:41:57 -05:00
Jim Miller	c6875d472d	Bump cli/web version too.	2013-03-28 20:41:43 -05:00
Jim Miller	5758e364e4	Bump version.	2013-03-28 20:39:18 -05:00
Jim Miller	d9a99fb7e9	Another fix for fanfiction.net changes.	2013-03-28 16:40:59 -05:00
Jim Miller	0a32cdb277	Added tag FanFictionDownLoader-4.4.51 for changeset aaf38d4e578b	2013-03-28 12:31:23 -05:00
Jim Miller	00c1fe8704	Added tag calibre-plugin-1.7.18 for changeset aaf38d4e578b	2013-03-28 12:31:10 -05:00
Jim Miller	bb96ecd5fc	Bump versions.	2013-03-28 12:30:57 -05:00
Jim Miller	64866f7da6	Add author to chapter TOC for multi-author stories on TtH and WraithBait. AO3 doesn't reliably report chapter author.	2013-03-28 12:19:32 -05:00
Jim Miller	73815bda43	CLI: Add normalized option when getting story URL list from a URL.	2013-03-28 11:01:52 -05:00
Jim Miller	d54acd936e	PI: Allow partial download with <url>[1-5] in addition to <url>[1,5].	2013-03-28 11:01:14 -05:00
Jim Miller	e88244bebd	Fix for failure to correctly load old JSON settings.	2013-03-28 11:00:21 -05:00
Jim Miller	02bdaf1086	Added tag FanFictionDownLoader-4.4.50 for changeset 3452b4bc5b3e	2013-03-26 13:46:20 -05:00
Jim Miller	21d5a39958	Added tag calibre-plugin-1.7.17 for changeset 3452b4bc5b3e	2013-03-26 13:46:08 -05:00