Bump versions, update ini files for new parameters.

options.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-61
+version: 4-4-65
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 28)
+    version             = (1, 7, 31)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -13,10 +13,11 @@ from collections import OrderedDict
 from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
                       QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
                       QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
-                      QDialogButtonBox )
+                      QDialogButtonBox, QGroupBox )
 
 from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog
+from calibre.constants import numeric_version as calibre_version
 
 from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
@@ -177,6 +178,7 @@ class ConfigWidget(QWidget):
         prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
         prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
+        prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
 
         if self.readinglist_tab:
             # lists
@@ -266,13 +268,16 @@ class BasicTab(QWidget):
         self.plugin_action = plugin_action
         QWidget.__init__(self)
         
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
+        topl = QVBoxLayout()
+        self.setLayout(topl)
 
         label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
         label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
+        topl.addWidget(label)
+
+        defs_gb = groupbox = QGroupBox("Defaults Options on Download")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
         horz = QHBoxLayout()
@@ -317,7 +322,15 @@ class BasicTab(QWidget):
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         self.l.addWidget(self.updateepubcover)
 
-        self.l.addSpacing(10)        
+        if calibre_version >= (0, 9, 39):
+            self.smarten_punctuation = QCheckBox('Smarten Punctuation (EPUB only)',self)
+            self.smarten_punctuation.setToolTip("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update.")
+            self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
+            self.l.addWidget(self.smarten_punctuation)
+
+        cali_gb = groupbox = QGroupBox("Updating Calibre Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
         self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
@@ -334,14 +347,24 @@ class BasicTab(QWidget):
         self.keeptags.setChecked(prefs['keeptags'])
         self.l.addWidget(self.keeptags)
 
-        self.l.addSpacing(10)        
+        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
+        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
+        self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
+        self.l.addWidget(self.checkforseriesurlid)
+
+        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
+        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
+        self.l.addWidget(self.lookforurlinhtml)
 
         self.showmarked = QCheckBox("Show added/updated books when finished?",self)
         self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
         self.showmarked.setChecked(prefs['showmarked'])
         self.l.addWidget(self.showmarked)
 
-        self.l.addSpacing(10)        
+        gui_gb = groupbox = QGroupBox("GUI Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
         self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
@@ -359,7 +382,9 @@ class BasicTab(QWidget):
         self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
         self.l.addWidget(self.adddialogstaysontop)
 
-        self.l.addSpacing(10)        
+        misc_gb = groupbox = QGroupBox("Misc Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         # this is a cheat to make it easier for users to realize there's a new include_images features.
         self.includeimages = QCheckBox("Include images in EPUBs?",self)
@@ -367,43 +392,43 @@ class BasicTab(QWidget):
         self.includeimages.setChecked(prefs['includeimages'])
         self.l.addWidget(self.includeimages)
 
-        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
-        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
-        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
-        self.l.addWidget(self.lookforurlinhtml)
-
-        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
-        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
-        self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
-        self.l.addWidget(self.checkforseriesurlid)
-
         self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
         self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
-        self.l.addSpacing(10)        
+        rej_gb = groupbox = QGroupBox("Reject List")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
-        horz = QHBoxLayout()
-        
         self.rejectlist = QPushButton('Edit Reject URL List', self)
         self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
         self.rejectlist.clicked.connect(self.show_rejectlist)
-        horz.addWidget(self.rejectlist)
+        self.l.addWidget(self.rejectlist)
         
         self.reject_urls = QPushButton('Add Reject URLs', self)
         self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
         self.reject_urls.clicked.connect(self.add_reject_urls)
-        horz.addWidget(self.reject_urls)
+        self.l.addWidget(self.reject_urls)
         
         self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
         self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
         self.reject_reasons.clicked.connect(self.show_reject_reasons)
-        horz.addWidget(self.reject_reasons)
+        self.l.addWidget(self.reject_reasons)
+
+        topl.addWidget(defs_gb)
+
+        horz = QHBoxLayout()
+        topl.addLayout(horz)
+        horz.addWidget(cali_gb)
+        horz.addWidget(rej_gb)
         
-        self.l.addLayout(horz)
-        
-        self.l.insertStretch(-1)
+        horz = QHBoxLayout()
+        topl.addLayout(horz)
+        horz.addWidget(gui_gb)
+        horz.addWidget(misc_gb)
+
+        topl.insertStretch(-1)
         
     def set_collisions(self):
         prev=self.collision.currentText()

calibre-plugin/dialogs.py

@@ -17,7 +17,8 @@ from PyQt4 import QtGui
 from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
                       QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
                       QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
-                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal)
+                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal,
+                      QGroupBox, QFrame)
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
@@ -48,6 +49,8 @@ anthology_collision_order=[UPDATE,
                            UPDATEALWAYS,
                            OVERWRITEALWAYS]
 
+gpstyle='QGroupBox {border:0; padding-top:10px; padding-bottom:0px; margin-bottom:0px;}' #  background-color:red;
+
 class RejectUrlEntry:
 
     matchpat=re.compile(r"^(?P<url>[^,]+)(,(?P<fullnote>(((?P<title>.+) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
@@ -200,6 +203,24 @@ class AddNewDialog(SizePersistedDialog):
         # elements to show again when doing *update* merge
         self.mergeupdateshow = []
 
+        self.groupbox = QGroupBox("Show Download Options")
+        self.groupbox.setCheckable(True)
+        self.groupbox.setChecked(False)
+        self.groupbox.setFlat(True)
+        print("style:%s"%self.groupbox.styleSheet())
+        self.groupbox.setStyleSheet(gpstyle);
+
+        self.gbf = QFrame()
+        self.gbl = QVBoxLayout()
+        self.gbl.addWidget(self.gbf)
+        self.groupbox.setLayout(self.gbl)
+        self.gbl = QVBoxLayout()
+        self.gbf.setLayout(self.gbl)
+        self.l.addWidget(self.groupbox)
+
+        self.gbf.setVisible(False)
+        self.groupbox.toggled.connect(self.gbf.setVisible)
+        
         horz = QHBoxLayout()
         label = QLabel('Output &Format:')
         self.mergehide.append(label)
@@ -215,7 +236,7 @@ class AddNewDialog(SizePersistedDialog):
         horz.addWidget(label)
         label.setBuddy(self.fileform)
         horz.addWidget(self.fileform)
-        self.l.addLayout(horz)
+        self.gbl.addLayout(horz)
         self.mergehide.append(self.fileform)
 
         horz = QHBoxLayout()
@@ -230,7 +251,7 @@ class AddNewDialog(SizePersistedDialog):
             self.collision.setCurrentIndex(i)
         self.collisionlabel.setBuddy(self.collision)
         horz.addWidget(self.collision)
-        self.l.addLayout(horz)
+        self.gbl.addLayout(horz)
         self.mergehide.append(self.collisionlabel)
         self.mergehide.append(self.collision)
         self.mergeupdateshow.append(self.collisionlabel)
@@ -250,7 +271,7 @@ class AddNewDialog(SizePersistedDialog):
         horz.addWidget(self.updateepubcover)
         self.mergehide.append(self.updateepubcover)
         
-        self.l.addLayout(horz)
+        self.gbl.addLayout(horz)
 
         self.button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         self.button_box.accepted.connect(self.ok_clicked)
@@ -288,6 +309,8 @@ class AddNewDialog(SizePersistedDialog):
         self.newmerge = newmerge
         self.extraoptions = extraoptions
         self.extrapayload = extrapayload
+
+        self.groupbox.setVisible(not(self.merge and self.newmerge))
         
         if self.merge:
             self.toplabel.setText('Story URL(s) for anthology, one per line:')
@@ -360,6 +383,7 @@ class AddNewDialog(SizePersistedDialog):
             'collision': unicode(self.collision.currentText()),
             'updatemeta': self.updatemeta.isChecked(),
             'updateepubcover': self.updateepubcover.isChecked(),
+            'smarten_punctuation':self.prefs['smarten_punctuation']
                 }
         
         if self.merge:
@@ -595,7 +619,8 @@ class UpdateExistingDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
                  save_size_name='fanfictiondownloader_plugin:update list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        
+
+        self.prefs = prefs
         self.setWindowTitle(header)
         self.setWindowIcon(icon)
         
@@ -625,8 +650,25 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         options_layout = QHBoxLayout()
 
+        groupbox = QGroupBox("Show Download Options")
+        groupbox.setCheckable(True)
+        groupbox.setChecked(False)
+        groupbox.setFlat(True)
+        groupbox.setStyleSheet(gpstyle)
+
+        gbf = QFrame()
+        gbl = QVBoxLayout()
+        gbl.addWidget(gbf)
+        groupbox.setLayout(gbl)
+        gbl = QHBoxLayout()
+        gbf.setLayout(gbl)
+        options_layout.addWidget(groupbox)
+
+        gbf.setVisible(False)
+        groupbox.toggled.connect(gbf.setVisible)
+        
         label = QLabel('Output &Format:')
-        options_layout.addWidget(label)
+        gbl.addWidget(label)
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
         self.fileform.addItem('mobi')
@@ -636,10 +678,10 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
-        options_layout.addWidget(self.fileform)
+        gbl.addWidget(self.fileform)
         
         label = QLabel('Update Mode:')
-        options_layout.addWidget(label)
+        gbl.addWidget(label)
         self.collision = QComboBox(self)
         self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
         # add collision options
@@ -648,17 +690,19 @@ class UpdateExistingDialog(SizePersistedDialog):
         if i > -1:
             self.collision.setCurrentIndex(i)
         label.setBuddy(self.collision)
-        options_layout.addWidget(self.collision)
+        gbl.addWidget(self.collision)
 
         self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
         self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
         self.updatemeta.setChecked(prefs['updatemeta'])
-        options_layout.addWidget(self.updatemeta)
+        gbl.addWidget(self.updatemeta)
                 
         self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
         self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
         self.updateepubcover.setChecked(prefs['updateepubcover'])
-        options_layout.addWidget(self.updateepubcover)
+        gbl.addWidget(self.updateepubcover)
+
+
         
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
@@ -694,6 +738,7 @@ class UpdateExistingDialog(SizePersistedDialog):
             'collision': unicode(self.collision.currentText()),
             'updatemeta': self.updatemeta.isChecked(),
             'updateepubcover': self.updateepubcover.isChecked(),
+            'smarten_punctuation':self.prefs['smarten_punctuation']
             }
 
 class StoryListTableWidget(QTableWidget):

calibre-plugin/ffdl_plugin.py

@@ -12,8 +12,10 @@ from StringIO import StringIO
 from functools import partial
 from datetime import datetime
 from string import Template
+import urllib
+import email
 
-from PyQt4.Qt import (QApplication, QMenu, QToolButton)
+from PyQt4.Qt import (QApplication, QMenu, QToolButton, QTimer)
 
 from PyQt4.Qt import QPixmap, Qt
 from PyQt4.QtCore import QBuffer
@@ -38,7 +40,7 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page, get_urls_from_html, get_urls_from_text
 
 from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config, get_ffdl_personalini)
 from calibre_plugins.fanfictiondownloader_plugin.config import (permitted_values, rejecturllist)
@@ -144,6 +146,72 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self.macmenuhack = os.access(file_path, os.F_OK)
             return self.macmenuhack
 
+    accepts_drops = True
+
+    def accept_enter_event(self, event, mime_data):
+        if mime_data.hasFormat("application/calibre+from_library") or \
+                mime_data.hasFormat("text/plain") or \
+                mime_data.hasFormat("text/uri-list"):
+            return True
+                
+        return False
+
+    def accept_drag_move_event(self, event, mime_data):
+        return self.accept_enter_event(event, mime_data)
+
+    def drop_event(self, event, mime_data):
+
+        dropped_ids=None
+        urllist=[]
+        
+        mime = 'application/calibre+from_library'
+        if mime_data.hasFormat(mime):
+            dropped_ids = tuple(map(int, str(mime_data.data(mime)).split()))
+        
+        mimetype='text/uri-list'
+        filelist="%s"%event.mimeData().data(mimetype)
+        if filelist:
+            for f in filelist.splitlines():
+                #print("filename:%s"%f)
+                if f.endswith(".eml"):
+                    fhandle = urllib.urlopen(f)
+                    msg = email.message_from_file(fhandle)
+                    if msg.is_multipart():
+                        for part in msg.walk():
+                            #print("part type:%s"%part.get_content_type())
+                            if part.get_content_type() == "text/html":
+                                #print("URL list:%s"%get_urls_from_data(part.get_payload(decode=True)))
+                                urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
+                            if part.get_content_type() == "text/plain":
+                                #print("part content:text/plain")
+                                #print("part content:%s"%part.get_payload(decode=True))
+                                urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
+                    else:
+                        urllist.extend(get_urls_from_text("%s"%msg))
+                else:
+                    urllist.extend(get_urls_from_text(f))
+        else:
+            mimetype='text/plain'
+            if mime_data.hasFormat(mimetype):
+                #print("text/plain:%s"%event.mimeData().data(mimetype))
+                urllist.extend(get_urls_from_text(event.mimeData().data(mimetype)))
+
+        #print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
+        if urllist or dropped_ids:
+            QTimer.singleShot(1, partial(self.do_drop,
+                                         dropped_ids=dropped_ids,
+                                         urllist=urllist))
+            return True
+        
+        return False
+
+    def do_drop(self,dropped_ids=None,urllist=None):
+        # shouldn't ever be both.
+        if dropped_ids:
+            self.update_dialog(dropped_ids)
+        elif urllist:
+            self.add_dialog("\n".join(urllist))
+    
     def about_to_show_menu(self):
         self.rebuild_menus()
 
@@ -554,19 +622,21 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # Now that we've 
         self.prep_downloads( options, update_books, merge=True )
         
-    def update_dialog(self):
+    def update_dialog(self, id_list=None):
         if not self.is_library_view():
             self.gui.status_bar.show_message(_('Cannot Update Books from Device View'), 3000)
             return
+
+        if not id_list:
+            id_list = self.gui.library_view.get_selected_ids();
         
-        if len(self.gui.library_view.get_selected_ids()) == 0:
+        if len(id_list) == 0:
             self.gui.status_bar.show_message(_('No Selected Books to Update'), 3000)
             return
         #print("update_dialog()")
         
         db = self.gui.current_db
-        book_list = map( partial(self.make_book_id_only), self.gui.library_view.get_selected_ids() )
-        #book_ids = self.gui.library_view.get_selected_ids()
+        book_list = map( self.make_book_id_only, id_list )
 
         for j, book in enumerate(book_list):
             book['listorder'] = j

calibre-plugin/jobs.py

@@ -14,6 +14,7 @@ from StringIO import StringIO
 
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
+from calibre.constants import numeric_version as calibre_version
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
@@ -49,7 +50,7 @@ def do_download_worker(book_list, options,
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
                     'do_download_for_worker',
                     (book,options)]
-            job = ParallelJob('arbitrary',
+            job = ParallelJob('arbitrary_n',
                               "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
                               done=None,
                               args=args)
@@ -84,7 +85,7 @@ def do_download_worker(book_list, options,
         book_id = job._book['calibre_id']
         #title = job._title
         count = count + 1
-        notification(float(count)/total, 'Downloaded Story')
+        notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
         # Add this job's output to the current log
         print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
         print(job.details)
@@ -106,7 +107,7 @@ def do_download_worker(book_list, options,
     # return the book list as the job result
     return book_list
 
-def do_download_for_worker(book,options):
+def do_download_for_worker(book,options,notification=lambda x,y:x):
     '''
     Child job, to extract isbn from formats for this specific book,
     when run as a worker job
@@ -206,6 +207,23 @@ def do_download_for_worker(book,options):
             book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
                 (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
         
+        if options['smarten_punctuation'] and options['fileform'] == "epub" \
+                and calibre_version >= (0, 9, 39):
+            # do smarten_punctuation from calibre's polish feature
+            from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+            from calibre.utils.logging import Log
+            from collections import namedtuple
+        
+            data = {'smarten_punctuation':True}
+            opts = ALL_OPTS.copy()
+            opts.update(data)
+            O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+            opts = O(**opts)
+            
+            log = Log(level=Log.DEBUG)
+            # report = []
+            polish({outfile:outfile}, opts, log, print) # report.append
+        
     except NotGoingToDownload as d:
         book['good']=False
         book['comment']=unicode(d)
@@ -218,6 +236,6 @@ def do_download_for_worker(book,options):
         book['status'] = 'Error'
         print("Exception: %s:%s"%(book,unicode(e)))
         traceback.print_exc()
-
+        
     #time.sleep(10)
     return book

calibre-plugin/prefs.py

@@ -41,6 +41,7 @@ default_prefs['includeimages'] = False
 default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
 default_prefs['injectseries'] = False
+default_prefs['smarten_punctuation'] = False
 
 default_prefs['send_lists'] = ''
 default_prefs['read_lists'] = ''

defaults.ini

@@ -133,6 +133,9 @@ zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 ## zip_filename.
 allow_unsafe_filename: false
 
+## The regex pattern of 'unsafe' filename chars for above.
+#output_filename_safepattern:[^a-zA-Z0-9_\. \[\]\(\)&'-]+
+
 ## entries to make epub subjects and calibre tags
 ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
 include_subject_tags: extratags, genre, category, characters, ships, lastupdate, status
@@ -171,6 +174,11 @@ extratags: FanFiction
 ## Note that if metakey == conditionalkey the conditional is ignored.
 ## You can use \s in the replacement to add explicit spaces.  (The config parser
 ## tends to discard trailing spaces.)
+## replace_metadata <entry>_LIST options: FFDL replace_metadata lines
+## operate on individual list items for list entries. But if you
+## want to do a replacement on the joined string for the whole list,
+## you can by using <entry>_LIST. Example, if you added
+## calibre_author: calibre_author_LIST=>^(.{,100}).*$=>\1
 #replace_metadata:
 # genre,category=>Sci-Fi=>SF
 # Puella Magi Madoka Magica.* => Madoka
@@ -233,6 +241,24 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## "The Beginning" => "1. The Beginning" 
 chapter_title_add_pattern:${index}. ${title}
 
+## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
+## on '/', so use replace_metadata to change separator first if
+## needed. Something like: ships=>[ ]*(/|&amp;|&)[ ]*=>/ You can use
+## ships_LIST to change the / back to something else if you want.
+sort_ships:false
+
+## join_string_<entry> options -- FFDL list entries are comma
+## separated by default. You can use this to change that. For example,
+## if you want authors separated with ' & ' instead, use
+## join_string_calibre_author:\s&\s. (\s == space)
+#join_string_author:,\s
+
+## keep_in_order_<entry> options: FFDL sorts list entries by default
+## (except for author/authorUrl/authorId). But if you want to use an
+## extra entry derived from author, it ends up sorted. For example, if
+## you added calibre_author: keep_in_order_calibre_author:true
+#keep_in_order_author:true
+
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -667,6 +693,11 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -674,11 +705,22 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+[fanfic.potterheadsanonymous.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [fanfiction.portkey.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -870,6 +912,22 @@ extracategories:Supernatural
 extracharacters:Sam,Dean
 extraships:Sam/Dean
 
+[scarhead.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1200,6 +1258,12 @@ extraships:Harry Potter/Ginny Weasley
 #username:YourName
 #password:yourpassword
 
+## phoenixsong.net, oddly, can have high rated chapters (login
+## required) in the middle of a lower rated story.  Use this to force
+## FFDL to always login to phoenixsong.net so those stories download
+## correctly.
+#force_login:true
+
 [www.potionsandsnitches.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1250,6 +1314,17 @@ extracharacters:Kurt Hummel,Blaine Anderson
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.simplyundeniable.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.sinful-desire.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Supernatural

downloader.py

@@ -105,6 +105,9 @@ def main(argv,
    parser.add_option("-n", "--normalize-list",
                      action="store_true", dest="normalize",default=False,
                      help="Get list of valid story URLs from page given, but normalized to standard forms.",)
+   parser.add_option("-s", "--sites-list",
+                     action="store_true", dest="siteslist",default=False,
+                     help="Get list of valid story URLs examples.",)
    parser.add_option("-d", "--debug",
                      action="store_true", dest="debug",
                      help="Show debug output while downloading.",)
@@ -115,17 +118,31 @@ def main(argv,
        logger = logging.getLogger("fanficdownloader")
        logger.setLevel(logging.INFO)
    
-   if len(args) != 1:
+   if not options.siteslist and len(args) != 1:
        parser.error("incorrect number of arguments")
 
+   if options.siteslist:
+       for (site,examples) in adapters.getSiteExamples():
+           print("\n====%s====\n\nExample URLs:"%site)
+           for u in examples:
+               print("  * %s"%u)
+       return
+
    if options.update and options.format != 'epub':
        parser.error("-u/--update-epub only works with epub")
 
    ## Attempt to update an existing epub.
+   chaptercount = None
+   output_filename = None
    if options.update:
-       (url,chaptercount) = get_dcsource_chaptercount(args[0])
-       print "Updating %s, URL: %s" % (args[0],url)
-       output_filename = args[0]
+       try:
+           (url,chaptercount) = get_dcsource_chaptercount(args[0])
+           print "Updating %s, URL: %s" % (args[0],url)
+           output_filename = args[0]
+       except:
+           # if there's an error reading the update file, maybe it's a URL?
+           # we'll look for an existing outputfile down below.
+           url = args[0]
    else:
        url = args[0]
            
@@ -164,8 +181,8 @@ def main(argv,
    if options.force:
        configuration.set("overrides","always_overwrite","true")
 
-   if options.update:
-       configuration.set("overrides","output_filename",args[0])
+   if options.update and chaptercount:
+       configuration.set("overrides","output_filename",output_filename)
        
    if options.update and not options.updatecover:
        configuration.set("overrides","never_make_cover","true")
@@ -189,7 +206,18 @@ def main(argv,
    try:
        adapter = adapters.getAdapter(configuration,url)
        adapter.setChaptersRange(options.begin,options.end)
-           
+
+       # check for updating from URL (vs from file)
+       if options.update and not chaptercount:
+           try:
+               writer = writers.getWriter("epub",configuration,adapter)
+               output_filename=writer.getOutputFileName()
+               (noturl,chaptercount) = get_dcsource_chaptercount(output_filename)
+               print "Updating %s, URL: %s" % (output_filename,url)
+           except:
+               options.update = False
+               pass
+       
        ## Check for include_images and absence of PIL, give warning.
        if adapter.getConfig('include_images'):
            try:
@@ -225,11 +253,11 @@ def main(argv,
 
        if options.update and not options.force:
            urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
-           
+
            if chaptercount == urlchaptercount and not options.metaonly:
-               print "%s already contains %d chapters." % (args[0],chaptercount)
+               print "%s already contains %d chapters." % (output_filename,chaptercount)
            elif chaptercount > urlchaptercount:
-               print "%s contains %d chapters, more than source: %d." % (args[0],chaptercount,urlchaptercount)
+               print "%s contains %d chapters, more than source: %d." % (output_filename,chaptercount,urlchaptercount)
            else:
                print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
                if not options.metaonly:
@@ -243,7 +271,7 @@ def main(argv,
                     adapter.oldimgs,
                     adapter.oldcover,
                     adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(args[0])
+                    adapter.logfile) = get_update_data(output_filename)
 
                    writeStory(configuration,adapter,"epub")
                    

fanficdownloader/adapters/__init__.py

@@ -116,6 +116,9 @@ import adapter_tokrafandomnetcom
 import adapter_netraptororg
 import adapter_asr3slashzoneorg
 import adapter_nickandgregnet
+import adapter_potterheadsanonymouscom
+import adapter_simplyundeniablecom
+import adapter_scarheadnet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -164,6 +167,12 @@ def getAdapter(config,url):
 def getConfigSections():
     return [cls.getConfigSection() for cls in __class_list]
 
+def getSiteExamples():
+    l=[]
+    for cls in sorted(__class_list, key=lambda x : x.getConfigSection()):
+        l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split()))
+    return l
+
 def getConfigSectionFor(url):
     (cls,fixedurl) = getClassFor(url)
     if cls:

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -52,6 +52,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.adastrafanfic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -77,8 +77,9 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archiveofourown.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456"
+        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -70,6 +70,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['archive.skyehawke.com','www.skyehawke.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
 

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -66,6 +66,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'ashwinder.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_asr3slashzoneorg.py

@@ -63,6 +63,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'asr3.slashzone.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -88,6 +88,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'bloodties-fans.com' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -89,8 +89,9 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
         "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
         self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
         
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://buffynfaith.net/fanfictions/index.php?act=vie&id=963 http://buffynfaith.net/fanfictions/index.php?act=vie&id=949 http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2"
+        return "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://buffynfaith.net/fanfictions/index.php?act=ovr&id=1234 http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234&ch=2"
 
     def getSiteURLPattern(self):
         #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -90,6 +90,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'castlefans.org' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -66,6 +66,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'chaos.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -64,6 +64,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.checkmated.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story.php?story=1234"
 

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -69,6 +69,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.dark-solace.org','dark-solace.org']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -65,6 +65,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.destinysgateway.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_dokugacom.py

@@ -75,6 +75,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.dokuga.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
 

fanficdownloader/adapters/adapter_dotmoonnet.py

@@ -65,6 +65,7 @@ class DotMoonNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.dotmoon.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/library_view.php?storyid=1234"
 

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -65,6 +65,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.dracoandginny.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -65,6 +65,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'dramione.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -65,6 +65,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'efiction.esteliel.de'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -65,6 +65,7 @@ class EFPFanFicNet(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.efpfanfic.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -66,6 +66,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'erosnsappho.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -64,8 +64,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.fanfiction.net','m.fanfiction.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
+        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"
 
     def getSiteURLPattern(self):
         return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
@@ -272,12 +273,18 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                         ## slow_down_sleep_time setting is.
         data = self._fetchUrl(url)
 
+        if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  FanFiction.net Site Error!" % url)
+        
         # some ancient stories have body tags inside them that cause
         # soup parsing to discard the content.  For story text we
         # don't care about anything before "<div class='storytextp"
         # (there's a space after storytextp, so no close quote(')) and
         # this kills any body tags.
-        data = data[data.index("<div class='storytextp"):]
+        if "<div class='storytextp" not in data:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        else:
+            data = data[data.index("<div class='storytextp"):]
         data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
         
         soup = bs.BeautifulSoup(data)

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -67,6 +67,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.fanfiktion.de'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
 

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -69,8 +69,9 @@ class FicBookNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.ficbook.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/readfic/12345"
+        return "http://"+self.getSiteDomain()+"/readfic/12345 http://"+self.getSiteDomain()+"/readfic/93626/246417#part_content"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -57,6 +57,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.fictionalley.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/authors/drt/DA.html http://"+self.getSiteDomain()+"/authors/drt/JOTP01a.html"
 

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -39,8 +39,9 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.fictionpress.com','m.fictionpress.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title"
+        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
 
     def getSiteURLPattern(self):
         return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -45,8 +45,9 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'ficwad.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://ficwad.com/story/137169"
+        return "http://ficwad.com/story/1234"
 
     def getSiteURLPattern(self):
         return re.escape(r"http://"+self.getSiteDomain())+"/story/\d+?$"

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -51,6 +51,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         # mobile.fimifction.com isn't actually a valid domain, but we can still get the story id from URLs anyway
         return ['www.fimfiction.net','mobile.fimfiction.net', 'www.fimfiction.com', 'mobile.fimfiction.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"
 

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -67,8 +67,9 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'finestories.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=10537"
+        return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -74,6 +74,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -56,8 +56,9 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.harrypotterfanfiction.com','harrypotterfanfiction.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234 http://harrypotterfanfiction.com/viewstory.php?psid=5678"
+        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("harrypotterfanfiction.com/viewstory.php?psid=")+r"\d+$"

fanficdownloader/adapters/adapter_hennethannunnet.py

@@ -65,6 +65,7 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.henneth-annun.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
 

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -65,6 +65,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'hlfiction.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -70,6 +70,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'www.hpfandom.net' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -65,6 +65,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.hpfanficarchive.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -65,6 +65,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.ik-eternal.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -65,6 +65,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'imagine.e-fic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_indeathnet.py

@@ -72,6 +72,7 @@ class InDeathNetAdapter(BaseSiteAdapter):
         return 'indeath.net'
 
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/blog/archive/123-story-in-death/"
 

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -74,6 +74,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'ksarchive.com' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -66,6 +66,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.libraryofmoria.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/a/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -66,6 +66,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'lumos.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -55,6 +55,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.mediaminer.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+self.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
 

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -65,6 +65,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'merlinfic.dtwins.co.uk'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -70,6 +70,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'www.midnightwhispers.ca' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -66,14 +66,19 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfiction.mugglenet.com' # XXX
+        # The site domain.
+        return 'fanfiction.mugglenet.com'
 
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['fanfiction.mugglenet.com','fanfic.mugglenet.com']
+
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://")+r"fanfic(tion)?\.mugglenet\.com"+re.escape("/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -68,8 +68,10 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.national-library.net','national-library.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+        # ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: 
+        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -68,6 +68,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.ncisfic.com','ncisfic.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
 

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -79,6 +79,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.ncisfiction.net','www.ncisfiction.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
 

fanficdownloader/adapters/adapter_netraptororg.py

@@ -63,6 +63,7 @@ class NetRaptorOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'netraptor.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -74,6 +74,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'nfacommunity.com' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -66,6 +66,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'nha.magical-worlds.us'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -66,6 +66,7 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.nickandgreg.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py

@@ -66,6 +66,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'occlumency.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -70,6 +70,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com']
 
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -65,6 +65,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.phoenixsong.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/"
 
@@ -112,6 +113,8 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         logger.debug("URL: "+url)
 
         try:
+            if self.getConfig('force_login'):
+                self.performLogin(url)
             data = self._fetchUrl(url)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -123,7 +126,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
             # need to log in for this one.
             self.performLogin(url)
             data = self._fetchUrl(url)
-			
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)

fanficdownloader/adapters/adapter_pommedesangcom.py

@@ -75,6 +75,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'pommedesang.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/sds/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -70,8 +70,9 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234 http://explicit."+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -72,6 +72,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'fanfiction.portkey.org' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story/1234"
 

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -55,8 +55,9 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.potionsandsnitches.net','potionsandsnitches.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234 http://potionsandsnitches.net/fanfiction/viewstory.php?sid=5678"
+        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_potterficscom.py

@@ -72,13 +72,9 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.potterfics.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return \
-                "http://www.potterfics.com/historias/127583 "\
-                "http://www.potterfics.com/historias/127583/capitulo-1 "\
-                "http://www.potterfics.com/historias/127583/capitulo-4 "\
-                "http://www.potterfics.com/historias/92810 "\
-                "http://www.potterfics.com/historias/111194"
+        return "http://www.potterfics.com/historias/12345 http://www.potterfics.com/historias/12345/capitulo-1 "
 
     def getSiteURLPattern(self):
         #http://www.potterfics.com/historias/127583

fanficdownloader/adapters/adapter_potterheadsanonymouscom.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PotterHeadsAnonymousComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pha')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfic.potterheadsanonymous.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -69,6 +69,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.pretendercentre.com','pretendercentre.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_psychficcom.py

@@ -65,6 +65,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.psychfic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_qafficcom.py

@@ -65,6 +65,7 @@ class QafFicComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.qaf-fic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_restrictedsectionorg.py

@@ -70,6 +70,7 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.restrictedsection.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story.php?story=1234 http://"+self.getSiteDomain()+"/file.php?file=1234"
 

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -69,6 +69,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.samdean.archive.nu','samdean.archive.nu']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_scarheadnet.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ScarHeadNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ScarHeadNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','shn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'scarhead.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('tr',{'valign':'top'})
+        
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        
+        cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            if '/' == cat.string[0]:
+                self.story.addToList('ships','Harry Potter'+cat.string.split('(')[0])
+            elif 'Harry' in cat.string:
+                self.story.addToList('ships',cat.string.split('(')[0])
+            else:
+                self.story.addToList('category',cat.string)
+            if '(' in cat.string:
+                self.story.addToList('category',cat.string.split('(')[1].split(')')[0])
+            
+                
+            
+
+        chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+
+        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)
+            
+        textsoup = stripHTML(soup)
+
+        a = textsoup.split('Published: ')[1].split(' ')[0]
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a), self.dateformat))
+        a = textsoup.split('Updated: ')[1].split(' ')[0]
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a), self.dateformat))
+        a = textsoup.split('Rating: ')[1].split(' ')[0]
+        self.story.setMetadata('rating', a)
+        a = textsoup.split('Length: ')[1].split('(')[1].split(' ')[0]
+        self.story.setMetadata('numWords', a)
+        a = textsoup.split('Completed: ')[1].split(' ')[0]
+        if 'Yes' in a:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        #a = textsoup.split('Summary: ')[1].split('Add Story to Favorites')[0]
+        #self.setDescription(url,a)
+        
+
+        
+        a=soup.find(text=re.compile("Summary: "))
+        i=0
+        svalue = ""
+        while i == 0:
+            try:
+                b = str(a)
+                svalue += b.split('Summary: ')[1]
+            except:
+                svalue += str(a)
+            if a.nextSibling != None:
+                a = a.nextSibling
+            else:
+                a = a.parent.nextSibling
+            if 'Disclaimer: ' in stripHTML(a):
+                i=1
+        self.setDescription(url,svalue)
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -65,6 +65,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.scarvesandcoffee.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -74,6 +74,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'sg1-heliopolis.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_simplyundeniablecom.py

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SimplyUndeniableComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SimplyUndeniableComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sud')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.simplyundeniable.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Please log in now' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        if "Please log in now" in data:
+                raise exceptions.FailedToDownload(self.getSiteDomain() +" says: You need to have access to the restricted section.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for info in asoup.findAll('table', {'cellpadding' : '5'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.string)
+                break
+                
+        # Find the chapters:
+        if "Disclaimer" in data:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')):
+            # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        
+        a = info.find('td', {'valign' : 'top'}).find('p')
+        self.setDescription(url,a)
+        
+        a = info.find('td', {'class' : 'greysm'}).findAll('b')
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a[0].nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].nextSibling), self.dateformat))
+        if 'Yes' in a[2].nextSibling:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        self.story.setMetadata('rating', a[3].nextSibling)
+        self.story.setMetadata('numWords', a[4].nextSibling)
+        
+        
+        warnings = info.find('td', {'width' : '45'}).nextSibling.nextSibling.text.split(', ')
+        for warning in warnings:
+            if 'none' not in warning:
+                self.story.addToList('warnings',warning)
+        chars = info.find('td', {'width' : '51'}).nextSibling.nextSibling.text.split(', ')
+        for char in chars:
+            if '&' in char:
+                self.story.addToList('ships',char)
+            else:
+                self.story.addToList('characters',char)
+        genres = info.find('td', {'width' : '36'}).nextSibling.nextSibling.text.split(', ')
+        for genre in genres:
+            self.story.addToList('genre',genre)
+            
+        cat = info.find('a', href=re.compile(r'categories.php'))
+        self.story.addToList('category',cat.string)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('span', {'class' : 'style'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -65,6 +65,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.sinful-desire.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_siyecouk.py

@@ -70,6 +70,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
     def getAcceptDomains(cls):
         return ['www.siye.co.uk','siye.co.uk']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -81,6 +81,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # The config section name.  Only override if != site domain.
         return cls.getSiteDomain()+'/peja'
     
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_stargateatlantisorg.py

@@ -65,6 +65,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'stargate-atlantis.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_storiesofardacom.py

@@ -65,6 +65,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.storiesofarda.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=1234"
 

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -55,6 +55,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'fanfiction.tenhawkpresents.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_test1.py

@@ -40,6 +40,7 @@ class TestSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'test1.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"?sid=1234"
 
@@ -140,14 +141,66 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.setMetadata('rating','Tweenie')
 
         if idstr == '673':
-            self.story.addToList('author','Author From List')
+            self.story.addToList('author','Author From List 1')
             self.story.addToList('author','Author From List 2')
+            self.story.addToList('author','Author From List 3')
+            self.story.addToList('author','Author From List 4')
+            self.story.addToList('author','Author From List 5')
+            self.story.addToList('author','Author From List 6')
+            self.story.addToList('author','Author From List 7')
+            self.story.addToList('author','Author From List 8')
+            self.story.addToList('author','Author From List 9')
+            self.story.addToList('author','Author From List 0')
+            self.story.addToList('author','Author From List q')
+            self.story.addToList('author','Author From List w')
+            self.story.addToList('author','Author From List e')
+            self.story.addToList('author','Author From List r')
+            self.story.addToList('author','Author From List t')
+            self.story.addToList('author','Author From List y')
+            self.story.addToList('author','Author From List u')
+            self.story.addToList('author','Author From List i')
+            self.story.addToList('author','Author From List o')
         
-            self.story.addToList('authorId','98765')
+            self.story.addToList('authorId','98765-1')
             self.story.addToList('authorId','98765-2')
+            self.story.addToList('authorId','98765-3')
+            self.story.addToList('authorId','98765-4')
+            self.story.addToList('authorId','98765-5')
+            self.story.addToList('authorId','98765-6')
+            self.story.addToList('authorId','98765-7')
+            self.story.addToList('authorId','98765-8')
+            self.story.addToList('authorId','98765-9')
+            self.story.addToList('authorId','98765-0')
+            self.story.addToList('authorId','98765-q')
+            self.story.addToList('authorId','98765-w')
+            self.story.addToList('authorId','98765-e')
+            self.story.addToList('authorId','98765-r')
+            self.story.addToList('authorId','98765-t')
+            self.story.addToList('authorId','98765-y')
+            self.story.addToList('authorId','98765-u')
+            self.story.addToList('authorId','98765-i')
+            self.story.addToList('authorId','98765-o')
         
-            self.story.addToList('authorUrl','http://author/url')
+            self.story.addToList('authorUrl','http://author/url-1')
             self.story.addToList('authorUrl','http://author/url-2')
+            self.story.addToList('authorUrl','http://author/url-3')
+            self.story.addToList('authorUrl','http://author/url-4')
+            self.story.addToList('authorUrl','http://author/url-5')
+            self.story.addToList('authorUrl','http://author/url-6')
+            self.story.addToList('authorUrl','http://author/url-7')
+            self.story.addToList('authorUrl','http://author/url-8')
+            self.story.addToList('authorUrl','http://author/url-9')
+            self.story.addToList('authorUrl','http://author/url-0')
+            self.story.addToList('authorUrl','http://author/url-q')
+            self.story.addToList('authorUrl','http://author/url-w')
+            self.story.addToList('authorUrl','http://author/url-e')
+            self.story.addToList('authorUrl','http://author/url-r')
+            self.story.addToList('authorUrl','http://author/url-t')
+            self.story.addToList('authorUrl','http://author/url-y')
+            self.story.addToList('authorUrl','http://author/url-u')
+            self.story.addToList('authorUrl','http://author/url-i')
+            self.story.addToList('authorUrl','http://author/url-o')
+
             self.story.addToList('category','Power Rangers')
             self.story.addToList('category','SG-1')
             self.story.addToList('genre','Porn')
@@ -160,13 +213,13 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('warnings','Violence')
 
         if idstr == '80':
-            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('category',u'Rizzoli & Isles')
             self.story.addToList('characters','J. Rizzoli')
         elif idstr == '81':
             self.story.addToList('category',u'Pitch Perfect')
             self.story.addToList('characters','Chloe B.')
         elif idstr == '83':
-            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('category',u'Rizzoli & Isles')
             self.story.addToList('characters','J. Rizzoli')
             self.story.addToList('category',u'Pitch Perfect')
             self.story.addToList('characters','Chloe B.')
@@ -184,6 +237,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
             self.story.addToList('characters','Bob Smith')
             self.story.addToList('characters','George Johnson')
             self.story.addToList('characters','Fred Smythe')
+            self.story.addToList('ships','Harry Potter/Ginny Weasley')
+            self.story.addToList('ships','Harry Potter/Ginny Weasley/Albus Dumbledore')
+            self.story.addToList('ships','Harry Potter & Hermione Granger')
             
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','Comedy')
@@ -294,7 +350,7 @@ Don't&#8212e;ver&#8212d;o—that&#8212a;gain, 法 é
 <hr>
 horizontal rules
 <hr size=1 noshade>
-<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<p>"Lorem ipsum dolor sit amet", consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore--et dolore magna aliqua. 'Ut enim ad minim veniam', quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 </div>
 '''

fanficdownloader/adapters/adapter_thealphagatecom.py

@@ -65,6 +65,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.thealphagate.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -69,6 +69,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.thehexfiles.net','thehexfiles.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_thehookupzonenet.py

@@ -90,6 +90,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'thehookupzone.net' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_themasquenet.py

@@ -72,6 +72,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'themasque.net'
         
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://themasque.net/wiktt/efiction/viewstory.php?sid=1234 http://themasque.net/efiction/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_thepetulantpoetesscom.py

@@ -65,6 +65,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.thepetulantpoetess.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -74,6 +74,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
     def getAcceptDomains(cls):
         return ['www.thequidditchpitch.org','thequidditchpitch.org']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -55,6 +55,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.thewriterscoffeeshop.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/library/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_tokrafandomnetcom.py

@@ -66,6 +66,7 @@ class TokraFandomnetComAdapter(BaseSiteAdapter):
         # doesn't matter too much anymore.
         return 'tokra.fandomnet.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -54,13 +54,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.tthfanfic.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm http://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
+        return "http://www.tthfanfic.org/Story-1234 http://www.tthfanfic.org/Story-1234/Author+Story+Title.htm http://www.tthfanfic.org/T-99999999/Story-1234-1/Author+Story+Title.htm http://www.tthfanfic.org/story.php?no=12345"
 
-    # http://www.tthfanfic.org/T-526321777848988007890480555880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm
-    # http://www.tthfanfic.org/Story-5583
-    # http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm
-    # http://www.tthfanfic.org/story.php?no=26093
+    # http://www.tthfanfic.org/T-999999999999/Story-12345-1/Author+Story+Title.htm
+    # http://www.tthfanfic.org/Story-12345
+    # http://www.tthfanfic.org/Story-12345/Author+Story+Title.htm
+    # http://www.tthfanfic.org/story.php?no=12345
     def getSiteURLPattern(self):
         return r"http://www.tthfanfic.org(/(T-\d+/)?Story-|/story.php\?no=)(?P<id>\d+)(-\d+)?(/.*)?$"
 

fanficdownloader/adapters/adapter_twilightarchivescom.py

@@ -66,6 +66,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         return 'www.twilightarchives.com'
        
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/read/1234"
 

fanficdownloader/adapters/adapter_twilightednet.py

@@ -57,8 +57,9 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.twilighted.net','twilighted.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.twilighted.net/viewstory.php?sid=1234 http://twilighted.net/viewstory.php?sid=5678"
+        return "http://www.twilighted.net/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("twilighted.net/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -58,8 +58,9 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.twiwrite.net','twiwrite.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.twiwrite.net/viewstory.php?sid=1234 http://twiwrite.net/viewstory.php?sid=5678"
+        return "http://www.twiwrite.net/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("twiwrite.net/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_walkingtheplankorg.py

@@ -63,6 +63,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.walkingtheplank.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_whoficcom.py

@@ -41,6 +41,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.whofic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_wizardtalesnet.py

@@ -65,6 +65,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.wizardtales.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"